Fossil SCM
In import --git store each artifact uncompressed, and then compress the ones that did not become deltas at the end of the import. Much more efficient than compressing all artifacts and then replacing most of them with compressed deltas.
Commit
99b0bd8f9d2246d6426ed2546ff1ffba9c1de9b1f8d888bead83b267bbd3f62b
Parent
b89bd70a3d091eb…
2 files changed
+12
-2
+12
+12
-2
| --- src/blob.c | ||
| +++ src/blob.c | ||
| @@ -1365,10 +1365,19 @@ | ||
| 1365 | 1365 | blob_size(pBlob), zFilename); |
| 1366 | 1366 | } |
| 1367 | 1367 | } |
| 1368 | 1368 | return nWrote; |
| 1369 | 1369 | } |
| 1370 | + | |
| 1371 | +/* | |
| 1372 | +** Set the zlib compression level used by blob_compress(). Level 0 | |
| 1373 | +** stores content without compressing it, and -1 is the zlib default. | |
| 1374 | +*/ | |
| 1375 | +static int blobCompressLevel = Z_DEFAULT_COMPRESSION; | |
| 1376 | +void blob_compress_level(int iLevel){ | |
| 1377 | + blobCompressLevel = iLevel; | |
| 1378 | +} | |
| 1370 | 1379 | |
| 1371 | 1380 | /* |
| 1372 | 1381 | ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and |
| 1373 | 1382 | ** pOut to be the same blob. |
| 1374 | 1383 | ** |
| @@ -1386,12 +1395,13 @@ | ||
| 1386 | 1395 | outBuf[0] = nIn>>24 & 0xff; |
| 1387 | 1396 | outBuf[1] = nIn>>16 & 0xff; |
| 1388 | 1397 | outBuf[2] = nIn>>8 & 0xff; |
| 1389 | 1398 | outBuf[3] = nIn & 0xff; |
| 1390 | 1399 | nOut2 = (long int)nOut; |
| 1391 | - compress(&outBuf[4], &nOut2, | |
| 1392 | - (unsigned char*)blob_buffer(pIn), blob_size(pIn)); | |
| 1400 | + compress2(&outBuf[4], &nOut2, | |
| 1401 | + (unsigned char*)blob_buffer(pIn), blob_size(pIn), | |
| 1402 | + blobCompressLevel); | |
| 1393 | 1403 | if( pOut==pIn ) blob_reset(pOut); |
| 1394 | 1404 | assert_blob_is_reset(pOut); |
| 1395 | 1405 | *pOut = temp; |
| 1396 | 1406 | blob_resize(pOut, nOut2+4); |
| 1397 | 1407 | } |
| 1398 | 1408 |
| --- src/blob.c | |
| +++ src/blob.c | |
| @@ -1365,10 +1365,19 @@ | |
| 1365 | blob_size(pBlob), zFilename); |
| 1366 | } |
| 1367 | } |
| 1368 | return nWrote; |
| 1369 | } |
| 1370 | |
| 1371 | /* |
| 1372 | ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and |
| 1373 | ** pOut to be the same blob. |
| 1374 | ** |
| @@ -1386,12 +1395,13 @@ | |
| 1386 | outBuf[0] = nIn>>24 & 0xff; |
| 1387 | outBuf[1] = nIn>>16 & 0xff; |
| 1388 | outBuf[2] = nIn>>8 & 0xff; |
| 1389 | outBuf[3] = nIn & 0xff; |
| 1390 | nOut2 = (long int)nOut; |
| 1391 | compress(&outBuf[4], &nOut2, |
| 1392 | (unsigned char*)blob_buffer(pIn), blob_size(pIn)); |
| 1393 | if( pOut==pIn ) blob_reset(pOut); |
| 1394 | assert_blob_is_reset(pOut); |
| 1395 | *pOut = temp; |
| 1396 | blob_resize(pOut, nOut2+4); |
| 1397 | } |
| 1398 |
| --- src/blob.c | |
| +++ src/blob.c | |
| @@ -1365,10 +1365,19 @@ | |
| 1365 | blob_size(pBlob), zFilename); |
| 1366 | } |
| 1367 | } |
| 1368 | return nWrote; |
| 1369 | } |
| 1370 | |
| 1371 | /* |
| 1372 | ** Set the zlib compression level used by blob_compress(). Level 0 |
| 1373 | ** stores content without compressing it, and -1 is the zlib default. |
| 1374 | */ |
| 1375 | static int blobCompressLevel = Z_DEFAULT_COMPRESSION; |
| 1376 | void blob_compress_level(int iLevel){ |
| 1377 | blobCompressLevel = iLevel; |
| 1378 | } |
| 1379 | |
| 1380 | /* |
| 1381 | ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and |
| 1382 | ** pOut to be the same blob. |
| 1383 | ** |
| @@ -1386,12 +1395,13 @@ | |
| 1395 | outBuf[0] = nIn>>24 & 0xff; |
| 1396 | outBuf[1] = nIn>>16 & 0xff; |
| 1397 | outBuf[2] = nIn>>8 & 0xff; |
| 1398 | outBuf[3] = nIn & 0xff; |
| 1399 | nOut2 = (long int)nOut; |
| 1400 | compress2(&outBuf[4], &nOut2, |
| 1401 | (unsigned char*)blob_buffer(pIn), blob_size(pIn), |
| 1402 | blobCompressLevel); |
| 1403 | if( pOut==pIn ) blob_reset(pOut); |
| 1404 | assert_blob_is_reset(pOut); |
| 1405 | *pOut = temp; |
| 1406 | blob_resize(pOut, nOut2+4); |
| 1407 | } |
| 1408 |
+12
| --- src/import.c | ||
| +++ src/import.c | ||
| @@ -147,11 +147,15 @@ | ||
| 147 | 147 | "INSERT INTO blob(uuid, size, rcvid, content)" |
| 148 | 148 | "VALUES(:uuid, :size, %d, :content)", g.rcvid |
| 149 | 149 | ); |
| 150 | 150 | db_bind_text(&ins, ":uuid", blob_str(&hash)); |
| 151 | 151 | db_bind_int(&ins, ":size", gg.nData); |
| 152 | + /* Most artifacts become deltas when a later version arrives, so | |
| 153 | + ** store them uncompressed, and import_cmd() compresses the rest. */ | |
| 154 | + blob_compress_level(0); | |
| 152 | 155 | blob_compress(pContent, &cmpr); |
| 156 | + blob_compress_level(-1); | |
| 153 | 157 | db_bind_blob(&ins, ":content", &cmpr); |
| 154 | 158 | db_step(&ins); |
| 155 | 159 | db_reset(&ins); |
| 156 | 160 | blob_reset(&cmpr); |
| 157 | 161 | rid = db_last_insert_rowid(); |
| @@ -2011,10 +2015,18 @@ | ||
| 2011 | 2015 | bag_clear(&vers); |
| 2012 | 2016 | } |
| 2013 | 2017 | manifest_crosslink_end(MC_NONE); |
| 2014 | 2018 | } |
| 2015 | 2019 | |
| 2020 | + /* Compress the artifacts that fast_insert_content() stored | |
| 2021 | + ** uncompressed and that did not become deltas. */ | |
| 2022 | + add_content_sql_commands(g.db); | |
| 2023 | + db_multi_exec( | |
| 2024 | + "UPDATE blob SET content=compress(decompress(content))" | |
| 2025 | + " WHERE rcvid=%d AND size>=0 AND rid NOT IN (SELECT rid FROM delta)", | |
| 2026 | + g.rcvid | |
| 2027 | + ); | |
| 2016 | 2028 | verify_cancel(); |
| 2017 | 2029 | db_end_transaction(0); |
| 2018 | 2030 | fossil_print(" \r"); |
| 2019 | 2031 | if( omitRebuild ){ |
| 2020 | 2032 | omitVacuum = 1; |
| 2021 | 2033 |
| --- src/import.c | |
| +++ src/import.c | |
| @@ -147,11 +147,15 @@ | |
| 147 | "INSERT INTO blob(uuid, size, rcvid, content)" |
| 148 | "VALUES(:uuid, :size, %d, :content)", g.rcvid |
| 149 | ); |
| 150 | db_bind_text(&ins, ":uuid", blob_str(&hash)); |
| 151 | db_bind_int(&ins, ":size", gg.nData); |
| 152 | blob_compress(pContent, &cmpr); |
| 153 | db_bind_blob(&ins, ":content", &cmpr); |
| 154 | db_step(&ins); |
| 155 | db_reset(&ins); |
| 156 | blob_reset(&cmpr); |
| 157 | rid = db_last_insert_rowid(); |
| @@ -2011,10 +2015,18 @@ | |
| 2011 | bag_clear(&vers); |
| 2012 | } |
| 2013 | manifest_crosslink_end(MC_NONE); |
| 2014 | } |
| 2015 | |
| 2016 | verify_cancel(); |
| 2017 | db_end_transaction(0); |
| 2018 | fossil_print(" \r"); |
| 2019 | if( omitRebuild ){ |
| 2020 | omitVacuum = 1; |
| 2021 |
| --- src/import.c | |
| +++ src/import.c | |
| @@ -147,11 +147,15 @@ | |
| 147 | "INSERT INTO blob(uuid, size, rcvid, content)" |
| 148 | "VALUES(:uuid, :size, %d, :content)", g.rcvid |
| 149 | ); |
| 150 | db_bind_text(&ins, ":uuid", blob_str(&hash)); |
| 151 | db_bind_int(&ins, ":size", gg.nData); |
| 152 | /* Most artifacts become deltas when a later version arrives, so |
| 153 | ** store them uncompressed, and import_cmd() compresses the rest. */ |
| 154 | blob_compress_level(0); |
| 155 | blob_compress(pContent, &cmpr); |
| 156 | blob_compress_level(-1); |
| 157 | db_bind_blob(&ins, ":content", &cmpr); |
| 158 | db_step(&ins); |
| 159 | db_reset(&ins); |
| 160 | blob_reset(&cmpr); |
| 161 | rid = db_last_insert_rowid(); |
| @@ -2011,10 +2015,18 @@ | |
| 2015 | bag_clear(&vers); |
| 2016 | } |
| 2017 | manifest_crosslink_end(MC_NONE); |
| 2018 | } |
| 2019 | |
| 2020 | /* Compress the artifacts that fast_insert_content() stored |
| 2021 | ** uncompressed and that did not become deltas. */ |
| 2022 | add_content_sql_commands(g.db); |
| 2023 | db_multi_exec( |
| 2024 | "UPDATE blob SET content=compress(decompress(content))" |
| 2025 | " WHERE rcvid=%d AND size>=0 AND rid NOT IN (SELECT rid FROM delta)", |
| 2026 | g.rcvid |
| 2027 | ); |
| 2028 | verify_cancel(); |
| 2029 | db_end_transaction(0); |
| 2030 | fossil_print(" \r"); |
| 2031 | if( omitRebuild ){ |
| 2032 | omitVacuum = 1; |
| 2033 |