Fossil SCM

In import --git store each artifact uncompressed, and then compress the ones that did not become deltas at the end of the import. Much more efficient than compressing all artifacts and then replacing most of them with compressed deltas.

danshearer 2026-10-26 22:30 UTC trunk
Commit 99b0bd8f9d2246d6426ed2546ff1ffba9c1de9b1f8d888bead83b267bbd3f62b
2 files changed +12 -2 +12
+12 -2
--- src/blob.c
+++ src/blob.c
@@ -1365,10 +1365,19 @@
13651365
blob_size(pBlob), zFilename);
13661366
}
13671367
}
13681368
return nWrote;
13691369
}
1370
+
1371
+/*
1372
+** Set the zlib compression level used by blob_compress(). Level 0
1373
+** stores content without compressing it, and -1 is the zlib default.
1374
+*/
1375
+static int blobCompressLevel = Z_DEFAULT_COMPRESSION;
1376
+void blob_compress_level(int iLevel){
1377
+ blobCompressLevel = iLevel;
1378
+}
13701379
13711380
/*
13721381
** Compress a blob pIn. Store the result in pOut. It is ok for pIn and
13731382
** pOut to be the same blob.
13741383
**
@@ -1386,12 +1395,13 @@
13861395
outBuf[0] = nIn>>24 & 0xff;
13871396
outBuf[1] = nIn>>16 & 0xff;
13881397
outBuf[2] = nIn>>8 & 0xff;
13891398
outBuf[3] = nIn & 0xff;
13901399
nOut2 = (long int)nOut;
1391
- compress(&outBuf[4], &nOut2,
1392
- (unsigned char*)blob_buffer(pIn), blob_size(pIn));
1400
+ compress2(&outBuf[4], &nOut2,
1401
+ (unsigned char*)blob_buffer(pIn), blob_size(pIn),
1402
+ blobCompressLevel);
13931403
if( pOut==pIn ) blob_reset(pOut);
13941404
assert_blob_is_reset(pOut);
13951405
*pOut = temp;
13961406
blob_resize(pOut, nOut2+4);
13971407
}
13981408
--- src/blob.c
+++ src/blob.c
@@ -1365,10 +1365,19 @@
1365 blob_size(pBlob), zFilename);
1366 }
1367 }
1368 return nWrote;
1369 }
 
 
 
 
 
 
 
 
 
1370
1371 /*
1372 ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and
1373 ** pOut to be the same blob.
1374 **
@@ -1386,12 +1395,13 @@
1386 outBuf[0] = nIn>>24 & 0xff;
1387 outBuf[1] = nIn>>16 & 0xff;
1388 outBuf[2] = nIn>>8 & 0xff;
1389 outBuf[3] = nIn & 0xff;
1390 nOut2 = (long int)nOut;
1391 compress(&outBuf[4], &nOut2,
1392 (unsigned char*)blob_buffer(pIn), blob_size(pIn));
 
1393 if( pOut==pIn ) blob_reset(pOut);
1394 assert_blob_is_reset(pOut);
1395 *pOut = temp;
1396 blob_resize(pOut, nOut2+4);
1397 }
1398
--- src/blob.c
+++ src/blob.c
@@ -1365,10 +1365,19 @@
1365 blob_size(pBlob), zFilename);
1366 }
1367 }
1368 return nWrote;
1369 }
1370
1371 /*
1372 ** Set the zlib compression level used by blob_compress(). Level 0
1373 ** stores content without compressing it, and -1 is the zlib default.
1374 */
1375 static int blobCompressLevel = Z_DEFAULT_COMPRESSION;
1376 void blob_compress_level(int iLevel){
1377 blobCompressLevel = iLevel;
1378 }
1379
1380 /*
1381 ** Compress a blob pIn. Store the result in pOut. It is ok for pIn and
1382 ** pOut to be the same blob.
1383 **
@@ -1386,12 +1395,13 @@
1395 outBuf[0] = nIn>>24 & 0xff;
1396 outBuf[1] = nIn>>16 & 0xff;
1397 outBuf[2] = nIn>>8 & 0xff;
1398 outBuf[3] = nIn & 0xff;
1399 nOut2 = (long int)nOut;
1400 compress2(&outBuf[4], &nOut2,
1401 (unsigned char*)blob_buffer(pIn), blob_size(pIn),
1402 blobCompressLevel);
1403 if( pOut==pIn ) blob_reset(pOut);
1404 assert_blob_is_reset(pOut);
1405 *pOut = temp;
1406 blob_resize(pOut, nOut2+4);
1407 }
1408
+12
--- src/import.c
+++ src/import.c
@@ -147,11 +147,15 @@
147147
"INSERT INTO blob(uuid, size, rcvid, content)"
148148
"VALUES(:uuid, :size, %d, :content)", g.rcvid
149149
);
150150
db_bind_text(&ins, ":uuid", blob_str(&hash));
151151
db_bind_int(&ins, ":size", gg.nData);
152
+ /* Most artifacts become deltas when a later version arrives, so
153
+ ** store them uncompressed, and import_cmd() compresses the rest. */
154
+ blob_compress_level(0);
152155
blob_compress(pContent, &cmpr);
156
+ blob_compress_level(-1);
153157
db_bind_blob(&ins, ":content", &cmpr);
154158
db_step(&ins);
155159
db_reset(&ins);
156160
blob_reset(&cmpr);
157161
rid = db_last_insert_rowid();
@@ -2011,10 +2015,18 @@
20112015
bag_clear(&vers);
20122016
}
20132017
manifest_crosslink_end(MC_NONE);
20142018
}
20152019
2020
+ /* Compress the artifacts that fast_insert_content() stored
2021
+ ** uncompressed and that did not become deltas. */
2022
+ add_content_sql_commands(g.db);
2023
+ db_multi_exec(
2024
+ "UPDATE blob SET content=compress(decompress(content))"
2025
+ " WHERE rcvid=%d AND size>=0 AND rid NOT IN (SELECT rid FROM delta)",
2026
+ g.rcvid
2027
+ );
20162028
verify_cancel();
20172029
db_end_transaction(0);
20182030
fossil_print(" \r");
20192031
if( omitRebuild ){
20202032
omitVacuum = 1;
20212033
--- src/import.c
+++ src/import.c
@@ -147,11 +147,15 @@
147 "INSERT INTO blob(uuid, size, rcvid, content)"
148 "VALUES(:uuid, :size, %d, :content)", g.rcvid
149 );
150 db_bind_text(&ins, ":uuid", blob_str(&hash));
151 db_bind_int(&ins, ":size", gg.nData);
 
 
 
152 blob_compress(pContent, &cmpr);
 
153 db_bind_blob(&ins, ":content", &cmpr);
154 db_step(&ins);
155 db_reset(&ins);
156 blob_reset(&cmpr);
157 rid = db_last_insert_rowid();
@@ -2011,10 +2015,18 @@
2011 bag_clear(&vers);
2012 }
2013 manifest_crosslink_end(MC_NONE);
2014 }
2015
 
 
 
 
 
 
 
 
2016 verify_cancel();
2017 db_end_transaction(0);
2018 fossil_print(" \r");
2019 if( omitRebuild ){
2020 omitVacuum = 1;
2021
--- src/import.c
+++ src/import.c
@@ -147,11 +147,15 @@
147 "INSERT INTO blob(uuid, size, rcvid, content)"
148 "VALUES(:uuid, :size, %d, :content)", g.rcvid
149 );
150 db_bind_text(&ins, ":uuid", blob_str(&hash));
151 db_bind_int(&ins, ":size", gg.nData);
152 /* Most artifacts become deltas when a later version arrives, so
153 ** store them uncompressed, and import_cmd() compresses the rest. */
154 blob_compress_level(0);
155 blob_compress(pContent, &cmpr);
156 blob_compress_level(-1);
157 db_bind_blob(&ins, ":content", &cmpr);
158 db_step(&ins);
159 db_reset(&ins);
160 blob_reset(&cmpr);
161 rid = db_last_insert_rowid();
@@ -2011,10 +2015,18 @@
2015 bag_clear(&vers);
2016 }
2017 manifest_crosslink_end(MC_NONE);
2018 }
2019
2020 /* Compress the artifacts that fast_insert_content() stored
2021 ** uncompressed and that did not become deltas. */
2022 add_content_sql_commands(g.db);
2023 db_multi_exec(
2024 "UPDATE blob SET content=compress(decompress(content))"
2025 " WHERE rcvid=%d AND size>=0 AND rid NOT IN (SELECT rid FROM delta)",
2026 g.rcvid
2027 );
2028 verify_cancel();
2029 db_end_transaction(0);
2030 fossil_print(" \r");
2031 if( omitRebuild ){
2032 omitVacuum = 1;
2033

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button