Fossil SCM

The extra-delta-compression step now reports the number of new deltas added and the bytes of storage space saved using those deltas.

drh 2023-02-09 19:45 trunk
Commit 59e21eb3e53ca2cc4299728b205f7f4703b8bcdf4e58eb7a133226c9c76ee00c
Parent 7449a10f2a533bb…
3 files changed +8 -2 +4 -2 +30 -8
~ src/clone.c ~ src/content.c ~ src/rebuild.c
M src/clone.c
+8 -2
--- src/clone.c
+++ src/clone.c
@@ -279,13 +279,19 @@
279279
 " the clone is probably incomplete and unusable.");
280280
 }
281281
 fossil_print("Rebuilding repository meta-data...\n");
282282
 rebuild_db(1, 0);
283283
 if( !noCompress ){
284
+ int nDelta = 0;
285
+ i64 nByte;
284286
 fossil_print("Extra delta compression... "); fflush(stdout);
285
- extra_deltification();
286
- fossil_print("\n");
287
+ nByte = extra_deltification(&nDelta);
288
+ if( nDelta ){
289
+ fossil_print("%d deltas save %,lld bytes\n", nDelta, nByte);
290
+ }else{
291
+ fossil_print("no extra compression found\n");
292
+ }
287293
 }
288294
 db_end_transaction(0);
289295
 fossil_print("Vacuuming the database... "); fflush(stdout);
290296
 if( db_int(0, "PRAGMA page_count")>1000
291297
 && db_int(0, "PRAGMA page_size")<8192 ){
292298
--- src/clone.c
+++ src/clone.c
@@ -279,13 +279,19 @@
279 " the clone is probably incomplete and unusable.");
280 }
281 fossil_print("Rebuilding repository meta-data...\n");
282 rebuild_db(1, 0);
283 if( !noCompress ){
 
 
284 fossil_print("Extra delta compression... "); fflush(stdout);
285 extra_deltification();
286 fossil_print("\n");
 
 
 
 
287 }
288 db_end_transaction(0);
289 fossil_print("Vacuuming the database... "); fflush(stdout);
290 if( db_int(0, "PRAGMA page_count")>1000
291 && db_int(0, "PRAGMA page_size")<8192 ){
292
--- src/clone.c
+++ src/clone.c
@@ -279,13 +279,19 @@
279 " the clone is probably incomplete and unusable.");
280 }
281 fossil_print("Rebuilding repository meta-data...\n");
282 rebuild_db(1, 0);
283 if( !noCompress ){
284 int nDelta = 0;
285 i64 nByte;
286 fossil_print("Extra delta compression... "); fflush(stdout);
287 nByte = extra_deltification(&nDelta);
288 if( nDelta ){
289 fossil_print("%d deltas save %,lld bytes\n", nDelta, nByte);
290 }else{
291 fossil_print("no extra compression found\n");
292 }
293 }
294 db_end_transaction(0);
295 fossil_print("Vacuuming the database... "); fflush(stdout);
296 if( db_int(0, "PRAGMA page_count")>1000
297 && db_int(0, "PRAGMA page_size")<8192 ){
298
M src/content.c
+4 -2
--- src/content.c
+++ src/content.c
@@ -814,11 +814,12 @@
814814
 **
815815
 ** If either rid or aSrc[i] contain less than 50 bytes, or if the
816816
 ** resulting delta does not achieve a compression of at least 25%
817817
 ** the rid is left untouched.
818818
 **
819
-** Return 1 if a delta is made and 0 if no delta occurs.
819
+** Return the number of bytes by which the storage associated with rid
820
+** is reduced. A return of 0 means no new deltification occurs.
820821
 */
821822
 int content_deltify(int rid, int *aSrc, int nSrc, int force){
822823
 int s;
823824
 Blob data; /* Content of rid */
824825
 Blob src; /* Content of aSrc[i] */
@@ -903,17 +904,18 @@
903904
 blob_compress(&bestDelta, &bestDelta);
904905
 db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
905906
 db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, bestSrc);
906907
 db_bind_blob(&s1, ":data", &bestDelta);
907908
 db_begin_transaction();
909
+ rc = db_int(0, "SELECT length(content) FROM blob WHERE rid=%d", rid);
908910
 db_exec(&s1);
909911
 db_exec(&s2);
910912
 db_end_transaction(0);
911913
 db_finalize(&s1);
912914
 db_finalize(&s2);
913915
 verify_before_commit(rid);
914
- rc = 1;
916
+ rc -= blob_size(&bestDelta);
915917
 }
916918
 blob_reset(&data);
917919
 blob_reset(&bestDelta);
918920
 return rc;
919921
 }
920922
--- src/content.c
+++ src/content.c
@@ -814,11 +814,12 @@
814 **
815 ** If either rid or aSrc[i] contain less than 50 bytes, or if the
816 ** resulting delta does not achieve a compression of at least 25%
817 ** the rid is left untouched.
818 **
819 ** Return 1 if a delta is made and 0 if no delta occurs.
 
820 */
821 int content_deltify(int rid, int *aSrc, int nSrc, int force){
822 int s;
823 Blob data; /* Content of rid */
824 Blob src; /* Content of aSrc[i] */
@@ -903,17 +904,18 @@
903 blob_compress(&bestDelta, &bestDelta);
904 db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
905 db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, bestSrc);
906 db_bind_blob(&s1, ":data", &bestDelta);
907 db_begin_transaction();
 
908 db_exec(&s1);
909 db_exec(&s2);
910 db_end_transaction(0);
911 db_finalize(&s1);
912 db_finalize(&s2);
913 verify_before_commit(rid);
914 rc = 1;
915 }
916 blob_reset(&data);
917 blob_reset(&bestDelta);
918 return rc;
919 }
920
--- src/content.c
+++ src/content.c
@@ -814,11 +814,12 @@
814 **
815 ** If either rid or aSrc[i] contain less than 50 bytes, or if the
816 ** resulting delta does not achieve a compression of at least 25%
817 ** the rid is left untouched.
818 **
819 ** Return the number of bytes by which the storage associated with rid
820 ** is reduced. A return of 0 means no new deltification occurs.
821 */
822 int content_deltify(int rid, int *aSrc, int nSrc, int force){
823 int s;
824 Blob data; /* Content of rid */
825 Blob src; /* Content of aSrc[i] */
@@ -903,17 +904,18 @@
904 blob_compress(&bestDelta, &bestDelta);
905 db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
906 db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, bestSrc);
907 db_bind_blob(&s1, ":data", &bestDelta);
908 db_begin_transaction();
909 rc = db_int(0, "SELECT length(content) FROM blob WHERE rid=%d", rid);
910 db_exec(&s1);
911 db_exec(&s2);
912 db_end_transaction(0);
913 db_finalize(&s1);
914 db_finalize(&s2);
915 verify_before_commit(rid);
916 rc -= blob_size(&bestDelta);
917 }
918 blob_reset(&data);
919 blob_reset(&bestDelta);
920 return rc;
921 }
922
M src/rebuild.c
+30 -8
--- src/rebuild.c
+++ src/rebuild.c
@@ -488,18 +488,22 @@
488488
 */
489489
 #define N_NEIGHBOR 5
490490
491491
 /*
492492
 ** Attempt to convert more full-text blobs into delta-blobs for
493
-** storage efficiency.
493
+** storage efficiency. Return the number of bytes of storage space
494
+** saved.
494495
 */
495
-void extra_deltification(void){
496
+i64 extra_deltification(int *pnDelta){
496497
 Stmt q;
497498
 int aPrev[N_NEIGHBOR];
498499
 int nPrev;
499500
 int rid;
500501
 int prevfnid, fnid;
502
+ int nDelta = 0;
503
+ i64 nByte = 0;
504
+ int nSaved;
501505
 db_begin_transaction();
502506
503507
 /* Look for manifests that have not been deltaed and try to make them
504508
 ** children of one of the 5 chronologically subsequent check-ins
505509
 */
@@ -512,11 +516,15 @@
512516
 );
513517
 nPrev = 0;
514518
 while( db_step(&q)==SQLITE_ROW ){
515519
 rid = db_column_int(&q, 0);
516520
 if( nPrev>0 ){
517
- content_deltify(rid, aPrev, nPrev, 0);
521
+ nSaved = content_deltify(rid, aPrev, nPrev, 0);
522
+ if( nSaved>0 ){
523
+ nDelta++;
524
+ nByte += nSaved;
525
+ }
518526
 }
519527
 if( nPrev<N_NEIGHBOR ){
520528
 aPrev[nPrev++] = rid;
521529
 }else{
522530
 int i;
@@ -543,11 +551,15 @@
543551
 rid = db_column_int(&q, 0);
544552
 fnid = db_column_int(&q, 1);
545553
 if( fnid!=prevfnid ) nPrev = 0;
546554
 prevfnid = fnid;
547555
 if( nPrev>0 ){
548
- content_deltify(rid, aPrev, nPrev, 0);
556
+ nSaved = content_deltify(rid, aPrev, nPrev, 0);
557
+ if( nSaved>0 ){
558
+ nDelta++;
559
+ nByte += nSaved;
560
+ }
549561
 }
550562
 if( nPrev<N_NEIGHBOR ){
551563
 aPrev[nPrev++] = rid;
552564
 }else{
553565
 int i;
@@ -556,10 +568,12 @@
556568
 }
557569
 }
558570
 db_finalize(&q);
559571
560572
 db_end_transaction(0);
573
+ if( pnDelta!=0 ) *pnDelta = nDelta;
574
+ return nByte;
561575
 }
562576
563577
564578
 /* Reconstruct the private table. The private table contains the rid
565579
 ** of every manifest that is tagged with "private" and every file that
@@ -636,11 +650,11 @@
636650
 showStats = find_option("stats",0,0)!=0;
637651
 optIndex = find_option("index",0,0)!=0;
638652
 optNoIndex = find_option("noindex",0,0)!=0;
639653
 optIfNeeded = find_option("ifneeded",0,0)!=0;
640654
 compressOnlyFlag = find_option("compress-only",0,0)!=0;
641
- if( compressOnlyFlag ) runCompress = runVacuum = 1;
655
+ if( compressOnlyFlag ) runCompress = 1;
642656
 if( zPagesize ){
643657
 newPagesize = atoi(zPagesize);
644658
 if( newPagesize<512 || newPagesize>65536
645659
 || (newPagesize&(newPagesize-1))!=0
646660
 ){
@@ -688,17 +702,25 @@
688702
 errCnt
689703
 );
690704
 db_end_transaction(1);
691705
 }else{
692706
 if( runCompress ){
707
+ i64 nByte = 0;
708
+ int nDelta = 0;
693709
 fossil_print("Extra delta compression... "); fflush(stdout);
694
- extra_deltification();
695
- runVacuum = 1;
710
+ nByte = extra_deltification(&nDelta);
711
+ if( nDelta>0 ){
712
+ fossil_print("%d new deltas save %,lld bytes", nDelta, nByte);
713
+ runVacuum = 1;
714
+ }else{
715
+ fossil_print("no additional compression found");
716
+ }
717
+ fflush(stdout);
696718
 }
697719
 if( omitVerify ) verify_cancel();
698720
 db_end_transaction(0);
699
- if( runCompress ) fossil_print("done\n");
721
+ if( runCompress ) fossil_print("\n");
700722
 db_close(0);
701723
 db_open_repository(g.zRepositoryName);
702724
 if( newPagesize ){
703725
 db_multi_exec("PRAGMA page_size=%d", newPagesize);
704726
 runVacuum = 1;
705727
--- src/rebuild.c
+++ src/rebuild.c
@@ -488,18 +488,22 @@
488 */
489 #define N_NEIGHBOR 5
490
491 /*
492 ** Attempt to convert more full-text blobs into delta-blobs for
493 ** storage efficiency.
 
494 */
495 void extra_deltification(void){
496 Stmt q;
497 int aPrev[N_NEIGHBOR];
498 int nPrev;
499 int rid;
500 int prevfnid, fnid;
 
 
 
501 db_begin_transaction();
502
503 /* Look for manifests that have not been deltaed and try to make them
504 ** children of one of the 5 chronologically subsequent check-ins
505 */
@@ -512,11 +516,15 @@
512 );
513 nPrev = 0;
514 while( db_step(&q)==SQLITE_ROW ){
515 rid = db_column_int(&q, 0);
516 if( nPrev>0 ){
517 content_deltify(rid, aPrev, nPrev, 0);
 
 
 
 
518 }
519 if( nPrev<N_NEIGHBOR ){
520 aPrev[nPrev++] = rid;
521 }else{
522 int i;
@@ -543,11 +551,15 @@
543 rid = db_column_int(&q, 0);
544 fnid = db_column_int(&q, 1);
545 if( fnid!=prevfnid ) nPrev = 0;
546 prevfnid = fnid;
547 if( nPrev>0 ){
548 content_deltify(rid, aPrev, nPrev, 0);
 
 
 
 
549 }
550 if( nPrev<N_NEIGHBOR ){
551 aPrev[nPrev++] = rid;
552 }else{
553 int i;
@@ -556,10 +568,12 @@
556 }
557 }
558 db_finalize(&q);
559
560 db_end_transaction(0);
 
 
561 }
562
563
564 /* Reconstruct the private table. The private table contains the rid
565 ** of every manifest that is tagged with "private" and every file that
@@ -636,11 +650,11 @@
636 showStats = find_option("stats",0,0)!=0;
637 optIndex = find_option("index",0,0)!=0;
638 optNoIndex = find_option("noindex",0,0)!=0;
639 optIfNeeded = find_option("ifneeded",0,0)!=0;
640 compressOnlyFlag = find_option("compress-only",0,0)!=0;
641 if( compressOnlyFlag ) runCompress = runVacuum = 1;
642 if( zPagesize ){
643 newPagesize = atoi(zPagesize);
644 if( newPagesize<512 || newPagesize>65536
645 || (newPagesize&(newPagesize-1))!=0
646 ){
@@ -688,17 +702,25 @@
688 errCnt
689 );
690 db_end_transaction(1);
691 }else{
692 if( runCompress ){
 
 
693 fossil_print("Extra delta compression... "); fflush(stdout);
694 extra_deltification();
695 runVacuum = 1;
 
 
 
 
 
 
696 }
697 if( omitVerify ) verify_cancel();
698 db_end_transaction(0);
699 if( runCompress ) fossil_print("done\n");
700 db_close(0);
701 db_open_repository(g.zRepositoryName);
702 if( newPagesize ){
703 db_multi_exec("PRAGMA page_size=%d", newPagesize);
704 runVacuum = 1;
705
--- src/rebuild.c
+++ src/rebuild.c
@@ -488,18 +488,22 @@
488 */
489 #define N_NEIGHBOR 5
490
491 /*
492 ** Attempt to convert more full-text blobs into delta-blobs for
493 ** storage efficiency. Return the number of bytes of storage space
494 ** saved.
495 */
496 i64 extra_deltification(int *pnDelta){
497 Stmt q;
498 int aPrev[N_NEIGHBOR];
499 int nPrev;
500 int rid;
501 int prevfnid, fnid;
502 int nDelta = 0;
503 i64 nByte = 0;
504 int nSaved;
505 db_begin_transaction();
506
507 /* Look for manifests that have not been deltaed and try to make them
508 ** children of one of the 5 chronologically subsequent check-ins
509 */
@@ -512,11 +516,15 @@
516 );
517 nPrev = 0;
518 while( db_step(&q)==SQLITE_ROW ){
519 rid = db_column_int(&q, 0);
520 if( nPrev>0 ){
521 nSaved = content_deltify(rid, aPrev, nPrev, 0);
522 if( nSaved>0 ){
523 nDelta++;
524 nByte += nSaved;
525 }
526 }
527 if( nPrev<N_NEIGHBOR ){
528 aPrev[nPrev++] = rid;
529 }else{
530 int i;
@@ -543,11 +551,15 @@
551 rid = db_column_int(&q, 0);
552 fnid = db_column_int(&q, 1);
553 if( fnid!=prevfnid ) nPrev = 0;
554 prevfnid = fnid;
555 if( nPrev>0 ){
556 nSaved = content_deltify(rid, aPrev, nPrev, 0);
557 if( nSaved>0 ){
558 nDelta++;
559 nByte += nSaved;
560 }
561 }
562 if( nPrev<N_NEIGHBOR ){
563 aPrev[nPrev++] = rid;
564 }else{
565 int i;
@@ -556,10 +568,12 @@
568 }
569 }
570 db_finalize(&q);
571
572 db_end_transaction(0);
573 if( pnDelta!=0 ) *pnDelta = nDelta;
574 return nByte;
575 }
576
577
578 /* Reconstruct the private table. The private table contains the rid
579 ** of every manifest that is tagged with "private" and every file that
@@ -636,11 +650,11 @@
650 showStats = find_option("stats",0,0)!=0;
651 optIndex = find_option("index",0,0)!=0;
652 optNoIndex = find_option("noindex",0,0)!=0;
653 optIfNeeded = find_option("ifneeded",0,0)!=0;
654 compressOnlyFlag = find_option("compress-only",0,0)!=0;
655 if( compressOnlyFlag ) runCompress = 1;
656 if( zPagesize ){
657 newPagesize = atoi(zPagesize);
658 if( newPagesize<512 || newPagesize>65536
659 || (newPagesize&(newPagesize-1))!=0
660 ){
@@ -688,17 +702,25 @@
702 errCnt
703 );
704 db_end_transaction(1);
705 }else{
706 if( runCompress ){
707 i64 nByte = 0;
708 int nDelta = 0;
709 fossil_print("Extra delta compression... "); fflush(stdout);
710 nByte = extra_deltification(&nDelta);
711 if( nDelta>0 ){
712 fossil_print("%d new deltas save %,lld bytes", nDelta, nByte);
713 runVacuum = 1;
714 }else{
715 fossil_print("no additional compression found");
716 }
717 fflush(stdout);
718 }
719 if( omitVerify ) verify_cancel();
720 db_end_transaction(0);
721 if( runCompress ) fossil_print("\n");
722 db_close(0);
723 db_open_repository(g.zRepositoryName);
724 if( newPagesize ){
725 db_multi_exec("PRAGMA page_size=%d", newPagesize);
726 runVacuum = 1;
727

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button