Fossil SCM

Implement the ability to generate delta manifests on import.

isaac.jurado 2013-07-07 19:23 git-better-import
Commit 2844434ef548a1374a3904f953ac4ad3ea5163b9
Parent 995ed75f9aca1a6…
1 file changed +137 -33
~ src/import.c
M src/import.c
+137 -33
--- src/import.c
+++ src/import.c
@@ -30,10 +30,11 @@
3030
 char *zName; /* Name of a file */
3131
 char *zUuid; /* UUID of the file */
3232
 char *zPrior; /* Prior name if the name was changed */
3333
 char isExe; /* True if executable */
3434
 char isLink; /* True if symlink */
35
+ char hasChanged; /* True when different than baseline */
3536
 };
3637
 #endif
3738
3839
3940
 /*
@@ -40,10 +41,11 @@
4041
 ** State information about an on-going fast-import parse.
4142
 */
4243
 static struct {
4344
 void (*xFinish)(void); /* Function to finish a prior record */
4445
 int nData; /* Bytes of data */
46
+ char *zBaseline; /* Baseline manifest. The B card. */
4547
 char *zTag; /* Name of a tag */
4648
 char *zBranch; /* Name of a branch for a commit */
4749
 char *zPrevBranch; /* The branch of the previous check-in */
4850
 char *aData; /* Data content */
4951
 char *zMark; /* The current mark */
@@ -56,14 +58,17 @@
5658
 int nMerge; /* Number of merge values */
5759
 int nMergeAlloc; /* Number of slots in azMerge[] */
5860
 char **azMerge; /* Merge values */
5961
 int nFile; /* Number of aFile values */
6062
 int nFileAlloc; /* Number of slots in aFile[] */
63
+ int nFileEffective; /* Number of aFile items with zUuid != 0 */
64
+ int nChanged; /* Number of aFile that differ from baseline */
6165
 ImportFile *aFile; /* Information about files in a commit */
6266
 int fromLoaded; /* True zFrom content loaded into aFile[] */
6367
 int hasLinks; /* True if git repository contains symlinks */
6468
 int tagCommit; /* True if the commit adds a tag */
69
+ int tryDelta; /* Attempt to generate delta manifests */
6570
 } gg;
6671
6772
 /*
6873
 ** Duplicate a string.
6974
 */
@@ -89,10 +94,11 @@
8994
 ** retained unless the freeAll flag is set.
9095
 */
9196
 static void import_reset(int freeAll){
9297
 int i;
9398
 gg.xFinish = 0;
99
+ fossil_free(gg.zBaseline); gg.zBaseline = 0;
94100
 fossil_free(gg.zTag); gg.zTag = 0;
95101
 fossil_free(gg.zBranch); gg.zBranch = 0;
96102
 fossil_free(gg.aData); gg.aData = 0;
97103
 fossil_free(gg.zMark); gg.zMark = 0;
98104
 fossil_free(gg.zDate); gg.zDate = 0;
@@ -109,10 +115,12 @@
109115
 fossil_free(gg.aFile[i].zUuid);
110116
 fossil_free(gg.aFile[i].zPrior);
111117
 gg.aFile[i].zPrior = 0;
112118
 }
113119
 memset(gg.aFile, 0, gg.nFile*sizeof(gg.aFile[0]));
120
+ gg.nChanged = 0;
121
+ gg.nFileEffective = 0;
114122
 gg.nFile = 0;
115123
 if( freeAll ){
116124
 fossil_free(gg.zPrevBranch);
117125
 fossil_free(gg.zPrevCheckin);
118126
 fossil_free(gg.azMerge);
@@ -228,35 +236,48 @@
228236
 ** Use data accumulated in gg from a "commit" record to add a new
229237
 ** manifest artifact to the BLOB table.
230238
 */
231239
 static void finish_commit(void){
232240
 int i;
241
+ int delta;
233242
 char *zFromBranch;
234243
 char *aTCard[4]; /* Array of T cards for manifest */
235244
 int nTCard = 0; /* Entries used in aTCard[] */
236245
 Blob record, cksum;
237246
238247
 import_prior_files();
239248
 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
249
+ /*
250
+ ** This is the same mechanism used by the commit command to decide whether to
251
+ ** generate a delta manifest or not. It is evaluated and saved for later
252
+ ** uses.
253
+ */
254
+ delta = gg.tryDelta && (gg.zBaseline!=0 || gg.zFrom!=0) &&
255
+ (gg.nChanged*gg.nChanged)<(gg.nFileEffective*3-9);
240256
 blob_zero(&record);
257
+ if( delta ){
258
+ blob_appendf(&record, "B %s\n", gg.zBaseline!=0 ? gg.zBaseline : gg.zFrom);
259
+ }
241260
 blob_appendf(&record, "C %F\n", gg.zComment);
242261
 blob_appendf(&record, "D %s\n", gg.zDate);
243262
 for(i=0; i<gg.nFile; i++){
244
- const char *zUuid = gg.aFile[i].zUuid;
245
- if( zUuid==0 ) continue;
246
- blob_appendf(&record, "F %F %s", gg.aFile[i].zName, zUuid);
247
- if( gg.aFile[i].isExe ){
248
- blob_append(&record, " x", 2);
249
- }else if( gg.aFile[i].isLink ){
250
- blob_append(&record, " l", 2);
251
- gg.hasLinks = 1;
252
- }
253
- if( gg.aFile[i].zPrior ){
254
- if( !gg.aFile[i].isExe && !gg.aFile[i].isLink ){
255
- blob_append(&record, " w", 2);
256
- }
257
- blob_appendf(&record, " %F", gg.aFile[i].zPrior);
263
+ const struct ImportFile *pFile = &gg.aFile[i];
264
+ if( (!delta && pFile->zUuid==0) || (delta && !pFile->hasChanged) ) continue;
265
+ blob_appendf(&record, "F %F", pFile->zName);
266
+ if( pFile->zUuid!=0 ) {
267
+ blob_appendf(&record, " %s", pFile->zUuid);
268
+ if( pFile->isExe ){
269
+ blob_append(&record, " x", 2);
270
+ }else if( pFile->isLink ){
271
+ blob_append(&record, " l", 2);
272
+ }
273
+ if( pFile->zPrior!=0 ){
274
+ if( !pFile->isExe && !pFile->isLink ){
275
+ blob_append(&record, " w", 2);
276
+ }
277
+ blob_appendf(&record, " %F", pFile->zPrior);
278
+ }
258279
 }
259280
 blob_append(&record, "\n", 1);
260281
 }
261282
 if( gg.zFrom ){
262283
 blob_appendf(&record, "P %s", gg.zFrom);
@@ -427,16 +448,55 @@
427448
 rid = fast_uuid_to_rid(gg.zFrom);
428449
 if( rid==0 ) return;
429450
 p = manifest_get(rid, CFTYPE_MANIFEST);
430451
 if( p==0 ) return;
431452
 manifest_file_rewind(p);
432
- while( (pOld = manifest_file_next(p, 0))!=0 ){
433
- pNew = import_add_file();
434
- pNew->zName = fossil_strdup(pOld->zName);
435
- pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
436
- pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
437
- pNew->zUuid = fossil_strdup(pOld->zUuid);
453
+ if( gg.tryDelta && p->pBaseline ){
454
+ /*
455
+ ** The manifest_file_next() iterator skips deletion "F" cards in delta
456
+ ** manifests. But, in order to build more delta manifests, this information
457
+ ** is necessary because it propagates. Therefore, in this case, the
458
+ ** manifest has to be traversed "manually".
459
+ */
460
+ Manifest *pB = p->pBaseline;
461
+ gg.zBaseline = fossil_strdup(p->zBaseline);
462
+ while( p->iFile<p->nFile || pB->iFile<pB->nFile ){
463
+ pNew = import_add_file();
464
+ if( p->iFile>=p->nFile ){
465
+ /* No more "F" cards in delta manifest, finish the baseline */
466
+ pOld = &pB->aFile[pB->iFile++];
467
+ }else if( pB->iFile>=pB->nFile ){
468
+ /* No more "F" cards in baseline, finish the delta manifest */
469
+ pOld = &p->aFile[p->iFile++];
470
+ pNew->hasChanged = 1;
471
+ }else{
472
+ int cmp = fossil_strcmp(pB->aFile[pB->iFile].zName,
473
+ p->aFile[p->iFile].zName);
474
+ if( cmp < 0 ){
475
+ pOld = &pB->aFile[pB->iFile++];
476
+ }else if( cmp >= 0 ){
477
+ pOld = &p->aFile[p->iFile++];
478
+ pNew->hasChanged = 1;
479
+ if( cmp==0 ) pB->iFile++;
480
+ }
481
+ }
482
+ pNew->zName = fossil_strdup(pOld->zName);
483
+ pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
484
+ pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
485
+ pNew->zUuid = fossil_strdup(pOld->zUuid);
486
+ gg.nChanged += pNew->hasChanged;
487
+ if( pNew->zUuid!=0 ) gg.nFileEffective++;
488
+ }
489
+ }else{
490
+ while( (pOld = manifest_file_next(p, 0))!=0 ){
491
+ pNew = import_add_file();
492
+ pNew->zName = fossil_strdup(pOld->zName);
493
+ pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
494
+ pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
495
+ pNew->zUuid = fossil_strdup(pOld->zUuid);
496
+ if( pNew->zUuid!=0 ) gg.nFileEffective++;
497
+ }
438498
 }
439499
 manifest_destroy(p);
440500
 }
441501
442502
 /*
@@ -480,11 +540,11 @@
480540
 ** Read the git-fast-import format from pIn and insert the corresponding
481541
 ** content into the database.
482542
 */
483543
 static void git_fast_import(FILE *pIn){
484544
 ImportFile *pFile, *pNew;
485
- int i, mx;
545
+ int i;
486546
 char *z;
487547
 char *zUuid;
488548
 char *zName;
489549
 char *zPerm;
490550
 char *zFrom;
@@ -623,50 +683,63 @@
623683
 i = 0;
624684
 pFile = import_find_file(zName, &i, gg.nFile);
625685
 if( pFile==0 ){
626686
 pFile = import_add_file();
627687
 pFile->zName = fossil_strdup(zName);
688
+ gg.nFileEffective++;
628689
 }
629690
 pFile->isExe = (fossil_strcmp(zPerm, "100755")==0);
630691
 pFile->isLink = (fossil_strcmp(zPerm, "120000")==0);
631692
 fossil_free(pFile->zUuid);
632693
 pFile->zUuid = resolve_committish(zUuid);
694
+ /*
695
+ ** This trick avoids counting multiple changes on the same filename as
696
+ ** changes to multiple filenames. When an entry in gg.aFile is already
697
+ ** different from its baseline, it is not counted again.
698
+ */
699
+ gg.nChanged += 1 - pFile->hasChanged;
700
+ pFile->hasChanged = 1;
633701
 }else
634702
 if( memcmp(zLine, "D ", 2)==0 ){
635703
 import_prior_files();
636704
 z = &zLine[2];
637705
 zName = rest_of_line(&z);
638706
 dequote_git_filename(zName);
639707
 i = 0;
640708
 pFile = import_find_file(zName, &i, gg.nFile);
641709
 if( pFile!=0 ){
642
- fossil_free(pFile->zName);
643
- fossil_free(pFile->zPrior);
644
- pFile->zPrior = 0;
710
+ /* Do not remove the item from gg.aFile, just mark as deleted */
645711
 fossil_free(pFile->zUuid);
646
- *pFile = gg.aFile[--gg.nFile];
712
+ pFile->zUuid = 0;
713
+ gg.nChanged += 1 - pFile->hasChanged;
714
+ pFile->hasChanged = 1;
715
+ gg.nFileEffective--;
647716
 }
648717
 }else
649718
 if( memcmp(zLine, "C ", 2)==0 ){
650
- int nFrom;
651719
 import_prior_files();
652720
 z = &zLine[2];
653721
 zFrom = next_token(&z);
654722
 zTo = rest_of_line(&z);
655723
 i = 0;
656
- nFrom = strlen(zFrom);
657724
 pFile = import_find_file(zFrom, &i, gg.nFile);
658725
 if( pFile!=0 ){
659
- pNew = import_add_file();
660
- if( strlen(pFile->zName)>nFrom ){
661
- pNew->zName = mprintf("%s%s", zTo, pFile->zName[nFrom]);
662
- }else{
726
+ int j = 0;
727
+ pNew = import_find_file(zTo, &j, gg.nFile);
728
+ if( pNew==0 ){
729
+ pNew = import_add_file();
730
+ pFile = &gg.aFile[i-1]; /* gg.aFile may have been realloc()-ed */
663731
 pNew->zName = fossil_strdup(zTo);
732
+ gg.nFileEffective++;
733
+ }else{
734
+ fossil_free(pNew->zUuid);
664735
 }
665736
 pNew->isExe = pFile->isExe;
666737
 pNew->isLink = pFile->isLink;
667738
 pNew->zUuid = fossil_strdup(pFile->zUuid);
739
+ gg.nChanged += 1 - pNew->hasChanged;
740
+ pNew->hasChanged = 1;
668741
 }
669742
 }else
670743
 if( memcmp(zLine, "R ", 2)==0 ){
671744
 import_prior_files();
672745
 z = &zLine[2];
@@ -673,12 +746,35 @@
673746
 zFrom = next_token(&z);
674747
 zTo = rest_of_line(&z);
675748
 i = 0;
676749
 pFile = import_find_file(zFrom, &i, gg.nFile);
677750
 if( pFile!=0 ){
678
- pFile->zPrior = pFile->zName;
679
- pFile->zName = fossil_strdup(zTo);
751
+ /*
752
+ ** File renames in delta manifests require two "F" cards: one to
753
+ ** delete the old file (without UUID) and another with the rename
754
+ ** (with prior name equals to the name in the other card).
755
+ **
756
+ ** This forces us to also lookup by the destination name, as it may
757
+ ** already exist in the form of a delta manifest deletion "F" card.
758
+ */
759
+ int j = 0;
760
+ pNew = import_find_file(zTo, &j, gg.nFile);
761
+ if( pNew==0 ){
762
+ pNew = import_add_file();
763
+ pFile = &gg.aFile[i-1];
764
+ pNew->zName = fossil_strdup(zTo);
765
+ }else{
766
+ fossil_free(pNew->zUuid); /* Just in case */
767
+ }
768
+ pNew->isExe = pFile->isExe;
769
+ pNew->isLink = pFile->isLink;
770
+ pNew->zPrior = fossil_strdup(zFrom);
771
+ pNew->zUuid = pFile->zUuid;
772
+ pFile->zUuid = 0;
773
+ gg.nChanged += 2 - (pNew->hasChanged + pFile->hasChanged);
774
+ pNew->hasChanged = 1;
775
+ pFile->hasChanged = 1;
680776
 }
681777
 }else
682778
 if( memcmp(zLine, "deleteall", 9)==0 ){
683779
 gg.fromLoaded = 1;
684780
 }else
@@ -719,11 +815,18 @@
719815
 **
720816
 ** The --incremental option allows an existing repository to be extended
721817
 ** with new content. Otherwise, if a file with the same name as NEW-REPOSITORY
722818
 ** is found, the command fails unless the --force option is used.
723819
 **
820
+** When the --delta option is used, delta manifests will be generated when they
821
+** are smaller than the equivalent baseline manifest. Please beware that delta
822
+** manifests are not understood by older versions of Fossil. Therefore, only
823
+** use this option when it can be assured that only newer clients will pull or
824
+** read from it.
825
+**
724826
 ** Options:
827
+** -d|--delta enable delta manifest generation
725828
 ** -f|--force remove existing file
726829
 ** -i|--incremental allow importing into an existing repository
727830
 **
728831
 ** See also: export
729832
 */
@@ -733,10 +836,11 @@
733836
 Stmt q;
734837
 int forceFlag = find_option("force", "f", 0)!=0;
735838
 int incrFlag = find_option("incremental", "i", 0)!=0;
736839
737840
 find_option("git",0,0); /* Skip the --git option for now */
841
+ gg.tryDelta = find_option("delta", "d", 0)!=0;
738842
 verify_all_options();
739843
 if( g.argc!=3 && g.argc!=4 ){
740844
 usage("REPOSITORY-NAME");
741845
 }
742846
 if( g.argc==4 ){
743847
--- src/import.c
+++ src/import.c
@@ -30,10 +30,11 @@
30 char *zName; /* Name of a file */
31 char *zUuid; /* UUID of the file */
32 char *zPrior; /* Prior name if the name was changed */
33 char isExe; /* True if executable */
34 char isLink; /* True if symlink */
 
35 };
36 #endif
37
38
39 /*
@@ -40,10 +41,11 @@
40 ** State information about an on-going fast-import parse.
41 */
42 static struct {
43 void (*xFinish)(void); /* Function to finish a prior record */
44 int nData; /* Bytes of data */
 
45 char *zTag; /* Name of a tag */
46 char *zBranch; /* Name of a branch for a commit */
47 char *zPrevBranch; /* The branch of the previous check-in */
48 char *aData; /* Data content */
49 char *zMark; /* The current mark */
@@ -56,14 +58,17 @@
56 int nMerge; /* Number of merge values */
57 int nMergeAlloc; /* Number of slots in azMerge[] */
58 char **azMerge; /* Merge values */
59 int nFile; /* Number of aFile values */
60 int nFileAlloc; /* Number of slots in aFile[] */
 
 
61 ImportFile *aFile; /* Information about files in a commit */
62 int fromLoaded; /* True zFrom content loaded into aFile[] */
63 int hasLinks; /* True if git repository contains symlinks */
64 int tagCommit; /* True if the commit adds a tag */
 
65 } gg;
66
67 /*
68 ** Duplicate a string.
69 */
@@ -89,10 +94,11 @@
89 ** retained unless the freeAll flag is set.
90 */
91 static void import_reset(int freeAll){
92 int i;
93 gg.xFinish = 0;
 
94 fossil_free(gg.zTag); gg.zTag = 0;
95 fossil_free(gg.zBranch); gg.zBranch = 0;
96 fossil_free(gg.aData); gg.aData = 0;
97 fossil_free(gg.zMark); gg.zMark = 0;
98 fossil_free(gg.zDate); gg.zDate = 0;
@@ -109,10 +115,12 @@
109 fossil_free(gg.aFile[i].zUuid);
110 fossil_free(gg.aFile[i].zPrior);
111 gg.aFile[i].zPrior = 0;
112 }
113 memset(gg.aFile, 0, gg.nFile*sizeof(gg.aFile[0]));
 
 
114 gg.nFile = 0;
115 if( freeAll ){
116 fossil_free(gg.zPrevBranch);
117 fossil_free(gg.zPrevCheckin);
118 fossil_free(gg.azMerge);
@@ -228,35 +236,48 @@
228 ** Use data accumulated in gg from a "commit" record to add a new
229 ** manifest artifact to the BLOB table.
230 */
231 static void finish_commit(void){
232 int i;
 
233 char *zFromBranch;
234 char *aTCard[4]; /* Array of T cards for manifest */
235 int nTCard = 0; /* Entries used in aTCard[] */
236 Blob record, cksum;
237
238 import_prior_files();
239 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
 
 
 
 
 
 
 
240 blob_zero(&record);
 
 
 
241 blob_appendf(&record, "C %F\n", gg.zComment);
242 blob_appendf(&record, "D %s\n", gg.zDate);
243 for(i=0; i<gg.nFile; i++){
244 const char *zUuid = gg.aFile[i].zUuid;
245 if( zUuid==0 ) continue;
246 blob_appendf(&record, "F %F %s", gg.aFile[i].zName, zUuid);
247 if( gg.aFile[i].isExe ){
248 blob_append(&record, " x", 2);
249 }else if( gg.aFile[i].isLink ){
250 blob_append(&record, " l", 2);
251 gg.hasLinks = 1;
252 }
253 if( gg.aFile[i].zPrior ){
254 if( !gg.aFile[i].isExe && !gg.aFile[i].isLink ){
255 blob_append(&record, " w", 2);
256 }
257 blob_appendf(&record, " %F", gg.aFile[i].zPrior);
 
 
258 }
259 blob_append(&record, "\n", 1);
260 }
261 if( gg.zFrom ){
262 blob_appendf(&record, "P %s", gg.zFrom);
@@ -427,16 +448,55 @@
427 rid = fast_uuid_to_rid(gg.zFrom);
428 if( rid==0 ) return;
429 p = manifest_get(rid, CFTYPE_MANIFEST);
430 if( p==0 ) return;
431 manifest_file_rewind(p);
432 while( (pOld = manifest_file_next(p, 0))!=0 ){
433 pNew = import_add_file();
434 pNew->zName = fossil_strdup(pOld->zName);
435 pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
436 pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
437 pNew->zUuid = fossil_strdup(pOld->zUuid);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438 }
439 manifest_destroy(p);
440 }
441
442 /*
@@ -480,11 +540,11 @@
480 ** Read the git-fast-import format from pIn and insert the corresponding
481 ** content into the database.
482 */
483 static void git_fast_import(FILE *pIn){
484 ImportFile *pFile, *pNew;
485 int i, mx;
486 char *z;
487 char *zUuid;
488 char *zName;
489 char *zPerm;
490 char *zFrom;
@@ -623,50 +683,63 @@
623 i = 0;
624 pFile = import_find_file(zName, &i, gg.nFile);
625 if( pFile==0 ){
626 pFile = import_add_file();
627 pFile->zName = fossil_strdup(zName);
 
628 }
629 pFile->isExe = (fossil_strcmp(zPerm, "100755")==0);
630 pFile->isLink = (fossil_strcmp(zPerm, "120000")==0);
631 fossil_free(pFile->zUuid);
632 pFile->zUuid = resolve_committish(zUuid);
 
 
 
 
 
 
 
633 }else
634 if( memcmp(zLine, "D ", 2)==0 ){
635 import_prior_files();
636 z = &zLine[2];
637 zName = rest_of_line(&z);
638 dequote_git_filename(zName);
639 i = 0;
640 pFile = import_find_file(zName, &i, gg.nFile);
641 if( pFile!=0 ){
642 fossil_free(pFile->zName);
643 fossil_free(pFile->zPrior);
644 pFile->zPrior = 0;
645 fossil_free(pFile->zUuid);
646 *pFile = gg.aFile[--gg.nFile];
 
 
 
647 }
648 }else
649 if( memcmp(zLine, "C ", 2)==0 ){
650 int nFrom;
651 import_prior_files();
652 z = &zLine[2];
653 zFrom = next_token(&z);
654 zTo = rest_of_line(&z);
655 i = 0;
656 nFrom = strlen(zFrom);
657 pFile = import_find_file(zFrom, &i, gg.nFile);
658 if( pFile!=0 ){
659 pNew = import_add_file();
660 if( strlen(pFile->zName)>nFrom ){
661 pNew->zName = mprintf("%s%s", zTo, pFile->zName[nFrom]);
662 }else{
 
663 pNew->zName = fossil_strdup(zTo);
 
 
 
664 }
665 pNew->isExe = pFile->isExe;
666 pNew->isLink = pFile->isLink;
667 pNew->zUuid = fossil_strdup(pFile->zUuid);
 
 
668 }
669 }else
670 if( memcmp(zLine, "R ", 2)==0 ){
671 import_prior_files();
672 z = &zLine[2];
@@ -673,12 +746,35 @@
673 zFrom = next_token(&z);
674 zTo = rest_of_line(&z);
675 i = 0;
676 pFile = import_find_file(zFrom, &i, gg.nFile);
677 if( pFile!=0 ){
678 pFile->zPrior = pFile->zName;
679 pFile->zName = fossil_strdup(zTo);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
680 }
681 }else
682 if( memcmp(zLine, "deleteall", 9)==0 ){
683 gg.fromLoaded = 1;
684 }else
@@ -719,11 +815,18 @@
719 **
720 ** The --incremental option allows an existing repository to be extended
721 ** with new content. Otherwise, if a file with the same name as NEW-REPOSITORY
722 ** is found, the command fails unless the --force option is used.
723 **
 
 
 
 
 
 
724 ** Options:
 
725 ** -f|--force remove existing file
726 ** -i|--incremental allow importing into an existing repository
727 **
728 ** See also: export
729 */
@@ -733,10 +836,11 @@
733 Stmt q;
734 int forceFlag = find_option("force", "f", 0)!=0;
735 int incrFlag = find_option("incremental", "i", 0)!=0;
736
737 find_option("git",0,0); /* Skip the --git option for now */
 
738 verify_all_options();
739 if( g.argc!=3 && g.argc!=4 ){
740 usage("REPOSITORY-NAME");
741 }
742 if( g.argc==4 ){
743
--- src/import.c
+++ src/import.c
@@ -30,10 +30,11 @@
30 char *zName; /* Name of a file */
31 char *zUuid; /* UUID of the file */
32 char *zPrior; /* Prior name if the name was changed */
33 char isExe; /* True if executable */
34 char isLink; /* True if symlink */
35 char hasChanged; /* True when different than baseline */
36 };
37 #endif
38
39
40 /*
@@ -40,10 +41,11 @@
41 ** State information about an on-going fast-import parse.
42 */
43 static struct {
44 void (*xFinish)(void); /* Function to finish a prior record */
45 int nData; /* Bytes of data */
46 char *zBaseline; /* Baseline manifest. The B card. */
47 char *zTag; /* Name of a tag */
48 char *zBranch; /* Name of a branch for a commit */
49 char *zPrevBranch; /* The branch of the previous check-in */
50 char *aData; /* Data content */
51 char *zMark; /* The current mark */
@@ -56,14 +58,17 @@
58 int nMerge; /* Number of merge values */
59 int nMergeAlloc; /* Number of slots in azMerge[] */
60 char **azMerge; /* Merge values */
61 int nFile; /* Number of aFile values */
62 int nFileAlloc; /* Number of slots in aFile[] */
63 int nFileEffective; /* Number of aFile items with zUuid != 0 */
64 int nChanged; /* Number of aFile that differ from baseline */
65 ImportFile *aFile; /* Information about files in a commit */
66 int fromLoaded; /* True zFrom content loaded into aFile[] */
67 int hasLinks; /* True if git repository contains symlinks */
68 int tagCommit; /* True if the commit adds a tag */
69 int tryDelta; /* Attempt to generate delta manifests */
70 } gg;
71
72 /*
73 ** Duplicate a string.
74 */
@@ -89,10 +94,11 @@
94 ** retained unless the freeAll flag is set.
95 */
96 static void import_reset(int freeAll){
97 int i;
98 gg.xFinish = 0;
99 fossil_free(gg.zBaseline); gg.zBaseline = 0;
100 fossil_free(gg.zTag); gg.zTag = 0;
101 fossil_free(gg.zBranch); gg.zBranch = 0;
102 fossil_free(gg.aData); gg.aData = 0;
103 fossil_free(gg.zMark); gg.zMark = 0;
104 fossil_free(gg.zDate); gg.zDate = 0;
@@ -109,10 +115,12 @@
115 fossil_free(gg.aFile[i].zUuid);
116 fossil_free(gg.aFile[i].zPrior);
117 gg.aFile[i].zPrior = 0;
118 }
119 memset(gg.aFile, 0, gg.nFile*sizeof(gg.aFile[0]));
120 gg.nChanged = 0;
121 gg.nFileEffective = 0;
122 gg.nFile = 0;
123 if( freeAll ){
124 fossil_free(gg.zPrevBranch);
125 fossil_free(gg.zPrevCheckin);
126 fossil_free(gg.azMerge);
@@ -228,35 +236,48 @@
236 ** Use data accumulated in gg from a "commit" record to add a new
237 ** manifest artifact to the BLOB table.
238 */
239 static void finish_commit(void){
240 int i;
241 int delta;
242 char *zFromBranch;
243 char *aTCard[4]; /* Array of T cards for manifest */
244 int nTCard = 0; /* Entries used in aTCard[] */
245 Blob record, cksum;
246
247 import_prior_files();
248 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
249 /*
250 ** This is the same mechanism used by the commit command to decide whether to
251 ** generate a delta manifest or not. It is evaluated and saved for later
252 ** uses.
253 */
254 delta = gg.tryDelta && (gg.zBaseline!=0 || gg.zFrom!=0) &&
255 (gg.nChanged*gg.nChanged)<(gg.nFileEffective*3-9);
256 blob_zero(&record);
257 if( delta ){
258 blob_appendf(&record, "B %s\n", gg.zBaseline!=0 ? gg.zBaseline : gg.zFrom);
259 }
260 blob_appendf(&record, "C %F\n", gg.zComment);
261 blob_appendf(&record, "D %s\n", gg.zDate);
262 for(i=0; i<gg.nFile; i++){
263 const struct ImportFile *pFile = &gg.aFile[i];
264 if( (!delta && pFile->zUuid==0) || (delta && !pFile->hasChanged) ) continue;
265 blob_appendf(&record, "F %F", pFile->zName);
266 if( pFile->zUuid!=0 ) {
267 blob_appendf(&record, " %s", pFile->zUuid);
268 if( pFile->isExe ){
269 blob_append(&record, " x", 2);
270 }else if( pFile->isLink ){
271 blob_append(&record, " l", 2);
272 }
273 if( pFile->zPrior!=0 ){
274 if( !pFile->isExe && !pFile->isLink ){
275 blob_append(&record, " w", 2);
276 }
277 blob_appendf(&record, " %F", pFile->zPrior);
278 }
279 }
280 blob_append(&record, "\n", 1);
281 }
282 if( gg.zFrom ){
283 blob_appendf(&record, "P %s", gg.zFrom);
@@ -427,16 +448,55 @@
448 rid = fast_uuid_to_rid(gg.zFrom);
449 if( rid==0 ) return;
450 p = manifest_get(rid, CFTYPE_MANIFEST);
451 if( p==0 ) return;
452 manifest_file_rewind(p);
453 if( gg.tryDelta && p->pBaseline ){
454 /*
455 ** The manifest_file_next() iterator skips deletion "F" cards in delta
456 ** manifests. But, in order to build more delta manifests, this information
457 ** is necessary because it propagates. Therefore, in this case, the
458 ** manifest has to be traversed "manually".
459 */
460 Manifest *pB = p->pBaseline;
461 gg.zBaseline = fossil_strdup(p->zBaseline);
462 while( p->iFile<p->nFile || pB->iFile<pB->nFile ){
463 pNew = import_add_file();
464 if( p->iFile>=p->nFile ){
465 /* No more "F" cards in delta manifest, finish the baseline */
466 pOld = &pB->aFile[pB->iFile++];
467 }else if( pB->iFile>=pB->nFile ){
468 /* No more "F" cards in baseline, finish the delta manifest */
469 pOld = &p->aFile[p->iFile++];
470 pNew->hasChanged = 1;
471 }else{
472 int cmp = fossil_strcmp(pB->aFile[pB->iFile].zName,
473 p->aFile[p->iFile].zName);
474 if( cmp < 0 ){
475 pOld = &pB->aFile[pB->iFile++];
476 }else if( cmp >= 0 ){
477 pOld = &p->aFile[p->iFile++];
478 pNew->hasChanged = 1;
479 if( cmp==0 ) pB->iFile++;
480 }
481 }
482 pNew->zName = fossil_strdup(pOld->zName);
483 pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
484 pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
485 pNew->zUuid = fossil_strdup(pOld->zUuid);
486 gg.nChanged += pNew->hasChanged;
487 if( pNew->zUuid!=0 ) gg.nFileEffective++;
488 }
489 }else{
490 while( (pOld = manifest_file_next(p, 0))!=0 ){
491 pNew = import_add_file();
492 pNew->zName = fossil_strdup(pOld->zName);
493 pNew->isExe = pOld->zPerm && strstr(pOld->zPerm, "x")!=0;
494 pNew->isLink = pOld->zPerm && strstr(pOld->zPerm, "l")!=0;
495 pNew->zUuid = fossil_strdup(pOld->zUuid);
496 if( pNew->zUuid!=0 ) gg.nFileEffective++;
497 }
498 }
499 manifest_destroy(p);
500 }
501
502 /*
@@ -480,11 +540,11 @@
540 ** Read the git-fast-import format from pIn and insert the corresponding
541 ** content into the database.
542 */
543 static void git_fast_import(FILE *pIn){
544 ImportFile *pFile, *pNew;
545 int i;
546 char *z;
547 char *zUuid;
548 char *zName;
549 char *zPerm;
550 char *zFrom;
@@ -623,50 +683,63 @@
683 i = 0;
684 pFile = import_find_file(zName, &i, gg.nFile);
685 if( pFile==0 ){
686 pFile = import_add_file();
687 pFile->zName = fossil_strdup(zName);
688 gg.nFileEffective++;
689 }
690 pFile->isExe = (fossil_strcmp(zPerm, "100755")==0);
691 pFile->isLink = (fossil_strcmp(zPerm, "120000")==0);
692 fossil_free(pFile->zUuid);
693 pFile->zUuid = resolve_committish(zUuid);
694 /*
695 ** This trick avoids counting multiple changes on the same filename as
696 ** changes to multiple filenames. When an entry in gg.aFile is already
697 ** different from its baseline, it is not counted again.
698 */
699 gg.nChanged += 1 - pFile->hasChanged;
700 pFile->hasChanged = 1;
701 }else
702 if( memcmp(zLine, "D ", 2)==0 ){
703 import_prior_files();
704 z = &zLine[2];
705 zName = rest_of_line(&z);
706 dequote_git_filename(zName);
707 i = 0;
708 pFile = import_find_file(zName, &i, gg.nFile);
709 if( pFile!=0 ){
710 /* Do not remove the item from gg.aFile, just mark as deleted */
 
 
711 fossil_free(pFile->zUuid);
712 pFile->zUuid = 0;
713 gg.nChanged += 1 - pFile->hasChanged;
714 pFile->hasChanged = 1;
715 gg.nFileEffective--;
716 }
717 }else
718 if( memcmp(zLine, "C ", 2)==0 ){
 
719 import_prior_files();
720 z = &zLine[2];
721 zFrom = next_token(&z);
722 zTo = rest_of_line(&z);
723 i = 0;
 
724 pFile = import_find_file(zFrom, &i, gg.nFile);
725 if( pFile!=0 ){
726 int j = 0;
727 pNew = import_find_file(zTo, &j, gg.nFile);
728 if( pNew==0 ){
729 pNew = import_add_file();
730 pFile = &gg.aFile[i-1]; /* gg.aFile may have been realloc()-ed */
731 pNew->zName = fossil_strdup(zTo);
732 gg.nFileEffective++;
733 }else{
734 fossil_free(pNew->zUuid);
735 }
736 pNew->isExe = pFile->isExe;
737 pNew->isLink = pFile->isLink;
738 pNew->zUuid = fossil_strdup(pFile->zUuid);
739 gg.nChanged += 1 - pNew->hasChanged;
740 pNew->hasChanged = 1;
741 }
742 }else
743 if( memcmp(zLine, "R ", 2)==0 ){
744 import_prior_files();
745 z = &zLine[2];
@@ -673,12 +746,35 @@
746 zFrom = next_token(&z);
747 zTo = rest_of_line(&z);
748 i = 0;
749 pFile = import_find_file(zFrom, &i, gg.nFile);
750 if( pFile!=0 ){
751 /*
752 ** File renames in delta manifests require two "F" cards: one to
753 ** delete the old file (without UUID) and another with the rename
754 ** (with prior name equals to the name in the other card).
755 **
756 ** This forces us to also lookup by the destination name, as it may
757 ** already exist in the form of a delta manifest deletion "F" card.
758 */
759 int j = 0;
760 pNew = import_find_file(zTo, &j, gg.nFile);
761 if( pNew==0 ){
762 pNew = import_add_file();
763 pFile = &gg.aFile[i-1];
764 pNew->zName = fossil_strdup(zTo);
765 }else{
766 fossil_free(pNew->zUuid); /* Just in case */
767 }
768 pNew->isExe = pFile->isExe;
769 pNew->isLink = pFile->isLink;
770 pNew->zPrior = fossil_strdup(zFrom);
771 pNew->zUuid = pFile->zUuid;
772 pFile->zUuid = 0;
773 gg.nChanged += 2 - (pNew->hasChanged + pFile->hasChanged);
774 pNew->hasChanged = 1;
775 pFile->hasChanged = 1;
776 }
777 }else
778 if( memcmp(zLine, "deleteall", 9)==0 ){
779 gg.fromLoaded = 1;
780 }else
@@ -719,11 +815,18 @@
815 **
816 ** The --incremental option allows an existing repository to be extended
817 ** with new content. Otherwise, if a file with the same name as NEW-REPOSITORY
818 ** is found, the command fails unless the --force option is used.
819 **
820 ** When the --delta option is used, delta manifests will be generated when they
821 ** are smaller than the equivalent baseline manifest. Please beware that delta
822 ** manifests are not understood by older versions of Fossil. Therefore, only
823 ** use this option when it can be assured that only newer clients will pull or
824 ** read from it.
825 **
826 ** Options:
827 ** -d|--delta enable delta manifest generation
828 ** -f|--force remove existing file
829 ** -i|--incremental allow importing into an existing repository
830 **
831 ** See also: export
832 */
@@ -733,10 +836,11 @@
836 Stmt q;
837 int forceFlag = find_option("force", "f", 0)!=0;
838 int incrFlag = find_option("incremental", "i", 0)!=0;
839
840 find_option("git",0,0); /* Skip the --git option for now */
841 gg.tryDelta = find_option("delta", "d", 0)!=0;
842 verify_all_options();
843 if( g.argc!=3 && g.argc!=4 ){
844 usage("REPOSITORY-NAME");
845 }
846 if( g.argc==4 ){
847

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button