Fossil SCM

Fix how Fossil handles 'import --git' into a repository that already exists when neither the '--increment' nor '--force' options are passed, as reported by Warren in [forum post 29e358909c|forum:/forumpost/29e358909c]. The proposed fix required duplicating a file descriptor for stdin to accept keyboard input from the user and piped input from git fast-import, which necessitated machine-dependent ifdefs that have not yet been tested on Windows platforms. The fix revealed another bug that this commit also solves by avoiding duplicate artifacts being inserted into the blob table during fast_insert_content(), which can arise when importing into a repository that has used the recently implemented '--attribute' feature if manifests are generated from the same content but with different U cards. Due to a bug in Git's output, this required an additional temp table for tracking tag artifacts (see comments). Due to the complexity of this commit, and having not been tested yet on Windows platforms, further testing is needed.

jamsek 2020-11-28 13:36 trunk
Commit a69f71a2758ad66daf82d8bb71afa202bb3514e63ee2fccbb44def0012c6c143
Parent a7343c6a0cb227f…
1 file changed +139 -19
~ src/import.c
M src/import.c
+139 -19
--- src/import.c
+++ src/import.c
@@ -34,10 +34,22 @@
3434
 char isExe; /* True if executable */
3535
 char isLink; /* True if symlink */
3636
 };
3737
 #endif
3838
39
+/*
40
+ * Flags to indicate whether the import is to an existing or new repository;
41
+ * or using the --attribute option and in the process of importing a commit or
42
+ * tag (to determine the artifact type during fast_insert_content()).
43
+ */
44
+enum import_mode {
45
+ OLD_REPO,
46
+ NEW_REPO,
47
+ COMMIT_ATTR,
48
+ TAG_ATTR
49
+};
50
+
3951
 /*
4052
 ** State information common to all import types.
4153
 */
4254
 static struct {
4355
 const char *zTrunkName; /* Name of trunk branch */
@@ -58,10 +70,11 @@
5870
 char *zPrevBranch; /* The branch of the previous check-in */
5971
 char *aData; /* Data content */
6072
 char *zMark; /* The current mark */
6173
 char *zDate; /* Date/time stamp */
6274
 char *zUser; /* User name */
75
+ char *zEmail; /* Email from Git committer string */
6376
 char *zComment; /* Comment of a commit */
6477
 char *zFrom; /* from value as a hash */
6578
 char *zPrevCheckin; /* Name of the previous check-in */
6679
 char *zFromMark; /* The mark of the "from" field */
6780
 int nMerge; /* Number of merge values */
@@ -115,10 +128,11 @@
115128
 fossil_free(gg.zBranch); gg.zBranch = 0;
116129
 fossil_free(gg.aData); gg.aData = 0;
117130
 fossil_free(gg.zMark); gg.zMark = 0;
118131
 fossil_free(gg.zDate); gg.zDate = 0;
119132
 fossil_free(gg.zUser); gg.zUser = 0;
133
+ fossil_free(gg.zEmail); gg.zEmail = 0;
120134
 fossil_free(gg.zComment); gg.zComment = 0;
121135
 fossil_free(gg.zFrom); gg.zFrom = 0;
122136
 fossil_free(gg.zFromMark); gg.zFromMark = 0;
123137
 for(i=0; i<gg.nMerge; i++){
124138
 fossil_free(gg.azMerge[i]); gg.azMerge[i] = 0;
@@ -139,10 +153,23 @@
139153
 memset(&gg, 0, sizeof(gg));
140154
 }
141155
 gg.xFinish = finish_noop;
142156
 }
143157
158
+static struct{
159
+ const char *zMasterName; /* Name of master branch */
160
+ int authorFlag; /* Use author as checkin committer */
161
+ Blob altRecord; /* Record to cross-check --attribute'd commits */
162
+ uint8_t commitType:2; /* To indicate --attribute'd manifest or tag */
163
+ uint8_t importType:2; /* To indicate import into new or existing repo */
164
+ int nGitAttr; /* Number of Git --attribute entries */
165
+ struct { /* Git --attribute details */
166
+ char *zUser;
167
+ char *zEmail;
168
+ } *gitUserInfo;
169
+} ggit;
170
+
144171
 /*
145172
 ** Insert an artifact into the BLOB table if it isn't there already.
146173
 ** If zMark is not zero, create a cross-reference from that mark back
147174
 ** to the newly inserted artifact.
148175
 **
@@ -160,10 +187,35 @@
160187
 Blob cmpr;
161188
 int rid;
162189
163190
 hname_hash(pContent, 0, &hash);
164191
 rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
192
+ /*
193
+ * If this repo has had --attribute'd commits, we need to check the blob
194
+ * table for manifests with U cards constructed from both the username and
195
+ * email address to ensure no duplicate entries are attempted.
196
+ */
197
+ if (ggit.commitType == COMMIT_ATTR && rid == 0) {
198
+ blob_reset(&hash);
199
+ hname_hash(&ggit.altRecord, 0, &hash);
200
+ rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
201
+ }
202
+ /*
203
+ * Likewise, for --attribute'd tags pop the tag artifact with the alternate
204
+ * U card from the temp xtag2 table; both these tags and the above commit
205
+ * artifacts are the same as pContent albeit with the {user|email} U card.
206
+ */
207
+ if (ggit.commitType == TAG_ATTR && rid == 0) {
208
+ db_blob(&ggit.altRecord,
209
+ "SELECT tcontent FROM xtag2 ORDER BY ROWID ASC LIMIT 1");
210
+ db_multi_exec(
211
+ "DELETE FROM xtag2 WHERE tcontent=%Q", blob_str(&ggit.altRecord)
212
+ );
213
+ blob_reset(&hash);
214
+ hname_hash(&ggit.altRecord, 0, &hash);
215
+ rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
216
+ }
165217
 if( rid==0 ){
166218
 static Stmt ins;
167219
 assert( g.rcvid>0 );
168220
 db_static_prepare(&ins,
169221
 "INSERT INTO blob(uuid, size, rcvid, content)"
@@ -280,11 +332,11 @@
280332
 static void finish_commit(void){
281333
 int i;
282334
 char *zFromBranch;
283335
 char *aTCard[4]; /* Array of T cards for manifest */
284336
 int nTCard = 0; /* Entries used in aTCard[] */
285
- Blob record, cksum;
337
+ Blob record, cksum, altCksum;
286338
287339
 import_prior_files();
288340
 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
289341
 blob_zero(&record);
290342
 blob_appendf(&record, "C %F\n", gg.zComment);
@@ -342,10 +394,24 @@
342394
 for(i=0; i<nTCard; i++) free(aTCard[i]);
343395
344396
 free(zFromBranch);
345397
 db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)",
346398
 gg.zMark, gg.zBranch);
399
+ /*
400
+ * The fx_git table indicates this repo has --attribute'd commits; therefore,
401
+ * blob entries may exist with U cards generated from either a username or
402
+ * email address. Create both for cross-checking to avoid adding duplicates.
403
+ */
404
+ if (db_table_exists("repository", "fx_git") &&
405
+ db_text(0, "SELECT user FROM fx_git WHERE email=%Q", gg.zEmail)) {
406
+ blob_copy(&ggit.altRecord, &record);
407
+ blob_appendf(&ggit.altRecord, "U %F\n", gg.zEmail);
408
+ md5sum_blob(&ggit.altRecord, &altCksum);
409
+ blob_appendf(&ggit.altRecord, "Z %b\n", &altCksum);
410
+ blob_reset(&altCksum);
411
+ ggit.commitType = COMMIT_ATTR;
412
+ }
347413
 blob_appendf(&record, "U %F\n", gg.zUser);
348414
 md5sum_blob(&record, &cksum);
349415
 blob_appendf(&record, "Z %b\n", &cksum);
350416
 fast_insert_content(&record, gg.zMark, 0, 1, 1);
351417
 blob_reset(&cksum);
@@ -359,16 +425,41 @@
359425
 **
360426
 ** This behavior seems like a bug in git-fast-export, but it is easier
361427
 ** to work around the problem than to fix git-fast-export.
362428
 */
363429
 if( gg.tagCommit && gg.zDate && gg.zUser && gg.zFrom ){
430
+ Blob altTag;
364431
 record.nUsed = 0
365432
 /*in case fast_insert_comment() did not indirectly blob_reset() it */;
366433
 blob_appendf(&record, "D %s\n", gg.zDate);
367434
 blob_appendf(&record, "T +sym-%F%F%F %s\n", gimport.zBranchPre, gg.zBranch,
368435
 gimport.zBranchSuf, gg.zPrevCheckin);
369
- blob_appendf(&record, "U %F\n", gg.zUser);
436
+ /*
437
+ * If --attribute'd commits are present, we also need tag artifacts of the
438
+ * other potential U card value to be cross-checked against the blob table.
439
+ * Due to the abovementioned Git bug, store in a different table: xtag2.
440
+ */
441
+ if (ggit.commitType == COMMIT_ATTR) {
442
+ blob_copy(&altTag, &record);
443
+ blob_appendf(&altTag, "U %F\n", gg.zEmail);
444
+ md5sum_blob(&altTag, &altCksum);
445
+ blob_appendf(&altTag, "Z %b\n", &altCksum);
446
+ db_multi_exec(
447
+ "INSERT OR REPLACE INTO xtag2(tname, tcontent)"
448
+ " VALUES(%Q,%Q)", gg.zBranch, blob_str(&altTag)
449
+ );
450
+ blob_reset(&altCksum);
451
+ blob_reset(&altTag);
452
+ ggit.commitType = TAG_ATTR;
453
+ }
454
+ /*
455
+ * IF this is a brand new repo, ONLY one type of tag artifact can exist in
456
+ * the blob table, which are those generated in THIS session---and these
457
+ * will only contain the U card created with the gg.zUser value.
458
+ */
459
+ blob_appendf(&record, "U %F\n", ggit.importType == NEW_REPO ?
460
+ gg.zUser : gg.zEmail);
370461
 md5sum_blob(&record, &cksum);
371462
 blob_appendf(&record, "Z %b\n", &cksum);
372463
 db_multi_exec(
373464
 "INSERT OR REPLACE INTO xtag(tname, tcontent)"
374465
 " VALUES(%Q,%Q)", gg.zBranch, blob_str(&record)
@@ -540,20 +631,10 @@
540631
 }
541632
 zName[i] = 0;
542633
 }
543634
544635
545
-static struct{
546
- const char *zMasterName; /* Name of master branch */
547
- int authorFlag; /* Use author as checkin committer */
548
- int nGitAttr; /* Number of Git --attribute entries */
549
- struct { /* Git --attribute details */
550
- char *zUser;
551
- char *zEmail;
552
- } *gitUserInfo;
553
-} ggit;
554
-
555636
 /*
556637
 ** Read the git-fast-import format from pIn and insert the corresponding
557638
 ** content into the database.
558639
 */
559640
 static void git_fast_import(FILE *pIn){
@@ -691,10 +772,11 @@
691772
 }
692773
 if (ggit.nGitAttr > 0 || db_table_exists("repository", "fx_git")) {
693774
 gg.zUser = db_text(gg.zUser,
694775
 "SELECT user FROM fx_git WHERE email=%Q", z);
695776
 }
777
+ gg.zEmail = fossil_strdup(z); /* Keep email for blob table cross-check */
696778
 secSince1970 = 0;
697779
 for(zTo++; fossil_isdigit(*zTo); zTo++){
698780
 secSince1970 = secSince1970*10 + *zTo - '0';
699781
 }
700782
 fossil_free(gg.zDate);
@@ -1723,10 +1805,11 @@
17231805
 */
17241806
 void import_cmd(void){
17251807
 char *zPassword;
17261808
 FILE *pIn;
17271809
 Stmt q;
1810
+ int fd; /* To duplicate stdin file descriptor. */
17281811
 int forceFlag = find_option("force", "f", 0)!=0;
17291812
 int svnFlag = find_option("svn", 0, 0)!=0;
17301813
 int gitFlag = find_option("git", 0, 0)!=0;
17311814
 int omitRebuild = find_option("no-rebuild",0,0)!=0;
17321815
 int omitVacuum = find_option("no-vacuum",0,0)!=0;
@@ -1826,16 +1909,45 @@
18261909
 }
18271910
 if( g.argc==4 ){
18281911
 pIn = fossil_fopen(g.argv[3], "rb");
18291912
 if( pIn==0 ) fossil_fatal("cannot open input file \"%s\"", g.argv[3]);
18301913
 }else{
1831
- pIn = stdin;
1914
+ /*
1915
+ * If piping from stdin with git fast-export, we need to duplicate a fd
1916
+ * so we can also accept keyboard input from the user (see: block at 1931).
1917
+ */
1918
+#if defined(_WIN32) || defined(_WIN64)
1919
+ fd = _dup(fileno(stdin));
1920
+#else /* if UNIX */
1921
+ fd = dup(fileno(stdin));
1922
+#endif /* dup() hack */
1923
+ pIn = fdopen(fd, "r");
1924
+ (void) freopen("/dev/tty", "r", stdin);
18321925
 fossil_binary_mode(pIn);
18331926
 }
1834
- if( !incrFlag ){
1835
- if( forceFlag ) file_delete(g.argv[2]);
1927
+ /*
1928
+ * If neither --incremental nor --force has been passed but the repository
1929
+ * file exists, prompt the user to continue with an incremental import.
1930
+ */
1931
+ if (forceFlag && file_size(g.argv[2], ExtFILE) != -1)
1932
+ file_delete(g.argv[2]);
1933
+ if (!incrFlag && file_size(g.argv[2], ExtFILE) == -1) {
18361934
 db_create_repository(g.argv[2]);
1935
+ ggit.importType = NEW_REPO;
1936
+ } else if (!incrFlag && file_size(g.argv[2], ExtFILE) != -1) {
1937
+ Blob x;
1938
+ char c;
1939
+ fossil_print( "[!] Repository file exists: <%s>\n", g.argv[2]);
1940
+ prompt_user(">>> Proceed with incremental import [Y/n]? ", &x);
1941
+ c = blob_str(&x)[0];
1942
+ blob_reset(&x);
1943
+ incrFlag = (c != 'n' && c != 'N' );
1944
+ if (!incrFlag) {
1945
+ fossil_print("Please either provide an alternative filename, delete "
1946
+ "the\nfile, or use --force to overwrite the existing repository.\n");
1947
+ exit(1);
1948
+ }
18371949
 }
18381950
 db_open_repository(g.argv[2]);
18391951
 db_open_config(0, 0);
18401952
 db_unprotect(PROTECT_ALL);
18411953
@@ -1920,17 +2032,21 @@
19202032
 ** The XTAG table records information about tags that need to be applied
19212033
 ** to various branches after the import finishes. The xtag.tcontent field
19222034
 ** contains the text of an artifact that will add a tag to a check-in.
19232035
 ** The git-fast-export file format might specify the same tag multiple
19242036
 ** times but only the last tag should be used. And we do not know which
1925
- ** occurrence of the tag is the last until the import finishes.
2037
+ ** occurrence of the tag is the last until the import finishes. Also,
2038
+ ** depending on whether --attribute'd, artifacts may contain U cards made
2039
+ ** with either username or emailaddr, so store the last seen version of
2040
+ ** each in the xtag and xtag2 tables, respectively.
19262041
 */
19272042
 db_multi_exec(
19282043
 "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);"
19292044
 "CREATE INDEX temp.i_xmark ON xmark(trid);"
19302045
 "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);"
19312046
 "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);"
2047
+ "CREATE TEMP TABLE xtag2(tname TEXT UNIQUE, tcontent TEXT);"
19322048
 );
19332049
19342050
 if( markfile_in ){
19352051
 FILE *f = fossil_fopen(markfile_in, "r");
19362052
 if( !f ){
@@ -1949,22 +2065,26 @@
19492065
 ** to either a desired username or full contact information string.
19502066
 */
19512067
 if(ggit.nGitAttr > 0) {
19522068
 int idx;
19532069
 db_unprotect(PROTECT_ALL);
1954
- db_multi_exec(
1955
- "CREATE TABLE fx_git(user TEXT, email TEXT UNIQUE);"
1956
- );
2070
+ if (!db_table_exists("repository", "fx_git")) {
2071
+ db_multi_exec(
2072
+ "CREATE TABLE fx_git(user TEXT, email TEXT UNIQUE);"
2073
+ );
2074
+ }
19572075
 for(idx = 0; idx < ggit.nGitAttr; ++idx ){
19582076
 db_multi_exec(
19592077
 "INSERT OR IGNORE INTO fx_git(user, email) VALUES(%Q, %Q)",
19602078
 ggit.gitUserInfo[idx].zUser, ggit.gitUserInfo[idx].zEmail
19612079
 );
19622080
 }
19632081
 db_protect_pop();
19642082
 }
19652083
 git_fast_import(pIn);
2084
+ if (ggit.commitType == COMMIT_ATTR)
2085
+ ggit.commitType = TAG_ATTR;
19662086
 db_prepare(&q, "SELECT tcontent FROM xtag");
19672087
 while( db_step(&q)==SQLITE_ROW ){
19682088
 Blob record;
19692089
 db_ephemeral_blob(&q, 0, &record);
19702090
 fast_insert_content(&record, 0, 0, 0, 1);
19712091
--- src/import.c
+++ src/import.c
@@ -34,10 +34,22 @@
34 char isExe; /* True if executable */
35 char isLink; /* True if symlink */
36 };
37 #endif
38
 
 
 
 
 
 
 
 
 
 
 
 
39 /*
40 ** State information common to all import types.
41 */
42 static struct {
43 const char *zTrunkName; /* Name of trunk branch */
@@ -58,10 +70,11 @@
58 char *zPrevBranch; /* The branch of the previous check-in */
59 char *aData; /* Data content */
60 char *zMark; /* The current mark */
61 char *zDate; /* Date/time stamp */
62 char *zUser; /* User name */
 
63 char *zComment; /* Comment of a commit */
64 char *zFrom; /* from value as a hash */
65 char *zPrevCheckin; /* Name of the previous check-in */
66 char *zFromMark; /* The mark of the "from" field */
67 int nMerge; /* Number of merge values */
@@ -115,10 +128,11 @@
115 fossil_free(gg.zBranch); gg.zBranch = 0;
116 fossil_free(gg.aData); gg.aData = 0;
117 fossil_free(gg.zMark); gg.zMark = 0;
118 fossil_free(gg.zDate); gg.zDate = 0;
119 fossil_free(gg.zUser); gg.zUser = 0;
 
120 fossil_free(gg.zComment); gg.zComment = 0;
121 fossil_free(gg.zFrom); gg.zFrom = 0;
122 fossil_free(gg.zFromMark); gg.zFromMark = 0;
123 for(i=0; i<gg.nMerge; i++){
124 fossil_free(gg.azMerge[i]); gg.azMerge[i] = 0;
@@ -139,10 +153,23 @@
139 memset(&gg, 0, sizeof(gg));
140 }
141 gg.xFinish = finish_noop;
142 }
143
 
 
 
 
 
 
 
 
 
 
 
 
 
144 /*
145 ** Insert an artifact into the BLOB table if it isn't there already.
146 ** If zMark is not zero, create a cross-reference from that mark back
147 ** to the newly inserted artifact.
148 **
@@ -160,10 +187,35 @@
160 Blob cmpr;
161 int rid;
162
163 hname_hash(pContent, 0, &hash);
164 rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165 if( rid==0 ){
166 static Stmt ins;
167 assert( g.rcvid>0 );
168 db_static_prepare(&ins,
169 "INSERT INTO blob(uuid, size, rcvid, content)"
@@ -280,11 +332,11 @@
280 static void finish_commit(void){
281 int i;
282 char *zFromBranch;
283 char *aTCard[4]; /* Array of T cards for manifest */
284 int nTCard = 0; /* Entries used in aTCard[] */
285 Blob record, cksum;
286
287 import_prior_files();
288 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
289 blob_zero(&record);
290 blob_appendf(&record, "C %F\n", gg.zComment);
@@ -342,10 +394,24 @@
342 for(i=0; i<nTCard; i++) free(aTCard[i]);
343
344 free(zFromBranch);
345 db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)",
346 gg.zMark, gg.zBranch);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347 blob_appendf(&record, "U %F\n", gg.zUser);
348 md5sum_blob(&record, &cksum);
349 blob_appendf(&record, "Z %b\n", &cksum);
350 fast_insert_content(&record, gg.zMark, 0, 1, 1);
351 blob_reset(&cksum);
@@ -359,16 +425,41 @@
359 **
360 ** This behavior seems like a bug in git-fast-export, but it is easier
361 ** to work around the problem than to fix git-fast-export.
362 */
363 if( gg.tagCommit && gg.zDate && gg.zUser && gg.zFrom ){
 
364 record.nUsed = 0
365 /*in case fast_insert_comment() did not indirectly blob_reset() it */;
366 blob_appendf(&record, "D %s\n", gg.zDate);
367 blob_appendf(&record, "T +sym-%F%F%F %s\n", gimport.zBranchPre, gg.zBranch,
368 gimport.zBranchSuf, gg.zPrevCheckin);
369 blob_appendf(&record, "U %F\n", gg.zUser);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
370 md5sum_blob(&record, &cksum);
371 blob_appendf(&record, "Z %b\n", &cksum);
372 db_multi_exec(
373 "INSERT OR REPLACE INTO xtag(tname, tcontent)"
374 " VALUES(%Q,%Q)", gg.zBranch, blob_str(&record)
@@ -540,20 +631,10 @@
540 }
541 zName[i] = 0;
542 }
543
544
545 static struct{
546 const char *zMasterName; /* Name of master branch */
547 int authorFlag; /* Use author as checkin committer */
548 int nGitAttr; /* Number of Git --attribute entries */
549 struct { /* Git --attribute details */
550 char *zUser;
551 char *zEmail;
552 } *gitUserInfo;
553 } ggit;
554
555 /*
556 ** Read the git-fast-import format from pIn and insert the corresponding
557 ** content into the database.
558 */
559 static void git_fast_import(FILE *pIn){
@@ -691,10 +772,11 @@
691 }
692 if (ggit.nGitAttr > 0 || db_table_exists("repository", "fx_git")) {
693 gg.zUser = db_text(gg.zUser,
694 "SELECT user FROM fx_git WHERE email=%Q", z);
695 }
 
696 secSince1970 = 0;
697 for(zTo++; fossil_isdigit(*zTo); zTo++){
698 secSince1970 = secSince1970*10 + *zTo - '0';
699 }
700 fossil_free(gg.zDate);
@@ -1723,10 +1805,11 @@
1723 */
1724 void import_cmd(void){
1725 char *zPassword;
1726 FILE *pIn;
1727 Stmt q;
 
1728 int forceFlag = find_option("force", "f", 0)!=0;
1729 int svnFlag = find_option("svn", 0, 0)!=0;
1730 int gitFlag = find_option("git", 0, 0)!=0;
1731 int omitRebuild = find_option("no-rebuild",0,0)!=0;
1732 int omitVacuum = find_option("no-vacuum",0,0)!=0;
@@ -1826,16 +1909,45 @@
1826 }
1827 if( g.argc==4 ){
1828 pIn = fossil_fopen(g.argv[3], "rb");
1829 if( pIn==0 ) fossil_fatal("cannot open input file \"%s\"", g.argv[3]);
1830 }else{
1831 pIn = stdin;
 
 
 
 
 
 
 
 
 
 
1832 fossil_binary_mode(pIn);
1833 }
1834 if( !incrFlag ){
1835 if( forceFlag ) file_delete(g.argv[2]);
 
 
 
 
 
1836 db_create_repository(g.argv[2]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1837 }
1838 db_open_repository(g.argv[2]);
1839 db_open_config(0, 0);
1840 db_unprotect(PROTECT_ALL);
1841
@@ -1920,17 +2032,21 @@
1920 ** The XTAG table records information about tags that need to be applied
1921 ** to various branches after the import finishes. The xtag.tcontent field
1922 ** contains the text of an artifact that will add a tag to a check-in.
1923 ** The git-fast-export file format might specify the same tag multiple
1924 ** times but only the last tag should be used. And we do not know which
1925 ** occurrence of the tag is the last until the import finishes.
 
 
 
1926 */
1927 db_multi_exec(
1928 "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);"
1929 "CREATE INDEX temp.i_xmark ON xmark(trid);"
1930 "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);"
1931 "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);"
 
1932 );
1933
1934 if( markfile_in ){
1935 FILE *f = fossil_fopen(markfile_in, "r");
1936 if( !f ){
@@ -1949,22 +2065,26 @@
1949 ** to either a desired username or full contact information string.
1950 */
1951 if(ggit.nGitAttr > 0) {
1952 int idx;
1953 db_unprotect(PROTECT_ALL);
1954 db_multi_exec(
1955 "CREATE TABLE fx_git(user TEXT, email TEXT UNIQUE);"
1956 );
 
 
1957 for(idx = 0; idx < ggit.nGitAttr; ++idx ){
1958 db_multi_exec(
1959 "INSERT OR IGNORE INTO fx_git(user, email) VALUES(%Q, %Q)",
1960 ggit.gitUserInfo[idx].zUser, ggit.gitUserInfo[idx].zEmail
1961 );
1962 }
1963 db_protect_pop();
1964 }
1965 git_fast_import(pIn);
 
 
1966 db_prepare(&q, "SELECT tcontent FROM xtag");
1967 while( db_step(&q)==SQLITE_ROW ){
1968 Blob record;
1969 db_ephemeral_blob(&q, 0, &record);
1970 fast_insert_content(&record, 0, 0, 0, 1);
1971
--- src/import.c
+++ src/import.c
@@ -34,10 +34,22 @@
34 char isExe; /* True if executable */
35 char isLink; /* True if symlink */
36 };
37 #endif
38
39 /*
40 * Flags to indicate whether the import is to an existing or new repository;
41 * or using the --attribute option and in the process of importing a commit or
42 * tag (to determine the artifact type during fast_insert_content()).
43 */
44 enum import_mode {
45 OLD_REPO,
46 NEW_REPO,
47 COMMIT_ATTR,
48 TAG_ATTR
49 };
50
51 /*
52 ** State information common to all import types.
53 */
54 static struct {
55 const char *zTrunkName; /* Name of trunk branch */
@@ -58,10 +70,11 @@
70 char *zPrevBranch; /* The branch of the previous check-in */
71 char *aData; /* Data content */
72 char *zMark; /* The current mark */
73 char *zDate; /* Date/time stamp */
74 char *zUser; /* User name */
75 char *zEmail; /* Email from Git committer string */
76 char *zComment; /* Comment of a commit */
77 char *zFrom; /* from value as a hash */
78 char *zPrevCheckin; /* Name of the previous check-in */
79 char *zFromMark; /* The mark of the "from" field */
80 int nMerge; /* Number of merge values */
@@ -115,10 +128,11 @@
128 fossil_free(gg.zBranch); gg.zBranch = 0;
129 fossil_free(gg.aData); gg.aData = 0;
130 fossil_free(gg.zMark); gg.zMark = 0;
131 fossil_free(gg.zDate); gg.zDate = 0;
132 fossil_free(gg.zUser); gg.zUser = 0;
133 fossil_free(gg.zEmail); gg.zEmail = 0;
134 fossil_free(gg.zComment); gg.zComment = 0;
135 fossil_free(gg.zFrom); gg.zFrom = 0;
136 fossil_free(gg.zFromMark); gg.zFromMark = 0;
137 for(i=0; i<gg.nMerge; i++){
138 fossil_free(gg.azMerge[i]); gg.azMerge[i] = 0;
@@ -139,10 +153,23 @@
153 memset(&gg, 0, sizeof(gg));
154 }
155 gg.xFinish = finish_noop;
156 }
157
158 static struct{
159 const char *zMasterName; /* Name of master branch */
160 int authorFlag; /* Use author as checkin committer */
161 Blob altRecord; /* Record to cross-check --attribute'd commits */
162 uint8_t commitType:2; /* To indicate --attribute'd manifest or tag */
163 uint8_t importType:2; /* To indicate import into new or existing repo */
164 int nGitAttr; /* Number of Git --attribute entries */
165 struct { /* Git --attribute details */
166 char *zUser;
167 char *zEmail;
168 } *gitUserInfo;
169 } ggit;
170
171 /*
172 ** Insert an artifact into the BLOB table if it isn't there already.
173 ** If zMark is not zero, create a cross-reference from that mark back
174 ** to the newly inserted artifact.
175 **
@@ -160,10 +187,35 @@
187 Blob cmpr;
188 int rid;
189
190 hname_hash(pContent, 0, &hash);
191 rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
192 /*
193 * If this repo has had --attribute'd commits, we need to check the blob
194 * table for manifests with U cards constructed from both the username and
195 * email address to ensure no duplicate entries are attempted.
196 */
197 if (ggit.commitType == COMMIT_ATTR && rid == 0) {
198 blob_reset(&hash);
199 hname_hash(&ggit.altRecord, 0, &hash);
200 rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
201 }
202 /*
203 * Likewise, for --attribute'd tags pop the tag artifact with the alternate
204 * U card from the temp xtag2 table; both these tags and the above commit
205 * artifacts are the same as pContent albeit with the {user|email} U card.
206 */
207 if (ggit.commitType == TAG_ATTR && rid == 0) {
208 db_blob(&ggit.altRecord,
209 "SELECT tcontent FROM xtag2 ORDER BY ROWID ASC LIMIT 1");
210 db_multi_exec(
211 "DELETE FROM xtag2 WHERE tcontent=%Q", blob_str(&ggit.altRecord)
212 );
213 blob_reset(&hash);
214 hname_hash(&ggit.altRecord, 0, &hash);
215 rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%B", &hash);
216 }
217 if( rid==0 ){
218 static Stmt ins;
219 assert( g.rcvid>0 );
220 db_static_prepare(&ins,
221 "INSERT INTO blob(uuid, size, rcvid, content)"
@@ -280,11 +332,11 @@
332 static void finish_commit(void){
333 int i;
334 char *zFromBranch;
335 char *aTCard[4]; /* Array of T cards for manifest */
336 int nTCard = 0; /* Entries used in aTCard[] */
337 Blob record, cksum, altCksum;
338
339 import_prior_files();
340 qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp);
341 blob_zero(&record);
342 blob_appendf(&record, "C %F\n", gg.zComment);
@@ -342,10 +394,24 @@
394 for(i=0; i<nTCard; i++) free(aTCard[i]);
395
396 free(zFromBranch);
397 db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)",
398 gg.zMark, gg.zBranch);
399 /*
400 * The fx_git table indicates this repo has --attribute'd commits; therefore,
401 * blob entries may exist with U cards generated from either a username or
402 * email address. Create both for cross-checking to avoid adding duplicates.
403 */
404 if (db_table_exists("repository", "fx_git") &&
405 db_text(0, "SELECT user FROM fx_git WHERE email=%Q", gg.zEmail)) {
406 blob_copy(&ggit.altRecord, &record);
407 blob_appendf(&ggit.altRecord, "U %F\n", gg.zEmail);
408 md5sum_blob(&ggit.altRecord, &altCksum);
409 blob_appendf(&ggit.altRecord, "Z %b\n", &altCksum);
410 blob_reset(&altCksum);
411 ggit.commitType = COMMIT_ATTR;
412 }
413 blob_appendf(&record, "U %F\n", gg.zUser);
414 md5sum_blob(&record, &cksum);
415 blob_appendf(&record, "Z %b\n", &cksum);
416 fast_insert_content(&record, gg.zMark, 0, 1, 1);
417 blob_reset(&cksum);
@@ -359,16 +425,41 @@
425 **
426 ** This behavior seems like a bug in git-fast-export, but it is easier
427 ** to work around the problem than to fix git-fast-export.
428 */
429 if( gg.tagCommit && gg.zDate && gg.zUser && gg.zFrom ){
430 Blob altTag;
431 record.nUsed = 0
432 /*in case fast_insert_comment() did not indirectly blob_reset() it */;
433 blob_appendf(&record, "D %s\n", gg.zDate);
434 blob_appendf(&record, "T +sym-%F%F%F %s\n", gimport.zBranchPre, gg.zBranch,
435 gimport.zBranchSuf, gg.zPrevCheckin);
436 /*
437 * If --attribute'd commits are present, we also need tag artifacts of the
438 * other potential U card value to be cross-checked against the blob table.
439 * Due to the abovementioned Git bug, store in a different table: xtag2.
440 */
441 if (ggit.commitType == COMMIT_ATTR) {
442 blob_copy(&altTag, &record);
443 blob_appendf(&altTag, "U %F\n", gg.zEmail);
444 md5sum_blob(&altTag, &altCksum);
445 blob_appendf(&altTag, "Z %b\n", &altCksum);
446 db_multi_exec(
447 "INSERT OR REPLACE INTO xtag2(tname, tcontent)"
448 " VALUES(%Q,%Q)", gg.zBranch, blob_str(&altTag)
449 );
450 blob_reset(&altCksum);
451 blob_reset(&altTag);
452 ggit.commitType = TAG_ATTR;
453 }
454 /*
455 * IF this is a brand new repo, ONLY one type of tag artifact can exist in
456 * the blob table, which are those generated in THIS session---and these
457 * will only contain the U card created with the gg.zUser value.
458 */
459 blob_appendf(&record, "U %F\n", ggit.importType == NEW_REPO ?
460 gg.zUser : gg.zEmail);
461 md5sum_blob(&record, &cksum);
462 blob_appendf(&record, "Z %b\n", &cksum);
463 db_multi_exec(
464 "INSERT OR REPLACE INTO xtag(tname, tcontent)"
465 " VALUES(%Q,%Q)", gg.zBranch, blob_str(&record)
@@ -540,20 +631,10 @@
631 }
632 zName[i] = 0;
633 }
634
635
 
 
 
 
 
 
 
 
 
 
636 /*
637 ** Read the git-fast-import format from pIn and insert the corresponding
638 ** content into the database.
639 */
640 static void git_fast_import(FILE *pIn){
@@ -691,10 +772,11 @@
772 }
773 if (ggit.nGitAttr > 0 || db_table_exists("repository", "fx_git")) {
774 gg.zUser = db_text(gg.zUser,
775 "SELECT user FROM fx_git WHERE email=%Q", z);
776 }
777 gg.zEmail = fossil_strdup(z); /* Keep email for blob table cross-check */
778 secSince1970 = 0;
779 for(zTo++; fossil_isdigit(*zTo); zTo++){
780 secSince1970 = secSince1970*10 + *zTo - '0';
781 }
782 fossil_free(gg.zDate);
@@ -1723,10 +1805,11 @@
1805 */
1806 void import_cmd(void){
1807 char *zPassword;
1808 FILE *pIn;
1809 Stmt q;
1810 int fd; /* To duplicate stdin file descriptor. */
1811 int forceFlag = find_option("force", "f", 0)!=0;
1812 int svnFlag = find_option("svn", 0, 0)!=0;
1813 int gitFlag = find_option("git", 0, 0)!=0;
1814 int omitRebuild = find_option("no-rebuild",0,0)!=0;
1815 int omitVacuum = find_option("no-vacuum",0,0)!=0;
@@ -1826,16 +1909,45 @@
1909 }
1910 if( g.argc==4 ){
1911 pIn = fossil_fopen(g.argv[3], "rb");
1912 if( pIn==0 ) fossil_fatal("cannot open input file \"%s\"", g.argv[3]);
1913 }else{
1914 /*
1915 * If piping from stdin with git fast-export, we need to duplicate a fd
1916 * so we can also accept keyboard input from the user (see: block at 1931).
1917 */
1918 #if defined(_WIN32) || defined(_WIN64)
1919 fd = _dup(fileno(stdin));
1920 #else /* if UNIX */
1921 fd = dup(fileno(stdin));
1922 #endif /* dup() hack */
1923 pIn = fdopen(fd, "r");
1924 (void) freopen("/dev/tty", "r", stdin);
1925 fossil_binary_mode(pIn);
1926 }
1927 /*
1928 * If neither --incremental nor --force has been passed but the repository
1929 * file exists, prompt the user to continue with an incremental import.
1930 */
1931 if (forceFlag && file_size(g.argv[2], ExtFILE) != -1)
1932 file_delete(g.argv[2]);
1933 if (!incrFlag && file_size(g.argv[2], ExtFILE) == -1) {
1934 db_create_repository(g.argv[2]);
1935 ggit.importType = NEW_REPO;
1936 } else if (!incrFlag && file_size(g.argv[2], ExtFILE) != -1) {
1937 Blob x;
1938 char c;
1939 fossil_print( "[!] Repository file exists: <%s>\n", g.argv[2]);
1940 prompt_user(">>> Proceed with incremental import [Y/n]? ", &x);
1941 c = blob_str(&x)[0];
1942 blob_reset(&x);
1943 incrFlag = (c != 'n' && c != 'N' );
1944 if (!incrFlag) {
1945 fossil_print("Please either provide an alternative filename, delete "
1946 "the\nfile, or use --force to overwrite the existing repository.\n");
1947 exit(1);
1948 }
1949 }
1950 db_open_repository(g.argv[2]);
1951 db_open_config(0, 0);
1952 db_unprotect(PROTECT_ALL);
1953
@@ -1920,17 +2032,21 @@
2032 ** The XTAG table records information about tags that need to be applied
2033 ** to various branches after the import finishes. The xtag.tcontent field
2034 ** contains the text of an artifact that will add a tag to a check-in.
2035 ** The git-fast-export file format might specify the same tag multiple
2036 ** times but only the last tag should be used. And we do not know which
2037 ** occurrence of the tag is the last until the import finishes. Also,
2038 ** depending on whether --attribute'd, artifacts may contain U cards made
2039 ** with either username or emailaddr, so store the last seen version of
2040 ** each in the xtag and xtag2 tables, respectively.
2041 */
2042 db_multi_exec(
2043 "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);"
2044 "CREATE INDEX temp.i_xmark ON xmark(trid);"
2045 "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);"
2046 "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);"
2047 "CREATE TEMP TABLE xtag2(tname TEXT UNIQUE, tcontent TEXT);"
2048 );
2049
2050 if( markfile_in ){
2051 FILE *f = fossil_fopen(markfile_in, "r");
2052 if( !f ){
@@ -1949,22 +2065,26 @@
2065 ** to either a desired username or full contact information string.
2066 */
2067 if(ggit.nGitAttr > 0) {
2068 int idx;
2069 db_unprotect(PROTECT_ALL);
2070 if (!db_table_exists("repository", "fx_git")) {
2071 db_multi_exec(
2072 "CREATE TABLE fx_git(user TEXT, email TEXT UNIQUE);"
2073 );
2074 }
2075 for(idx = 0; idx < ggit.nGitAttr; ++idx ){
2076 db_multi_exec(
2077 "INSERT OR IGNORE INTO fx_git(user, email) VALUES(%Q, %Q)",
2078 ggit.gitUserInfo[idx].zUser, ggit.gitUserInfo[idx].zEmail
2079 );
2080 }
2081 db_protect_pop();
2082 }
2083 git_fast_import(pIn);
2084 if (ggit.commitType == COMMIT_ATTR)
2085 ggit.commitType = TAG_ATTR;
2086 db_prepare(&q, "SELECT tcontent FROM xtag");
2087 while( db_step(&q)==SQLITE_ROW ){
2088 Blob record;
2089 db_ephemeral_blob(&q, 0, &record);
2090 fast_insert_content(&record, 0, 0, 0, 1);
2091

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button