Fossil SCM

Add the "fossil git export" command, designed to simplify making a Git mirror of a working Fossil repository.

drh 2019-03-16 00:58 trunk merge
Commit 702959420fa6a5878fd0dc2db0b5b88d19c461d8a42f88b88b5e7a01efc9af83
Parent 60af514d13969c7…
1 file changed +618 -38
~ src/export.c
M src/export.c
+618 -38
--- src/export.c
+++ src/export.c
@@ -28,23 +28,17 @@
2828
 const char *zTrunkName; /* Name of trunk branch */
2929
 } gexport;
3030
3131
 #if INTERFACE
3232
 /*
33
-** struct mark_t
34
-** holds information for translating between git commits
35
-** and fossil commits.
36
-** -git_name: This is the mark name that identifies the commit to git.
37
-** It will always begin with a ':'.
38
-** -rid: The unique object ID that identifies this commit within the
39
-** repository database.
40
-** -uuid: The SHA-1/SHA-3 of artifact corresponding to rid.
33
+** Each line in a git-fast-export "marK" file is an instance of
34
+** this object.
4135
 */
42
-struct mark_t{
43
- char *name;
44
- int rid;
45
- char uuid[65];
36
+struct mark_t {
37
+ char *name; /* Name of the mark. Also starts with ":" */
38
+ int rid; /* Corresponding object in the BLOB table */
39
+ char uuid[65]; /* The GIT hash name for this object */
4640
 };
4741
 #endif
4842
4943
 /*
5044
 ** Output a "committer" record for the given user.
@@ -297,18 +291,18 @@
297291
 }
298292
 return zMark;
299293
 }
300294
301295
 /*
302
-** parse_mark()
303
-** Create a new (mark,rid,uuid) entry in the 'xmark' table given a line
304
-** from a marks file. Return the cross-ref information as a struct mark_t
305
-** in *mark.
306
-** This function returns -1 in the case that the line is blank, malformed, or
307
-** the rid/uuid named in 'line' does not match what is in the repository
308
-** database. Otherwise, 0 is returned.
309
-** mark->name is dynamically allocated, and owned by the caller.
296
+** Parse a single line of the mark file. Store the result in the mark object.
297
+**
298
+** "line" is a single line of input.
299
+** This function returns -1 in the case that the line is blank, malformed, or
300
+** the rid/uuid named in 'line' does not match what is in the repository
301
+** database. Otherwise, 0 is returned.
302
+**
303
+** mark->name is dynamically allocated, and owned by the caller.
310304
 */
311305
 int parse_mark(char *line, struct mark_t *mark){
312306
 char *cur_tok;
313307
 char type_;
314308
 cur_tok = strtok(line, " \t");
@@ -361,23 +355,24 @@
361355
 insert_commit_xref(mark->rid, mark->name, mark->uuid);
362356
 return 0;
363357
 }
364358
365359
 /*
366
-** import_marks()
367
-** Import the marks specified in file 'f' into the 'xmark' table.
368
-** If 'blobs' is non-null, insert all blob marks into it.
369
-** If 'vers' is non-null, insert all commit marks into it.
370
-** If 'unused_marks' is non-null, upon return of this function, all values
371
-** x >= *unused_marks are free to use as marks, i.e. they do not clash with
372
-** any marks appearing in the marks file.
373
-** Each line in the file must be at most 100 characters in length. This
374
-** seems like a reasonable maximum for a 40-character uuid, and 1-13
375
-** character rid.
376
-** The function returns -1 if any of the lines in file 'f' are malformed,
377
-** or the rid/uuid information doesn't match what is in the repository
378
-** database. Otherwise, 0 is returned.
360
+** Import the marks specified in file 'f';
361
+** If 'blobs' is non-null, insert all blob marks into it.
362
+** If 'vers' is non-null, insert all commit marks into it.
363
+** If 'unused_marks' is non-null, upon return of this function, all values
364
+** x >= *unused_marks are free to use as marks, i.e. they do not clash with
365
+** any marks appearing in the marks file.
366
+**
367
+** Each line in the file must be at most 100 characters in length. This
368
+** seems like a reasonable maximum for a 40-character uuid, and 1-13
369
+** character rid.
370
+**
371
+** The function returns -1 if any of the lines in file 'f' are malformed,
372
+** or the rid/uuid information doesn't match what is in the repository
373
+** database. Otherwise, 0 is returned.
379374
 */
380375
 int import_marks(FILE* f, Bag *blobs, Bag *vers, unsigned int *unused_mark){
381376
 char line[101];
382377
 while(fgets(line, sizeof(line), f)){
383378
 struct mark_t mark;
@@ -452,13 +447,13 @@
452447
 }while( (rid = bag_next(vers, rid))!=0 );
453448
 }
454449
 }
455450
 }
456451
457
-/*
458
-** COMMAND: export
459
-**
452
+/* This is the original header command (and hence documentation) for
453
+** the "fossil export" command:
454
+**
460455
 ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY?
461456
 **
462457
 ** Write an export of all check-ins to standard output. The export is
463458
 ** written in the git-fast-export file format assuming the --git option is
464459
 ** provided. The git-fast-export format is currently the only VCS
@@ -483,10 +478,15 @@
483478
 ** --rename-trunk NAME use NAME as name of exported trunk branch
484479
 ** --repository|-R REPOSITORY export the given REPOSITORY
485480
 **
486481
 ** See also: import
487482
 */
483
+/*
484
+** COMMAND: export*
485
+**
486
+** This command is deprecated. Use "fossil git export" instead.
487
+*/
488488
 void export_cmd(void){
489489
 Stmt q, q2, q3;
490490
 Bag blobs, vers;
491491
 unsigned int unused_mark = 1;
492492
 const char *markfile_in;
@@ -507,11 +507,12 @@
507507
 verify_all_options();
508508
 if( g.argc!=2 && g.argc!=3 ){ usage("--git ?REPOSITORY?"); }
509509
510510
 db_multi_exec("CREATE TEMPORARY TABLE oldblob(rid INTEGER PRIMARY KEY)");
511511
 db_multi_exec("CREATE TEMPORARY TABLE oldcommit(rid INTEGER PRIMARY KEY)");
512
- db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT)");
512
+ db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT,"
513
+ " tuuid TEXT)");
513514
 db_multi_exec("CREATE INDEX xmark_trid ON xmark(trid)");
514515
 if( markfile_in!=0 ){
515516
 Stmt qb,qc;
516517
 FILE *f;
517518
 int rid;
@@ -754,11 +755,11 @@
754755
 ** );
755756
 **
756757
 ** This table contains all check-ins of the repository in topological
757758
 ** order. "Topological order" means that every parent check-in comes
758759
 ** before all of its children. Topological order is *almost* the same
759
-** thing as "ORDER BY event.mtime". Differences only arrise when there
760
+** thing as "ORDER BY event.mtime". Differences only arise when there
760761
 ** are timewarps. In as much as Git hates timewarps, we have to compute
761762
 ** a correct topological order when doing an export.
762763
 **
763764
 ** Since mtime is a usually already nearly in topological order, the
764765
 ** algorithm is to start with mtime, then make adjustments as necessary
@@ -835,5 +836,584 @@
835836
 int n;
836837
 db_find_and_open_repository(0, 0);
837838
 n = topological_sort_checkins(1);
838839
 fossil_print("%d reorderings required\n", n);
839840
 }
841
+
842
+/***************************************************************************
843
+** Implementation of the "fossil git" command follows. We hope that the
844
+** new code that follows will largely replace the legacy "fossil export"
845
+** and "fossil import" code above.
846
+*/
847
+
848
+/*
849
+** Convert characters of z[] that are not allowed to be in branch or
850
+** tag names into "_".
851
+*/
852
+static void gitmirror_sanitize_name(char *z){
853
+ static unsigned char aSafe[] = {
854
+ /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
855
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
856
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
857
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 2x */
858
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* 3x */
859
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
860
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, /* 5x */
861
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
862
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, /* 7x */
863
+ };
864
+ unsigned char *zu = (unsigned char*)z;
865
+ int i;
866
+ for(i=0; zu[i]; i++){
867
+ if( zu[i]>0x7f || !aSafe[zu[i]] ){
868
+ zu[i] = '_';
869
+ }else if( zu[i]=='/' && (i==0 || zu[i+1]==0 || zu[i+1]=='/') ){
870
+ zu[i] = '_';
871
+ }else if( zu[i]=='.' && (zu[i+1]==0 || zu[i+1]=='.'
872
+ || (i>0 && zu[i-1]=='.')) ){
873
+ zu[i] = '_';
874
+ }
875
+ }
876
+}
877
+
878
+/*
879
+** Quote a filename as a C-style string using \\ and \" if necessary.
880
+** If quoting is not necessary, just return a copy of the input string.
881
+**
882
+** The return value is a held in memory obtained from fossil_malloc()
883
+** and must be freed by the caller.
884
+*/
885
+static char *gitmirror_quote_filename_if_needed(const char *zIn){
886
+ int i, j;
887
+ char c;
888
+ int nSpecial = 0;
889
+ char *zOut;
890
+ for(i=0; (c = zIn[i])!=0; i++){
891
+ if( c=='\\' || c=='"' || c=='\n' ){
892
+ nSpecial++;
893
+ }
894
+ }
895
+ if( nSpecial==0 ){
896
+ return fossil_strdup(zIn);
897
+ }
898
+ zOut = fossil_malloc( i+nSpecial+3 );
899
+ zOut[0] = '"';
900
+ for(i=0, j=1; (c = zIn[i])!=0; i++){
901
+ if( c=='\\' || c=='"' || c=='\n' ){
902
+ zOut[j++] = '\\';
903
+ if( c=='\n' ){
904
+ zOut[j++] = 'n';
905
+ }else{
906
+ zOut[j++] = c;
907
+ }
908
+ }else{
909
+ zOut[j++] = c;
910
+ }
911
+ }
912
+ zOut[j++] = '"';
913
+ zOut[j] = 0;
914
+ return zOut;
915
+}
916
+
917
+/*
918
+** Locate the mark for a UUID.
919
+**
920
+** If the mark does not exist and if the bCreate flag is false, then
921
+** return 0. If the mark does not exist and the bCreate flag is true,
922
+** then create the mark.
923
+*/
924
+static int gitmirror_find_mark(const char *zUuid, int bCreate){
925
+ int iMark;
926
+ static Stmt sFind, sIns;
927
+ db_static_prepare(&sFind,
928
+ "SELECT id FROM mirror.mmark WHERE uuid=:uuid"
929
+ );
930
+ db_bind_text(&sFind, ":uuid", zUuid);
931
+ if( db_step(&sFind)==SQLITE_ROW ){
932
+ iMark = db_column_int(&sFind, 0);
933
+ db_reset(&sFind);
934
+ return iMark;
935
+ }
936
+ db_reset(&sFind);
937
+ if( !bCreate ) return 0;
938
+ db_static_prepare(&sIns,
939
+ "INSERT INTO mirror.mmark(uuid) VALUES(:uuid)"
940
+ );
941
+ db_bind_text(&sIns, ":uuid", zUuid);
942
+ db_step(&sIns);
943
+ db_reset(&sIns);
944
+ return db_last_insert_rowid();
945
+}
946
+
947
+/* This is the SHA3-256 hash of an empty file */
948
+static const char zEmptySha3[] =
949
+ "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a";
950
+
951
+/*
952
+** Export a single file named by zUuid.
953
+*/
954
+static void gitmirror_send_file(FILE *xCmd, const char *zUuid){
955
+ int iMark;
956
+ int rid;
957
+ int rc;
958
+ Blob data;
959
+ rid = fast_uuid_to_rid(zUuid);
960
+ if( rid<0 ){
961
+ zUuid = zEmptySha3;
962
+ }else{
963
+ rc = content_get(rid, &data);
964
+ if( rc==0 ){
965
+ blob_init(&data, 0, 0);
966
+ zUuid = zEmptySha3;
967
+ }
968
+ }
969
+ iMark = gitmirror_find_mark(zUuid, 1);
970
+ fprintf(xCmd, "blob\nmark :%d\ndata %d\n", iMark, blob_size(&data));
971
+ fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd);
972
+ fprintf(xCmd, "\n");
973
+ blob_reset(&data);
974
+}
975
+
976
+/*
977
+** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for
978
+** the check-in to export.
979
+**
980
+** If any ancestor of the check-in has not yet been exported, then
981
+** invoke this routine recursively to export the ancestor first.
982
+** This can only happen on a timewarp, so deep nesting is unlikely.
983
+**
984
+** Before sending the check-in, first make sure all associated files
985
+** have already been exported, and send "blob" records for any that
986
+** have not been. Update the MIRROR.MMARK table so that it holds the
987
+** marks for the exported files.
988
+*/
989
+static void gitmirror_send_checkin(
990
+ FILE *xCmd, /* Write fast-import text on this pipe */
991
+ int rid, /* BLOB.RID for the check-in to export */
992
+ const char *zUuid, /* BLOB.UUID for the check-in to export */
993
+ int *pnLimit, /* Stop when the counter reaches zero */
994
+ int fManifest /* MFESTFLG_* values */
995
+){
996
+ Manifest *pMan; /* The check-in to be output */
997
+ int i; /* Loop counter */
998
+ int iParent; /* Which immediate ancestor is primary. -1 for none */
999
+ Stmt q; /* An SQL query */
1000
+ char *zBranch; /* The branch of the check-in */
1001
+ int iMark; /* The mark for the check-in */
1002
+ Blob sql; /* String of SQL for part of the query */
1003
+ Blob comment; /* The comment text for the check-in */
1004
+
1005
+ pMan = manifest_get(rid, CFTYPE_MANIFEST, 0);
1006
+ if( pMan==0 ){
1007
+ /* Must be a phantom. Return without doing anything, and in particular
1008
+ ** without creating a mark for this check-in. */
1009
+ return;
1010
+ }
1011
+
1012
+ /* Check to see if any parent logins have not yet been processed, and
1013
+ ** if so, create them */
1014
+ for(i=0; i<pMan->nParent; i++){
1015
+ int iMark = gitmirror_find_mark(pMan->azParent[i], 0);
1016
+ if( iMark<=0 ){
1017
+ int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q",
1018
+ pMan->azParent[i]);
1019
+ gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], pnLimit, fManifest);
1020
+ if( *pnLimit<=0 ){
1021
+ manifest_destroy(pMan);
1022
+ return;
1023
+ }
1024
+ }
1025
+ }
1026
+
1027
+ /* Make sure all necessary files have been exported */
1028
+ db_prepare(&q,
1029
+ "SELECT uuid FROM files_of_checkin(%Q)"
1030
+ " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)",
1031
+ zUuid
1032
+ );
1033
+ while( db_step(&q)==SQLITE_ROW ){
1034
+ const char *zFUuid = db_column_text(&q, 0);
1035
+ gitmirror_send_file(xCmd, zFUuid);
1036
+ }
1037
+ db_finalize(&q);
1038
+
1039
+ /* Figure out which branch this check-in is a member of */
1040
+ zBranch = db_text(0,
1041
+ "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d",
1042
+ TAG_BRANCH, rid
1043
+ );
1044
+ if( fossil_strcmp(zBranch,"trunk")==0 ){
1045
+ fossil_free(zBranch);
1046
+ zBranch = mprintf("master");
1047
+ }else if( zBranch==0 ){
1048
+ zBranch = mprintf("unknown");
1049
+ }else{
1050
+ gitmirror_sanitize_name(zBranch);
1051
+ }
1052
+
1053
+ /* Export the check-in */
1054
+ fprintf(xCmd, "commit refs/heads/%s\n", zBranch);
1055
+ fossil_free(zBranch);
1056
+ iMark = gitmirror_find_mark(zUuid, 1);
1057
+ fprintf(xCmd, "mark :%d\n", iMark);
1058
+ fprintf(xCmd, "committer %s <%[email protected]> %lld +0000\n",
1059
+ pMan->zUser, pMan->zUser,
1060
+ (sqlite3_int64)((pMan->rDate-2440587.5)*86400.0)
1061
+ );
1062
+ blob_init(&comment, pMan->zComment, -1);
1063
+ if( blob_size(&comment)==0 ){
1064
+ blob_append(&comment, "(no comment)", -1);
1065
+ }
1066
+ blob_appendf(&comment, "\n\nFossilOrigin-Name: %s", zUuid);
1067
+ fprintf(xCmd, "data %d\n%s\n", blob_size(&comment), blob_str(&comment));
1068
+ blob_reset(&comment);
1069
+ iParent = -1; /* Which ancestor is the primary parent */
1070
+ for(i=0; i<pMan->nParent; i++){
1071
+ int iOther = gitmirror_find_mark(pMan->azParent[i], 0);
1072
+ if( iOther==0 ) continue;
1073
+ if( iParent<0 ){
1074
+ iParent = i;
1075
+ fprintf(xCmd, "from :%d\n", iOther);
1076
+ }else{
1077
+ fprintf(xCmd, "merge :%d\n", iOther);
1078
+ }
1079
+ }
1080
+ if( iParent>=0 ){
1081
+ db_prepare(&q,
1082
+ "SELECT filename FROM files_of_checkin(%Q)"
1083
+ " EXCEPT SELECT filename FROM files_of_checkin(%Q)",
1084
+ pMan->azParent[iParent], zUuid
1085
+ );
1086
+ while( db_step(&q)==SQLITE_ROW ){
1087
+ fprintf(xCmd, "D %s\n", db_column_text(&q,0));
1088
+ }
1089
+ db_finalize(&q);
1090
+ }
1091
+ blob_init(&sql, 0, 0);
1092
+ blob_append_sql(&sql,
1093
+ "SELECT filename, uuid, perm FROM files_of_checkin(%Q)",
1094
+ zUuid
1095
+ );
1096
+ if( pMan->nParent ){
1097
+ blob_append_sql(&sql,
1098
+ " EXCEPT SELECT filename, uuid, perm FROM files_of_checkin(%Q)",
1099
+ pMan->azParent[0]);
1100
+ }
1101
+ db_prepare(&q,
1102
+ "SELECT x.filename, x.perm, mmark.id FROM (%s) AS x, mirror.mmark"
1103
+ " WHERE mmark.uuid=x.uuid",
1104
+ blob_sql_text(&sql)
1105
+ );
1106
+ blob_reset(&sql);
1107
+ while( db_step(&q)==SQLITE_ROW ){
1108
+ const char *zFilename = db_column_text(&q,0);
1109
+ const char *zMode = db_column_text(&q,1);
1110
+ int iMark = db_column_int(&q,2);
1111
+ const char *zGitMode = "100644";
1112
+ char *zFNQuoted = 0;
1113
+ if( zMode ){
1114
+ if( strchr(zMode,'x') ) zGitMode = "100755";
1115
+ if( strchr(zMode,'l') ) zGitMode = "120000";
1116
+ }
1117
+ zFNQuoted = gitmirror_quote_filename_if_needed(zFilename);
1118
+ fprintf(xCmd,"M %s :%d %s\n", zGitMode, iMark, zFNQuoted);
1119
+ fossil_free(zFNQuoted);
1120
+ }
1121
+ db_finalize(&q);
1122
+
1123
+ /* Include Fossil-generated auxiliary files in the check-in */
1124
+ if( fManifest & MFESTFLG_RAW ){
1125
+ Blob manifest;
1126
+ content_get(rid, &manifest);
1127
+ fprintf(xCmd,"M 100644 inline manifest\ndata %d\n%s\n",
1128
+ blob_size(&manifest), blob_str(&manifest));
1129
+ blob_reset(&manifest);
1130
+ }
1131
+ if( fManifest & MFESTFLG_UUID ){
1132
+ int n = (int)strlen(zUuid);
1133
+ fprintf(xCmd,"M 100644 inline manifest.uuid\ndata %d\n%s\n", n, zUuid);
1134
+ }
1135
+ if( fManifest & MFESTFLG_TAGS ){
1136
+ Blob tagslist;
1137
+ blob_init(&tagslist, 0, 0);
1138
+ get_checkin_taglist(rid, &tagslist);
1139
+ fprintf(xCmd,"M 100644 inline manifest.tags\ndata %d\n%s\n",
1140
+ blob_size(&tagslist), blob_str(&tagslist));
1141
+ blob_reset(&tagslist);
1142
+ }
1143
+
1144
+ /* The check-in is finished, so decrement the counter */
1145
+ (*pnLimit)--;
1146
+}
1147
+
1148
+/*
1149
+** Implementation of the "fossil git export" command.
1150
+*/
1151
+void gitmirror_export_command(void){
1152
+ const char *zLimit; /* Text of the --limit flag */
1153
+ int nLimit = 0x7fffffff; /* Numeric value of the --limit flag */
1154
+ int nTotal = 0; /* Total number of check-ins to export */
1155
+ char *zMirror; /* Name of the mirror */
1156
+ char *z; /* Generic string */
1157
+ char *zCmd; /* git command to run as a subprocess */
1158
+ const char *zDebug = 0; /* Value of the --debug flag */
1159
+ double rEnd; /* time of most recent export */
1160
+ int rc; /* Result code */
1161
+ int fManifest; /* Current "manifest" setting */
1162
+ FILE *xCmd; /* Pipe to the "git fast-import" command */
1163
+ FILE *pIn, *pOut; /* Git mark files */
1164
+ Stmt q; /* Queries */
1165
+ char zLine[200]; /* One line of a mark file */
1166
+
1167
+ zDebug = find_option("debug",0,1);
1168
+ db_find_and_open_repository(0, 0);
1169
+ zLimit = find_option("limit", 0, 1);
1170
+ if( zLimit ){
1171
+ nLimit = (unsigned int)atoi(zLimit);
1172
+ if( nLimit<=0 ) fossil_fatal("--limit must be positive");
1173
+ }
1174
+ verify_all_options();
1175
+ if( g.argc!=4 ){ usage("export MIRROR"); }
1176
+ zMirror = g.argv[3];
1177
+
1178
+ /* Make sure the GIT repository directory exists */
1179
+ rc = file_mkdir(zMirror, ExtFILE, 0);
1180
+ if( rc ) fossil_fatal("cannot create directory \"%s\"", zMirror);
1181
+
1182
+ /* Make sure GIT has been initialized */
1183
+ z = mprintf("%s/.git", zMirror);
1184
+ if( !file_isdir(z, ExtFILE) ){
1185
+ zCmd = mprintf("git init '%s'",zMirror);
1186
+ fossil_print("%s\n", zCmd);
1187
+ rc = fossil_system(zCmd);
1188
+ if( rc ){
1189
+ fossil_fatal("command failed: \"%s\"", zCmd);
1190
+ }
1191
+ fossil_free(zCmd);
1192
+ }
1193
+ fossil_free(z);
1194
+
1195
+ /* Make sure the .mirror_state subdirectory exists */
1196
+ z = mprintf("%s/.mirror_state", zMirror);
1197
+ rc = file_mkdir(z, ExtFILE, 0);
1198
+ if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
1199
+ fossil_free(z);
1200
+
1201
+ /* Attach the .mirror_state/db database */
1202
+ db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
1203
+ db_begin_write();
1204
+ db_multi_exec(
1205
+ "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
1206
+ " key TEXT PRIMARY KEY,\n"
1207
+ " Value ANY\n"
1208
+ ") WITHOUT ROWID;\n"
1209
+ "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
1210
+ " id INTEGER PRIMARY KEY,\n"
1211
+ " uuid TEXT UNIQUE,\n"
1212
+ " githash TEXT\n"
1213
+ ");"
1214
+ );
1215
+
1216
+ /* See if there is any work to be done. Exit early if not, before starting
1217
+ ** the "git fast-import" command. */
1218
+ if( !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')"
1219
+ " AND mtime>coalesce((SELECT value FROM mconfig"
1220
+ " WHERE key='start'),0.0)")
1221
+ ){
1222
+ fossil_print("no changes\n");
1223
+ return;
1224
+ }
1225
+
1226
+ /* Do we need to include manifest files in the clone? */
1227
+ fManifest = db_get_manifest_setting();
1228
+
1229
+ /* Change to the MIRROR directory so that the Git commands will work */
1230
+ rc = file_chdir(zMirror, 0);
1231
+ if( rc ) fossil_fatal("cannot change the working directory to \"%s\"",
1232
+ zMirror);
1233
+
1234
+ /* Start up the git fast-import command */
1235
+ if( zDebug ){
1236
+ if( fossil_strcmp(zDebug,"stdout")==0 ){
1237
+ xCmd = stdout;
1238
+ }else{
1239
+ xCmd = fopen(zDebug, "wb");
1240
+ if( xCmd==0 ) fossil_fatal("cannot open file \"%s\" for writing", zDebug);
1241
+ }
1242
+ }else{
1243
+ zCmd = mprintf("git fast-import"
1244
+ " --import-marks-if-exists=.mirror_state/in"
1245
+ " --export-marks=.mirror_state/out"
1246
+ " --quiet --done");
1247
+ fossil_print("%s\n", zCmd);
1248
+ xCmd = popen(zCmd, "w");
1249
+ if( zCmd==0 ){
1250
+ fossil_fatal("cannot start the \"git fast-import\" command");
1251
+ }
1252
+ fossil_free(zCmd);
1253
+ }
1254
+
1255
+ /* Run the export */
1256
+ rEnd = 0.0;
1257
+ db_multi_exec(
1258
+ "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n"
1259
+ "INSERT INTO tomirror "
1260
+ "SELECT objid, mtime, blob.uuid FROM event, blob\n"
1261
+ " WHERE type='ci'"
1262
+ " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
1263
+ " AND blob.rid=event.objid"
1264
+ " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
1265
+ );
1266
+ nTotal = db_int(0, "SELECT count(*) FROM tomirror");
1267
+ if( nLimit<nTotal ){
1268
+ nTotal = nLimit;
1269
+ }else if( nLimit>nTotal ){
1270
+ nLimit = nTotal;
1271
+ }
1272
+ db_prepare(&q,
1273
+ "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime"
1274
+ );
1275
+ while( nLimit && db_step(&q)==SQLITE_ROW ){
1276
+ int rid = db_column_int(&q, 0);
1277
+ double rMTime = db_column_double(&q, 1);
1278
+ const char *zUuid = db_column_text(&q, 2);
1279
+ if( rMTime>rEnd ) rEnd = rMTime;
1280
+ gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1281
+ printf("\r%d/%d ", nTotal-nLimit, nTotal);
1282
+ fflush(stdout);
1283
+ }
1284
+ db_finalize(&q);
1285
+ fprintf(xCmd, "done\n");
1286
+ if( zDebug ){
1287
+ if( xCmd!=stdout ) fclose(xCmd);
1288
+ }else{
1289
+ pclose(xCmd);
1290
+ }
1291
+ fossil_print("%d check-ins added to the mirror\n", nTotal-nLimit);
1292
+
1293
+ /* Read the export-marks file. Transfer the new marks over into
1294
+ ** the import-marks file.
1295
+ */
1296
+ pOut = fopen(".mirror_state/out", "rb");
1297
+ if( pOut ){
1298
+ pIn = fopen(".mirror_state/in", "ab");
1299
+ if( pIn==0 ){
1300
+ fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
1301
+ }
1302
+ db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id");
1303
+ while( fgets(zLine, sizeof(zLine), pOut) ){
1304
+ int j, k;
1305
+ if( zLine[0]!=':' ) continue;
1306
+ db_bind_int(&q, ":id", atoi(zLine+1));
1307
+ for(j=1; zLine[j] && zLine[j]!=' '; j++){}
1308
+ if( zLine[j]!=' ' ) continue;
1309
+ j++;
1310
+ if( zLine[j]==0 ) continue;
1311
+ for(k=j; fossil_isalnum(zLine[k]); k++){}
1312
+ zLine[k] = 0;
1313
+ db_bind_text(&q, ":githash", &zLine[j]);
1314
+ db_step(&q);
1315
+ db_reset(&q);
1316
+ zLine[k] = '\n';
1317
+ fputs(zLine, pIn);
1318
+ }
1319
+ db_finalize(&q);
1320
+ fclose(pOut);
1321
+ fclose(pIn);
1322
+ file_delete(".mirror_state/out");
1323
+ }else{
1324
+ fossil_fatal("git fast-import didn't generate a marks file!");
1325
+ }
1326
+ db_multi_exec(
1327
+ "CREATE INDEX IF NOT EXISTS mirror.mmarkx1 ON mmark(githash);"
1328
+ );
1329
+
1330
+ /* Do any tags that have been created since the start time */
1331
+ db_prepare(&q,
1332
+ "SELECT substr(tagname,5), githash"
1333
+ " FROM (SELECT tagxref.tagid AS xtagid, tagname, rid, max(mtime) AS mtime"
1334
+ " FROM tagxref JOIN tag ON tag.tagid=tagxref.tagid"
1335
+ " WHERE tag.tagname GLOB 'sym-*'"
1336
+ " AND tagxref.tagtype=1"
1337
+ " AND tagxref.mtime > coalesce((SELECT value FROM mconfig"
1338
+ " WHERE key='start'),0.0)"
1339
+ " GROUP BY tagxref.tagid) AS tx"
1340
+ " JOIN blob ON tx.rid=blob.rid"
1341
+ " JOIN mmark ON mmark.uuid=blob.uuid;"
1342
+ );
1343
+ while( db_step(&q)==SQLITE_ROW ){
1344
+ char *zTagname = fossil_strdup(db_column_text(&q,0));
1345
+ const char *zObj = db_column_text(&q,1);
1346
+ char *zTagCmd;
1347
+ gitmirror_sanitize_name(zTagname);
1348
+ zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj);
1349
+ fossil_free(zTagname);
1350
+ fossil_print("%s\n", zTagCmd);
1351
+ fossil_system(zTagCmd);
1352
+ fossil_free(zTagCmd);
1353
+ }
1354
+ db_finalize(&q);
1355
+
1356
+ /* Update the start time */
1357
+ db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
1358
+ db_bind_double(&q, ":x", rEnd);
1359
+ db_step(&q);
1360
+ db_finalize(&q);
1361
+ db_commit_transaction();
1362
+
1363
+ /* Optionally do a "git push" */
1364
+}
1365
+
1366
+/*
1367
+** COMMAND: git
1368
+**
1369
+** Usage: %fossil git SUBCOMMAND
1370
+**
1371
+** Do incremental import or export operations between Fossil and Git.
1372
+** Subcommands:
1373
+**
1374
+** fossil git export MIRROR [OPTIONS]
1375
+**
1376
+** Write content from the Fossil repository into the Git repository
1377
+** in directory MIRROR. The Git repository is created if it does not
1378
+** already exist. If the Git repository does already exist, then
1379
+** new content added to fossil since the previous export is appended.
1380
+**
1381
+** Repeat this command whenever new checkins are added to the Fossil
1382
+** repository in order to reflect those changes into the mirror.
1383
+**
1384
+** The MIRROR directory will contain a subdirectory named
1385
+** ".mirror_state" that contains information that Fossil needs to
1386
+** do incremental exports. Do not attempt to manage or edit the files
1387
+** in that directory since doing so can disrupt future incremental
1388
+** exports.
1389
+**
1390
+** Options:
1391
+** --debug FILE Write fast-export text to FILE rather than
1392
+** piping it into "git fast-import".
1393
+** --limit N Add no more than N new check-ins to MIRROR.
1394
+** Useful for debugging
1395
+**
1396
+** fossil git import MIRROR
1397
+**
1398
+** TBD...
1399
+*/
1400
+void gitmirror_command(void){
1401
+ char *zCmd;
1402
+ int nCmd;
1403
+ if( g.argc<3 ){
1404
+ usage("export ARGS...");
1405
+ }
1406
+ zCmd = g.argv[2];
1407
+ nCmd = (int)strlen(zCmd);
1408
+ if( nCmd>2 && strncmp(zCmd,"export",nCmd)==0 ){
1409
+ gitmirror_export_command();
1410
+ }else
1411
+ if( nCmd>2 && strncmp(zCmd,"import",nCmd)==0 ){
1412
+ fossil_fatal("not yet implemented - check back later");
1413
+ }else
1414
+ {
1415
+ fossil_fatal("unknown subcommand \"%s\": should be one of "
1416
+ "\"export\", \"import\"",
1417
+ zCmd);
1418
+ }
1419
+}
8401420
--- src/export.c
+++ src/export.c
@@ -28,23 +28,17 @@
28 const char *zTrunkName; /* Name of trunk branch */
29 } gexport;
30
31 #if INTERFACE
32 /*
33 ** struct mark_t
34 ** holds information for translating between git commits
35 ** and fossil commits.
36 ** -git_name: This is the mark name that identifies the commit to git.
37 ** It will always begin with a ':'.
38 ** -rid: The unique object ID that identifies this commit within the
39 ** repository database.
40 ** -uuid: The SHA-1/SHA-3 of artifact corresponding to rid.
41 */
42 struct mark_t{
43 char *name;
44 int rid;
45 char uuid[65];
46 };
47 #endif
48
49 /*
50 ** Output a "committer" record for the given user.
@@ -297,18 +291,18 @@
297 }
298 return zMark;
299 }
300
301 /*
302 ** parse_mark()
303 ** Create a new (mark,rid,uuid) entry in the 'xmark' table given a line
304 ** from a marks file. Return the cross-ref information as a struct mark_t
305 ** in *mark.
306 ** This function returns -1 in the case that the line is blank, malformed, or
307 ** the rid/uuid named in 'line' does not match what is in the repository
308 ** database. Otherwise, 0 is returned.
309 ** mark->name is dynamically allocated, and owned by the caller.
310 */
311 int parse_mark(char *line, struct mark_t *mark){
312 char *cur_tok;
313 char type_;
314 cur_tok = strtok(line, " \t");
@@ -361,23 +355,24 @@
361 insert_commit_xref(mark->rid, mark->name, mark->uuid);
362 return 0;
363 }
364
365 /*
366 ** import_marks()
367 ** Import the marks specified in file 'f' into the 'xmark' table.
368 ** If 'blobs' is non-null, insert all blob marks into it.
369 ** If 'vers' is non-null, insert all commit marks into it.
370 ** If 'unused_marks' is non-null, upon return of this function, all values
371 ** x >= *unused_marks are free to use as marks, i.e. they do not clash with
372 ** any marks appearing in the marks file.
373 ** Each line in the file must be at most 100 characters in length. This
374 ** seems like a reasonable maximum for a 40-character uuid, and 1-13
375 ** character rid.
376 ** The function returns -1 if any of the lines in file 'f' are malformed,
377 ** or the rid/uuid information doesn't match what is in the repository
378 ** database. Otherwise, 0 is returned.
 
379 */
380 int import_marks(FILE* f, Bag *blobs, Bag *vers, unsigned int *unused_mark){
381 char line[101];
382 while(fgets(line, sizeof(line), f)){
383 struct mark_t mark;
@@ -452,13 +447,13 @@
452 }while( (rid = bag_next(vers, rid))!=0 );
453 }
454 }
455 }
456
457 /*
458 ** COMMAND: export
459 **
460 ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY?
461 **
462 ** Write an export of all check-ins to standard output. The export is
463 ** written in the git-fast-export file format assuming the --git option is
464 ** provided. The git-fast-export format is currently the only VCS
@@ -483,10 +478,15 @@
483 ** --rename-trunk NAME use NAME as name of exported trunk branch
484 ** --repository|-R REPOSITORY export the given REPOSITORY
485 **
486 ** See also: import
487 */
 
 
 
 
 
488 void export_cmd(void){
489 Stmt q, q2, q3;
490 Bag blobs, vers;
491 unsigned int unused_mark = 1;
492 const char *markfile_in;
@@ -507,11 +507,12 @@
507 verify_all_options();
508 if( g.argc!=2 && g.argc!=3 ){ usage("--git ?REPOSITORY?"); }
509
510 db_multi_exec("CREATE TEMPORARY TABLE oldblob(rid INTEGER PRIMARY KEY)");
511 db_multi_exec("CREATE TEMPORARY TABLE oldcommit(rid INTEGER PRIMARY KEY)");
512 db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT)");
 
513 db_multi_exec("CREATE INDEX xmark_trid ON xmark(trid)");
514 if( markfile_in!=0 ){
515 Stmt qb,qc;
516 FILE *f;
517 int rid;
@@ -754,11 +755,11 @@
754 ** );
755 **
756 ** This table contains all check-ins of the repository in topological
757 ** order. "Topological order" means that every parent check-in comes
758 ** before all of its children. Topological order is *almost* the same
759 ** thing as "ORDER BY event.mtime". Differences only arrise when there
760 ** are timewarps. In as much as Git hates timewarps, we have to compute
761 ** a correct topological order when doing an export.
762 **
763 ** Since mtime is a usually already nearly in topological order, the
764 ** algorithm is to start with mtime, then make adjustments as necessary
@@ -835,5 +836,584 @@
835 int n;
836 db_find_and_open_repository(0, 0);
837 n = topological_sort_checkins(1);
838 fossil_print("%d reorderings required\n", n);
839 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
840
--- src/export.c
+++ src/export.c
@@ -28,23 +28,17 @@
28 const char *zTrunkName; /* Name of trunk branch */
29 } gexport;
30
31 #if INTERFACE
32 /*
33 ** Each line in a git-fast-export "marK" file is an instance of
34 ** this object.
 
 
 
 
 
 
35 */
36 struct mark_t {
37 char *name; /* Name of the mark. Also starts with ":" */
38 int rid; /* Corresponding object in the BLOB table */
39 char uuid[65]; /* The GIT hash name for this object */
40 };
41 #endif
42
43 /*
44 ** Output a "committer" record for the given user.
@@ -297,18 +291,18 @@
291 }
292 return zMark;
293 }
294
295 /*
296 ** Parse a single line of the mark file. Store the result in the mark object.
297 **
298 ** "line" is a single line of input.
299 ** This function returns -1 in the case that the line is blank, malformed, or
300 ** the rid/uuid named in 'line' does not match what is in the repository
301 ** database. Otherwise, 0 is returned.
302 **
303 ** mark->name is dynamically allocated, and owned by the caller.
304 */
305 int parse_mark(char *line, struct mark_t *mark){
306 char *cur_tok;
307 char type_;
308 cur_tok = strtok(line, " \t");
@@ -361,23 +355,24 @@
355 insert_commit_xref(mark->rid, mark->name, mark->uuid);
356 return 0;
357 }
358
359 /*
360 ** Import the marks specified in file 'f';
361 ** If 'blobs' is non-null, insert all blob marks into it.
362 ** If 'vers' is non-null, insert all commit marks into it.
363 ** If 'unused_marks' is non-null, upon return of this function, all values
364 ** x >= *unused_marks are free to use as marks, i.e. they do not clash with
365 ** any marks appearing in the marks file.
366 **
367 ** Each line in the file must be at most 100 characters in length. This
368 ** seems like a reasonable maximum for a 40-character uuid, and 1-13
369 ** character rid.
370 **
371 ** The function returns -1 if any of the lines in file 'f' are malformed,
372 ** or the rid/uuid information doesn't match what is in the repository
373 ** database. Otherwise, 0 is returned.
374 */
375 int import_marks(FILE* f, Bag *blobs, Bag *vers, unsigned int *unused_mark){
376 char line[101];
377 while(fgets(line, sizeof(line), f)){
378 struct mark_t mark;
@@ -452,13 +447,13 @@
447 }while( (rid = bag_next(vers, rid))!=0 );
448 }
449 }
450 }
451
452 /* This is the original header command (and hence documentation) for
453 ** the "fossil export" command:
454 **
455 ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY?
456 **
457 ** Write an export of all check-ins to standard output. The export is
458 ** written in the git-fast-export file format assuming the --git option is
459 ** provided. The git-fast-export format is currently the only VCS
@@ -483,10 +478,15 @@
478 ** --rename-trunk NAME use NAME as name of exported trunk branch
479 ** --repository|-R REPOSITORY export the given REPOSITORY
480 **
481 ** See also: import
482 */
483 /*
484 ** COMMAND: export*
485 **
486 ** This command is deprecated. Use "fossil git export" instead.
487 */
488 void export_cmd(void){
489 Stmt q, q2, q3;
490 Bag blobs, vers;
491 unsigned int unused_mark = 1;
492 const char *markfile_in;
@@ -507,11 +507,12 @@
507 verify_all_options();
508 if( g.argc!=2 && g.argc!=3 ){ usage("--git ?REPOSITORY?"); }
509
510 db_multi_exec("CREATE TEMPORARY TABLE oldblob(rid INTEGER PRIMARY KEY)");
511 db_multi_exec("CREATE TEMPORARY TABLE oldcommit(rid INTEGER PRIMARY KEY)");
512 db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT,"
513 " tuuid TEXT)");
514 db_multi_exec("CREATE INDEX xmark_trid ON xmark(trid)");
515 if( markfile_in!=0 ){
516 Stmt qb,qc;
517 FILE *f;
518 int rid;
@@ -754,11 +755,11 @@
755 ** );
756 **
757 ** This table contains all check-ins of the repository in topological
758 ** order. "Topological order" means that every parent check-in comes
759 ** before all of its children. Topological order is *almost* the same
760 ** thing as "ORDER BY event.mtime". Differences only arise when there
761 ** are timewarps. In as much as Git hates timewarps, we have to compute
762 ** a correct topological order when doing an export.
763 **
764 ** Since mtime is a usually already nearly in topological order, the
765 ** algorithm is to start with mtime, then make adjustments as necessary
@@ -835,5 +836,584 @@
836 int n;
837 db_find_and_open_repository(0, 0);
838 n = topological_sort_checkins(1);
839 fossil_print("%d reorderings required\n", n);
840 }
841
842 /***************************************************************************
843 ** Implementation of the "fossil git" command follows. We hope that the
844 ** new code that follows will largely replace the legacy "fossil export"
845 ** and "fossil import" code above.
846 */
847
848 /*
849 ** Convert characters of z[] that are not allowed to be in branch or
850 ** tag names into "_".
851 */
852 static void gitmirror_sanitize_name(char *z){
853 static unsigned char aSafe[] = {
854 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
855 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
856 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
857 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 2x */
858 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* 3x */
859 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
860 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, /* 5x */
861 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
862 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, /* 7x */
863 };
864 unsigned char *zu = (unsigned char*)z;
865 int i;
866 for(i=0; zu[i]; i++){
867 if( zu[i]>0x7f || !aSafe[zu[i]] ){
868 zu[i] = '_';
869 }else if( zu[i]=='/' && (i==0 || zu[i+1]==0 || zu[i+1]=='/') ){
870 zu[i] = '_';
871 }else if( zu[i]=='.' && (zu[i+1]==0 || zu[i+1]=='.'
872 || (i>0 && zu[i-1]=='.')) ){
873 zu[i] = '_';
874 }
875 }
876 }
877
878 /*
879 ** Quote a filename as a C-style string using \\ and \" if necessary.
880 ** If quoting is not necessary, just return a copy of the input string.
881 **
882 ** The return value is a held in memory obtained from fossil_malloc()
883 ** and must be freed by the caller.
884 */
885 static char *gitmirror_quote_filename_if_needed(const char *zIn){
886 int i, j;
887 char c;
888 int nSpecial = 0;
889 char *zOut;
890 for(i=0; (c = zIn[i])!=0; i++){
891 if( c=='\\' || c=='"' || c=='\n' ){
892 nSpecial++;
893 }
894 }
895 if( nSpecial==0 ){
896 return fossil_strdup(zIn);
897 }
898 zOut = fossil_malloc( i+nSpecial+3 );
899 zOut[0] = '"';
900 for(i=0, j=1; (c = zIn[i])!=0; i++){
901 if( c=='\\' || c=='"' || c=='\n' ){
902 zOut[j++] = '\\';
903 if( c=='\n' ){
904 zOut[j++] = 'n';
905 }else{
906 zOut[j++] = c;
907 }
908 }else{
909 zOut[j++] = c;
910 }
911 }
912 zOut[j++] = '"';
913 zOut[j] = 0;
914 return zOut;
915 }
916
917 /*
918 ** Locate the mark for a UUID.
919 **
920 ** If the mark does not exist and if the bCreate flag is false, then
921 ** return 0. If the mark does not exist and the bCreate flag is true,
922 ** then create the mark.
923 */
924 static int gitmirror_find_mark(const char *zUuid, int bCreate){
925 int iMark;
926 static Stmt sFind, sIns;
927 db_static_prepare(&sFind,
928 "SELECT id FROM mirror.mmark WHERE uuid=:uuid"
929 );
930 db_bind_text(&sFind, ":uuid", zUuid);
931 if( db_step(&sFind)==SQLITE_ROW ){
932 iMark = db_column_int(&sFind, 0);
933 db_reset(&sFind);
934 return iMark;
935 }
936 db_reset(&sFind);
937 if( !bCreate ) return 0;
938 db_static_prepare(&sIns,
939 "INSERT INTO mirror.mmark(uuid) VALUES(:uuid)"
940 );
941 db_bind_text(&sIns, ":uuid", zUuid);
942 db_step(&sIns);
943 db_reset(&sIns);
944 return db_last_insert_rowid();
945 }
946
947 /* This is the SHA3-256 hash of an empty file */
948 static const char zEmptySha3[] =
949 "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a";
950
951 /*
952 ** Export a single file named by zUuid.
953 */
954 static void gitmirror_send_file(FILE *xCmd, const char *zUuid){
955 int iMark;
956 int rid;
957 int rc;
958 Blob data;
959 rid = fast_uuid_to_rid(zUuid);
960 if( rid<0 ){
961 zUuid = zEmptySha3;
962 }else{
963 rc = content_get(rid, &data);
964 if( rc==0 ){
965 blob_init(&data, 0, 0);
966 zUuid = zEmptySha3;
967 }
968 }
969 iMark = gitmirror_find_mark(zUuid, 1);
970 fprintf(xCmd, "blob\nmark :%d\ndata %d\n", iMark, blob_size(&data));
971 fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd);
972 fprintf(xCmd, "\n");
973 blob_reset(&data);
974 }
975
976 /*
977 ** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for
978 ** the check-in to export.
979 **
980 ** If any ancestor of the check-in has not yet been exported, then
981 ** invoke this routine recursively to export the ancestor first.
982 ** This can only happen on a timewarp, so deep nesting is unlikely.
983 **
984 ** Before sending the check-in, first make sure all associated files
985 ** have already been exported, and send "blob" records for any that
986 ** have not been. Update the MIRROR.MMARK table so that it holds the
987 ** marks for the exported files.
988 */
989 static void gitmirror_send_checkin(
990 FILE *xCmd, /* Write fast-import text on this pipe */
991 int rid, /* BLOB.RID for the check-in to export */
992 const char *zUuid, /* BLOB.UUID for the check-in to export */
993 int *pnLimit, /* Stop when the counter reaches zero */
994 int fManifest /* MFESTFLG_* values */
995 ){
996 Manifest *pMan; /* The check-in to be output */
997 int i; /* Loop counter */
998 int iParent; /* Which immediate ancestor is primary. -1 for none */
999 Stmt q; /* An SQL query */
1000 char *zBranch; /* The branch of the check-in */
1001 int iMark; /* The mark for the check-in */
1002 Blob sql; /* String of SQL for part of the query */
1003 Blob comment; /* The comment text for the check-in */
1004
1005 pMan = manifest_get(rid, CFTYPE_MANIFEST, 0);
1006 if( pMan==0 ){
1007 /* Must be a phantom. Return without doing anything, and in particular
1008 ** without creating a mark for this check-in. */
1009 return;
1010 }
1011
1012 /* Check to see if any parent logins have not yet been processed, and
1013 ** if so, create them */
1014 for(i=0; i<pMan->nParent; i++){
1015 int iMark = gitmirror_find_mark(pMan->azParent[i], 0);
1016 if( iMark<=0 ){
1017 int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q",
1018 pMan->azParent[i]);
1019 gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], pnLimit, fManifest);
1020 if( *pnLimit<=0 ){
1021 manifest_destroy(pMan);
1022 return;
1023 }
1024 }
1025 }
1026
1027 /* Make sure all necessary files have been exported */
1028 db_prepare(&q,
1029 "SELECT uuid FROM files_of_checkin(%Q)"
1030 " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)",
1031 zUuid
1032 );
1033 while( db_step(&q)==SQLITE_ROW ){
1034 const char *zFUuid = db_column_text(&q, 0);
1035 gitmirror_send_file(xCmd, zFUuid);
1036 }
1037 db_finalize(&q);
1038
1039 /* Figure out which branch this check-in is a member of */
1040 zBranch = db_text(0,
1041 "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d",
1042 TAG_BRANCH, rid
1043 );
1044 if( fossil_strcmp(zBranch,"trunk")==0 ){
1045 fossil_free(zBranch);
1046 zBranch = mprintf("master");
1047 }else if( zBranch==0 ){
1048 zBranch = mprintf("unknown");
1049 }else{
1050 gitmirror_sanitize_name(zBranch);
1051 }
1052
1053 /* Export the check-in */
1054 fprintf(xCmd, "commit refs/heads/%s\n", zBranch);
1055 fossil_free(zBranch);
1056 iMark = gitmirror_find_mark(zUuid, 1);
1057 fprintf(xCmd, "mark :%d\n", iMark);
1058 fprintf(xCmd, "committer %s <%[email protected]> %lld +0000\n",
1059 pMan->zUser, pMan->zUser,
1060 (sqlite3_int64)((pMan->rDate-2440587.5)*86400.0)
1061 );
1062 blob_init(&comment, pMan->zComment, -1);
1063 if( blob_size(&comment)==0 ){
1064 blob_append(&comment, "(no comment)", -1);
1065 }
1066 blob_appendf(&comment, "\n\nFossilOrigin-Name: %s", zUuid);
1067 fprintf(xCmd, "data %d\n%s\n", blob_size(&comment), blob_str(&comment));
1068 blob_reset(&comment);
1069 iParent = -1; /* Which ancestor is the primary parent */
1070 for(i=0; i<pMan->nParent; i++){
1071 int iOther = gitmirror_find_mark(pMan->azParent[i], 0);
1072 if( iOther==0 ) continue;
1073 if( iParent<0 ){
1074 iParent = i;
1075 fprintf(xCmd, "from :%d\n", iOther);
1076 }else{
1077 fprintf(xCmd, "merge :%d\n", iOther);
1078 }
1079 }
1080 if( iParent>=0 ){
1081 db_prepare(&q,
1082 "SELECT filename FROM files_of_checkin(%Q)"
1083 " EXCEPT SELECT filename FROM files_of_checkin(%Q)",
1084 pMan->azParent[iParent], zUuid
1085 );
1086 while( db_step(&q)==SQLITE_ROW ){
1087 fprintf(xCmd, "D %s\n", db_column_text(&q,0));
1088 }
1089 db_finalize(&q);
1090 }
1091 blob_init(&sql, 0, 0);
1092 blob_append_sql(&sql,
1093 "SELECT filename, uuid, perm FROM files_of_checkin(%Q)",
1094 zUuid
1095 );
1096 if( pMan->nParent ){
1097 blob_append_sql(&sql,
1098 " EXCEPT SELECT filename, uuid, perm FROM files_of_checkin(%Q)",
1099 pMan->azParent[0]);
1100 }
1101 db_prepare(&q,
1102 "SELECT x.filename, x.perm, mmark.id FROM (%s) AS x, mirror.mmark"
1103 " WHERE mmark.uuid=x.uuid",
1104 blob_sql_text(&sql)
1105 );
1106 blob_reset(&sql);
1107 while( db_step(&q)==SQLITE_ROW ){
1108 const char *zFilename = db_column_text(&q,0);
1109 const char *zMode = db_column_text(&q,1);
1110 int iMark = db_column_int(&q,2);
1111 const char *zGitMode = "100644";
1112 char *zFNQuoted = 0;
1113 if( zMode ){
1114 if( strchr(zMode,'x') ) zGitMode = "100755";
1115 if( strchr(zMode,'l') ) zGitMode = "120000";
1116 }
1117 zFNQuoted = gitmirror_quote_filename_if_needed(zFilename);
1118 fprintf(xCmd,"M %s :%d %s\n", zGitMode, iMark, zFNQuoted);
1119 fossil_free(zFNQuoted);
1120 }
1121 db_finalize(&q);
1122
1123 /* Include Fossil-generated auxiliary files in the check-in */
1124 if( fManifest & MFESTFLG_RAW ){
1125 Blob manifest;
1126 content_get(rid, &manifest);
1127 fprintf(xCmd,"M 100644 inline manifest\ndata %d\n%s\n",
1128 blob_size(&manifest), blob_str(&manifest));
1129 blob_reset(&manifest);
1130 }
1131 if( fManifest & MFESTFLG_UUID ){
1132 int n = (int)strlen(zUuid);
1133 fprintf(xCmd,"M 100644 inline manifest.uuid\ndata %d\n%s\n", n, zUuid);
1134 }
1135 if( fManifest & MFESTFLG_TAGS ){
1136 Blob tagslist;
1137 blob_init(&tagslist, 0, 0);
1138 get_checkin_taglist(rid, &tagslist);
1139 fprintf(xCmd,"M 100644 inline manifest.tags\ndata %d\n%s\n",
1140 blob_size(&tagslist), blob_str(&tagslist));
1141 blob_reset(&tagslist);
1142 }
1143
1144 /* The check-in is finished, so decrement the counter */
1145 (*pnLimit)--;
1146 }
1147
1148 /*
1149 ** Implementation of the "fossil git export" command.
1150 */
1151 void gitmirror_export_command(void){
1152 const char *zLimit; /* Text of the --limit flag */
1153 int nLimit = 0x7fffffff; /* Numeric value of the --limit flag */
1154 int nTotal = 0; /* Total number of check-ins to export */
1155 char *zMirror; /* Name of the mirror */
1156 char *z; /* Generic string */
1157 char *zCmd; /* git command to run as a subprocess */
1158 const char *zDebug = 0; /* Value of the --debug flag */
1159 double rEnd; /* time of most recent export */
1160 int rc; /* Result code */
1161 int fManifest; /* Current "manifest" setting */
1162 FILE *xCmd; /* Pipe to the "git fast-import" command */
1163 FILE *pIn, *pOut; /* Git mark files */
1164 Stmt q; /* Queries */
1165 char zLine[200]; /* One line of a mark file */
1166
1167 zDebug = find_option("debug",0,1);
1168 db_find_and_open_repository(0, 0);
1169 zLimit = find_option("limit", 0, 1);
1170 if( zLimit ){
1171 nLimit = (unsigned int)atoi(zLimit);
1172 if( nLimit<=0 ) fossil_fatal("--limit must be positive");
1173 }
1174 verify_all_options();
1175 if( g.argc!=4 ){ usage("export MIRROR"); }
1176 zMirror = g.argv[3];
1177
1178 /* Make sure the GIT repository directory exists */
1179 rc = file_mkdir(zMirror, ExtFILE, 0);
1180 if( rc ) fossil_fatal("cannot create directory \"%s\"", zMirror);
1181
1182 /* Make sure GIT has been initialized */
1183 z = mprintf("%s/.git", zMirror);
1184 if( !file_isdir(z, ExtFILE) ){
1185 zCmd = mprintf("git init '%s'",zMirror);
1186 fossil_print("%s\n", zCmd);
1187 rc = fossil_system(zCmd);
1188 if( rc ){
1189 fossil_fatal("command failed: \"%s\"", zCmd);
1190 }
1191 fossil_free(zCmd);
1192 }
1193 fossil_free(z);
1194
1195 /* Make sure the .mirror_state subdirectory exists */
1196 z = mprintf("%s/.mirror_state", zMirror);
1197 rc = file_mkdir(z, ExtFILE, 0);
1198 if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
1199 fossil_free(z);
1200
1201 /* Attach the .mirror_state/db database */
1202 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
1203 db_begin_write();
1204 db_multi_exec(
1205 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
1206 " key TEXT PRIMARY KEY,\n"
1207 " Value ANY\n"
1208 ") WITHOUT ROWID;\n"
1209 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
1210 " id INTEGER PRIMARY KEY,\n"
1211 " uuid TEXT UNIQUE,\n"
1212 " githash TEXT\n"
1213 ");"
1214 );
1215
1216 /* See if there is any work to be done. Exit early if not, before starting
1217 ** the "git fast-import" command. */
1218 if( !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')"
1219 " AND mtime>coalesce((SELECT value FROM mconfig"
1220 " WHERE key='start'),0.0)")
1221 ){
1222 fossil_print("no changes\n");
1223 return;
1224 }
1225
1226 /* Do we need to include manifest files in the clone? */
1227 fManifest = db_get_manifest_setting();
1228
1229 /* Change to the MIRROR directory so that the Git commands will work */
1230 rc = file_chdir(zMirror, 0);
1231 if( rc ) fossil_fatal("cannot change the working directory to \"%s\"",
1232 zMirror);
1233
1234 /* Start up the git fast-import command */
1235 if( zDebug ){
1236 if( fossil_strcmp(zDebug,"stdout")==0 ){
1237 xCmd = stdout;
1238 }else{
1239 xCmd = fopen(zDebug, "wb");
1240 if( xCmd==0 ) fossil_fatal("cannot open file \"%s\" for writing", zDebug);
1241 }
1242 }else{
1243 zCmd = mprintf("git fast-import"
1244 " --import-marks-if-exists=.mirror_state/in"
1245 " --export-marks=.mirror_state/out"
1246 " --quiet --done");
1247 fossil_print("%s\n", zCmd);
1248 xCmd = popen(zCmd, "w");
1249 if( zCmd==0 ){
1250 fossil_fatal("cannot start the \"git fast-import\" command");
1251 }
1252 fossil_free(zCmd);
1253 }
1254
1255 /* Run the export */
1256 rEnd = 0.0;
1257 db_multi_exec(
1258 "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n"
1259 "INSERT INTO tomirror "
1260 "SELECT objid, mtime, blob.uuid FROM event, blob\n"
1261 " WHERE type='ci'"
1262 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
1263 " AND blob.rid=event.objid"
1264 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
1265 );
1266 nTotal = db_int(0, "SELECT count(*) FROM tomirror");
1267 if( nLimit<nTotal ){
1268 nTotal = nLimit;
1269 }else if( nLimit>nTotal ){
1270 nLimit = nTotal;
1271 }
1272 db_prepare(&q,
1273 "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime"
1274 );
1275 while( nLimit && db_step(&q)==SQLITE_ROW ){
1276 int rid = db_column_int(&q, 0);
1277 double rMTime = db_column_double(&q, 1);
1278 const char *zUuid = db_column_text(&q, 2);
1279 if( rMTime>rEnd ) rEnd = rMTime;
1280 gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1281 printf("\r%d/%d ", nTotal-nLimit, nTotal);
1282 fflush(stdout);
1283 }
1284 db_finalize(&q);
1285 fprintf(xCmd, "done\n");
1286 if( zDebug ){
1287 if( xCmd!=stdout ) fclose(xCmd);
1288 }else{
1289 pclose(xCmd);
1290 }
1291 fossil_print("%d check-ins added to the mirror\n", nTotal-nLimit);
1292
1293 /* Read the export-marks file. Transfer the new marks over into
1294 ** the import-marks file.
1295 */
1296 pOut = fopen(".mirror_state/out", "rb");
1297 if( pOut ){
1298 pIn = fopen(".mirror_state/in", "ab");
1299 if( pIn==0 ){
1300 fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
1301 }
1302 db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id");
1303 while( fgets(zLine, sizeof(zLine), pOut) ){
1304 int j, k;
1305 if( zLine[0]!=':' ) continue;
1306 db_bind_int(&q, ":id", atoi(zLine+1));
1307 for(j=1; zLine[j] && zLine[j]!=' '; j++){}
1308 if( zLine[j]!=' ' ) continue;
1309 j++;
1310 if( zLine[j]==0 ) continue;
1311 for(k=j; fossil_isalnum(zLine[k]); k++){}
1312 zLine[k] = 0;
1313 db_bind_text(&q, ":githash", &zLine[j]);
1314 db_step(&q);
1315 db_reset(&q);
1316 zLine[k] = '\n';
1317 fputs(zLine, pIn);
1318 }
1319 db_finalize(&q);
1320 fclose(pOut);
1321 fclose(pIn);
1322 file_delete(".mirror_state/out");
1323 }else{
1324 fossil_fatal("git fast-import didn't generate a marks file!");
1325 }
1326 db_multi_exec(
1327 "CREATE INDEX IF NOT EXISTS mirror.mmarkx1 ON mmark(githash);"
1328 );
1329
1330 /* Do any tags that have been created since the start time */
1331 db_prepare(&q,
1332 "SELECT substr(tagname,5), githash"
1333 " FROM (SELECT tagxref.tagid AS xtagid, tagname, rid, max(mtime) AS mtime"
1334 " FROM tagxref JOIN tag ON tag.tagid=tagxref.tagid"
1335 " WHERE tag.tagname GLOB 'sym-*'"
1336 " AND tagxref.tagtype=1"
1337 " AND tagxref.mtime > coalesce((SELECT value FROM mconfig"
1338 " WHERE key='start'),0.0)"
1339 " GROUP BY tagxref.tagid) AS tx"
1340 " JOIN blob ON tx.rid=blob.rid"
1341 " JOIN mmark ON mmark.uuid=blob.uuid;"
1342 );
1343 while( db_step(&q)==SQLITE_ROW ){
1344 char *zTagname = fossil_strdup(db_column_text(&q,0));
1345 const char *zObj = db_column_text(&q,1);
1346 char *zTagCmd;
1347 gitmirror_sanitize_name(zTagname);
1348 zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj);
1349 fossil_free(zTagname);
1350 fossil_print("%s\n", zTagCmd);
1351 fossil_system(zTagCmd);
1352 fossil_free(zTagCmd);
1353 }
1354 db_finalize(&q);
1355
1356 /* Update the start time */
1357 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
1358 db_bind_double(&q, ":x", rEnd);
1359 db_step(&q);
1360 db_finalize(&q);
1361 db_commit_transaction();
1362
1363 /* Optionally do a "git push" */
1364 }
1365
1366 /*
1367 ** COMMAND: git
1368 **
1369 ** Usage: %fossil git SUBCOMMAND
1370 **
1371 ** Do incremental import or export operations between Fossil and Git.
1372 ** Subcommands:
1373 **
1374 ** fossil git export MIRROR [OPTIONS]
1375 **
1376 ** Write content from the Fossil repository into the Git repository
1377 ** in directory MIRROR. The Git repository is created if it does not
1378 ** already exist. If the Git repository does already exist, then
1379 ** new content added to fossil since the previous export is appended.
1380 **
1381 ** Repeat this command whenever new checkins are added to the Fossil
1382 ** repository in order to reflect those changes into the mirror.
1383 **
1384 ** The MIRROR directory will contain a subdirectory named
1385 ** ".mirror_state" that contains information that Fossil needs to
1386 ** do incremental exports. Do not attempt to manage or edit the files
1387 ** in that directory since doing so can disrupt future incremental
1388 ** exports.
1389 **
1390 ** Options:
1391 ** --debug FILE Write fast-export text to FILE rather than
1392 ** piping it into "git fast-import".
1393 ** --limit N Add no more than N new check-ins to MIRROR.
1394 ** Useful for debugging
1395 **
1396 ** fossil git import MIRROR
1397 **
1398 ** TBD...
1399 */
1400 void gitmirror_command(void){
1401 char *zCmd;
1402 int nCmd;
1403 if( g.argc<3 ){
1404 usage("export ARGS...");
1405 }
1406 zCmd = g.argv[2];
1407 nCmd = (int)strlen(zCmd);
1408 if( nCmd>2 && strncmp(zCmd,"export",nCmd)==0 ){
1409 gitmirror_export_command();
1410 }else
1411 if( nCmd>2 && strncmp(zCmd,"import",nCmd)==0 ){
1412 fossil_fatal("not yet implemented - check back later");
1413 }else
1414 {
1415 fossil_fatal("unknown subcommand \"%s\": should be one of "
1416 "\"export\", \"import\"",
1417 zCmd);
1418 }
1419 }
1420

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button