Fossil SCM
Abandon an export to Git if a phantom artifact is encountered on any check-in that is less than one year old. This is a defense against generating an incorrect export from a repository that has an incomplete sync. Shunned artifacts are always ignored, regardless of age.
Commit
de0bbcb53c5463cc7da8cdc16d38d2940ffc1eec322adf96d14488d4fd42dd2f
Parent
aaa937a8e14ec04…
2 files changed
+90
-20
+10
+90
-20
| --- src/export.c | ||
| +++ src/export.c | ||
| @@ -842,10 +842,33 @@ | ||
| 842 | 842 | /*************************************************************************** |
| 843 | 843 | ** Implementation of the "fossil git" command follows. We hope that the |
| 844 | 844 | ** new code that follows will largely replace the legacy "fossil export" |
| 845 | 845 | ** and "fossil import" code above. |
| 846 | 846 | */ |
| 847 | + | |
| 848 | +/* Verbosity level. Higher means more output. | |
| 849 | +** | |
| 850 | +** 0 print nothing at all | |
| 851 | +** 1 Errors only | |
| 852 | +** 2 Progress information (This is the default) | |
| 853 | +** 3 Extra details | |
| 854 | +*/ | |
| 855 | +#define VERB_ERROR 1 | |
| 856 | +#define VERB_NORMAL 2 | |
| 857 | +#define VERB_EXTRA 3 | |
| 858 | +static int gitmirror_verbosity = VERB_NORMAL; | |
| 859 | + | |
| 860 | +/* | |
| 861 | +** Output routine that depends on verbosity | |
| 862 | +*/ | |
| 863 | +static void gitmirror_message(int iLevel, const char *zFormat, ...){ | |
| 864 | + va_list ap; | |
| 865 | + if( iLevel>gitmirror_verbosity ) return; | |
| 866 | + va_start(ap, zFormat); | |
| 867 | + fossil_vprint(zFormat, ap); | |
| 868 | + va_end(ap); | |
| 869 | +} | |
| 847 | 870 | |
| 848 | 871 | /* |
| 849 | 872 | ** Convert characters of z[] that are not allowed to be in branch or |
| 850 | 873 | ** tag names into "_". |
| 851 | 874 | */ |
| @@ -948,31 +971,49 @@ | ||
| 948 | 971 | static const char zEmptySha3[] = |
| 949 | 972 | "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"; |
| 950 | 973 | |
| 951 | 974 | /* |
| 952 | 975 | ** Export a single file named by zUuid. |
| 976 | +** | |
| 977 | +** Return 0 on success and non-zero on any failure. | |
| 978 | +** | |
| 979 | +** If zUuid is a shunned file, then treat it as if it were any empty file. | |
| 980 | +** But files that are missing from the repository but have not been officially | |
| 981 | +** shunned cause an error return. Except, if bPhantomOk is true, then missing | |
| 982 | +** files are replaced by an empty file. | |
| 953 | 983 | */ |
| 954 | -static void gitmirror_send_file(FILE *xCmd, const char *zUuid){ | |
| 984 | +static int gitmirror_send_file(FILE *xCmd, const char *zUuid, int bPhantomOk){ | |
| 955 | 985 | int iMark; |
| 956 | 986 | int rid; |
| 957 | 987 | int rc; |
| 958 | 988 | Blob data; |
| 959 | 989 | rid = fast_uuid_to_rid(zUuid); |
| 960 | 990 | if( rid<0 ){ |
| 961 | - zUuid = zEmptySha3; | |
| 991 | + if( bPhantomOk || uuid_is_shunned(zUuid) ){ | |
| 992 | + gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); | |
| 993 | + zUuid = zEmptySha3; | |
| 994 | + }else{ | |
| 995 | + return 1; | |
| 996 | + } | |
| 962 | 997 | }else{ |
| 963 | 998 | rc = content_get(rid, &data); |
| 964 | 999 | if( rc==0 ){ |
| 965 | - blob_init(&data, 0, 0); | |
| 966 | - zUuid = zEmptySha3; | |
| 1000 | + if( bPhantomOk ){ | |
| 1001 | + blob_init(&data, 0, 0); | |
| 1002 | + gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); | |
| 1003 | + zUuid = zEmptySha3; | |
| 1004 | + }else{ | |
| 1005 | + return 1; | |
| 1006 | + } | |
| 967 | 1007 | } |
| 968 | 1008 | } |
| 969 | 1009 | iMark = gitmirror_find_mark(zUuid, 1); |
| 970 | 1010 | fprintf(xCmd, "blob\nmark :%d\ndata %d\n", iMark, blob_size(&data)); |
| 971 | 1011 | fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd); |
| 972 | 1012 | fprintf(xCmd, "\n"); |
| 973 | 1013 | blob_reset(&data); |
| 1014 | + return 0; | |
| 974 | 1015 | } |
| 975 | 1016 | |
| 976 | 1017 | /* |
| 977 | 1018 | ** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for |
| 978 | 1019 | ** the check-in to export. |
| @@ -982,13 +1023,15 @@ | ||
| 982 | 1023 | ** This can only happen on a timewarp, so deep nesting is unlikely. |
| 983 | 1024 | ** |
| 984 | 1025 | ** Before sending the check-in, first make sure all associated files |
| 985 | 1026 | ** have already been exported, and send "blob" records for any that |
| 986 | 1027 | ** have not been. Update the MIRROR.MMARK table so that it holds the |
| 987 | -** marks for the exported files. | |
| 1028 | +** marks for the exported files. | |
| 1029 | +** | |
| 1030 | +** Return zero on success and non-zero if the export should be stopped. | |
| 988 | 1031 | */ |
| 989 | -static void gitmirror_send_checkin( | |
| 1032 | +static int gitmirror_send_checkin( | |
| 990 | 1033 | FILE *xCmd, /* Write fast-import text on this pipe */ |
| 991 | 1034 | int rid, /* BLOB.RID for the check-in to export */ |
| 992 | 1035 | const char *zUuid, /* BLOB.UUID for the check-in to export */ |
| 993 | 1036 | int *pnLimit, /* Stop when the counter reaches zero */ |
| 994 | 1037 | int fManifest /* MFESTFLG_* values */ |
| @@ -999,44 +1042,63 @@ | ||
| 999 | 1042 | Stmt q; /* An SQL query */ |
| 1000 | 1043 | char *zBranch; /* The branch of the check-in */ |
| 1001 | 1044 | int iMark; /* The mark for the check-in */ |
| 1002 | 1045 | Blob sql; /* String of SQL for part of the query */ |
| 1003 | 1046 | Blob comment; /* The comment text for the check-in */ |
| 1047 | + int nErr = 0; /* Number of errors */ | |
| 1048 | + int bPhantomOk; /* True if phantom files should be ignored */ | |
| 1004 | 1049 | |
| 1005 | 1050 | pMan = manifest_get(rid, CFTYPE_MANIFEST, 0); |
| 1006 | 1051 | if( pMan==0 ){ |
| 1007 | 1052 | /* Must be a phantom. Return without doing anything, and in particular |
| 1008 | 1053 | ** without creating a mark for this check-in. */ |
| 1009 | - return; | |
| 1054 | + gitmirror_message(VERB_NORMAL, "missing check-in: %s\n", zUuid); | |
| 1055 | + return 0; | |
| 1010 | 1056 | } |
| 1011 | 1057 | |
| 1012 | 1058 | /* Check to see if any parent logins have not yet been processed, and |
| 1013 | 1059 | ** if so, create them */ |
| 1014 | 1060 | for(i=0; i<pMan->nParent; i++){ |
| 1015 | 1061 | int iMark = gitmirror_find_mark(pMan->azParent[i], 0); |
| 1016 | 1062 | if( iMark<=0 ){ |
| 1017 | 1063 | int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", |
| 1018 | 1064 | pMan->azParent[i]); |
| 1019 | - gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], pnLimit, fManifest); | |
| 1020 | - if( *pnLimit<=0 ){ | |
| 1065 | + int rc = gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], | |
| 1066 | + pnLimit, fManifest); | |
| 1067 | + if( rc || *pnLimit<=0 ){ | |
| 1021 | 1068 | manifest_destroy(pMan); |
| 1022 | - return; | |
| 1069 | + return 1; | |
| 1023 | 1070 | } |
| 1024 | 1071 | } |
| 1025 | 1072 | } |
| 1073 | + | |
| 1074 | + /* Ignore phantom files on check-ins that are over one year old */ | |
| 1075 | + bPhantomOk = db_int(0, "SELECT %.6f<julianday('now','-1 year')", | |
| 1076 | + pMan->rDate); | |
| 1026 | 1077 | |
| 1027 | 1078 | /* Make sure all necessary files have been exported */ |
| 1028 | 1079 | db_prepare(&q, |
| 1029 | 1080 | "SELECT uuid FROM files_of_checkin(%Q)" |
| 1030 | 1081 | " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)", |
| 1031 | 1082 | zUuid |
| 1032 | 1083 | ); |
| 1033 | 1084 | while( db_step(&q)==SQLITE_ROW ){ |
| 1034 | 1085 | const char *zFUuid = db_column_text(&q, 0); |
| 1035 | - gitmirror_send_file(xCmd, zFUuid); | |
| 1086 | + int n = gitmirror_send_file(xCmd, zFUuid, bPhantomOk); | |
| 1087 | + nErr += n; | |
| 1088 | + if( n ) gitmirror_message(VERB_ERROR, "missing file: %s\n", zFUuid); | |
| 1036 | 1089 | } |
| 1037 | 1090 | db_finalize(&q); |
| 1091 | + | |
| 1092 | + /* If some required files could not be exported, abandon the check-in | |
| 1093 | + ** export */ | |
| 1094 | + if( nErr ){ | |
| 1095 | + gitmirror_message(VERB_ERROR, | |
| 1096 | + "export of %s abandoned due to missing files\n", zUuid); | |
| 1097 | + *pnLimit = 0; | |
| 1098 | + return 1; | |
| 1099 | + } | |
| 1038 | 1100 | |
| 1039 | 1101 | /* Figure out which branch this check-in is a member of */ |
| 1040 | 1102 | zBranch = db_text(0, |
| 1041 | 1103 | "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d", |
| 1042 | 1104 | TAG_BRANCH, rid |
| @@ -1141,10 +1203,11 @@ | ||
| 1141 | 1203 | blob_reset(&tagslist); |
| 1142 | 1204 | } |
| 1143 | 1205 | |
| 1144 | 1206 | /* The check-in is finished, so decrement the counter */ |
| 1145 | 1207 | (*pnLimit)--; |
| 1208 | + return 0; | |
| 1146 | 1209 | } |
| 1147 | 1210 | |
| 1148 | 1211 | /* |
| 1149 | 1212 | ** Implementation of the "fossil git export" command. |
| 1150 | 1213 | */ |
| @@ -1174,10 +1237,13 @@ | ||
| 1174 | 1237 | nLimit = (unsigned int)atoi(zLimit); |
| 1175 | 1238 | if( nLimit<=0 ) fossil_fatal("--limit must be positive"); |
| 1176 | 1239 | } |
| 1177 | 1240 | zAutoPush = find_option("autopush",0,1); |
| 1178 | 1241 | bForce = find_option("force","f",0)!=0; |
| 1242 | + gitmirror_verbosity = VERB_NORMAL; | |
| 1243 | + while( find_option("quiet","q",0)!=0 ){ gitmirror_verbosity--; } | |
| 1244 | + while( find_option("verbose","v",0)!=0 ){ gitmirror_verbosity++; } | |
| 1179 | 1245 | verify_all_options(); |
| 1180 | 1246 | if( g.argc!=4 && g.argc!=3 ){ usage("export ?MIRROR?"); } |
| 1181 | 1247 | if( g.argc==4 ){ |
| 1182 | 1248 | Blob mirror; |
| 1183 | 1249 | file_canonical_name(g.argv[3], &mirror, 0); |
| @@ -1195,14 +1261,14 @@ | ||
| 1195 | 1261 | |
| 1196 | 1262 | /* Make sure GIT has been initialized */ |
| 1197 | 1263 | z = mprintf("%s/.git", zMirror); |
| 1198 | 1264 | if( !file_isdir(z, ExtFILE) ){ |
| 1199 | 1265 | zCmd = mprintf("git init '%s'",zMirror); |
| 1200 | - fossil_print("%s\n", zCmd); | |
| 1266 | + gitmirror_message(VERB_NORMAL, "%s\n", zCmd); | |
| 1201 | 1267 | rc = fossil_system(zCmd); |
| 1202 | 1268 | if( rc ){ |
| 1203 | - fossil_fatal("command failed: \"%s\"", zCmd); | |
| 1269 | + fossil_fatal("cannot initialize the git repository using: \"%s\"", zCmd); | |
| 1204 | 1270 | } |
| 1205 | 1271 | fossil_free(zCmd); |
| 1206 | 1272 | } |
| 1207 | 1273 | fossil_free(z); |
| 1208 | 1274 | |
| @@ -1245,11 +1311,11 @@ | ||
| 1245 | 1311 | if( !bForce |
| 1246 | 1312 | && !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')" |
| 1247 | 1313 | " AND mtime>coalesce((SELECT value FROM mconfig" |
| 1248 | 1314 | " WHERE key='start'),0.0)") |
| 1249 | 1315 | ){ |
| 1250 | - fossil_print("no changes\n"); | |
| 1316 | + gitmirror_message(VERB_NORMAL, "no changes\n"); | |
| 1251 | 1317 | db_commit_transaction(); |
| 1252 | 1318 | return; |
| 1253 | 1319 | } |
| 1254 | 1320 | |
| 1255 | 1321 | /* Do we need to include manifest files in the clone? */ |
| @@ -1271,11 +1337,11 @@ | ||
| 1271 | 1337 | }else{ |
| 1272 | 1338 | zCmd = mprintf("git fast-import" |
| 1273 | 1339 | " --import-marks-if-exists=.mirror_state/in" |
| 1274 | 1340 | " --export-marks=.mirror_state/out" |
| 1275 | 1341 | " --quiet --done"); |
| 1276 | - fossil_print("%s\n", zCmd); | |
| 1342 | + gitmirror_message(VERB_NORMAL, "%s\n", zCmd); | |
| 1277 | 1343 | xCmd = popen(zCmd, "w"); |
| 1278 | 1344 | if( zCmd==0 ){ |
| 1279 | 1345 | fossil_fatal("cannot start the \"git fast-import\" command"); |
| 1280 | 1346 | } |
| 1281 | 1347 | fossil_free(zCmd); |
| @@ -1304,22 +1370,24 @@ | ||
| 1304 | 1370 | while( nLimit && db_step(&q)==SQLITE_ROW ){ |
| 1305 | 1371 | int rid = db_column_int(&q, 0); |
| 1306 | 1372 | double rMTime = db_column_double(&q, 1); |
| 1307 | 1373 | const char *zUuid = db_column_text(&q, 2); |
| 1308 | 1374 | if( rMTime>rEnd ) rEnd = rMTime; |
| 1309 | - gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); | |
| 1310 | - printf("\r%d/%d ", nTotal-nLimit, nTotal); | |
| 1375 | + rc = gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); | |
| 1376 | + if( rc ) break; | |
| 1377 | + gitmirror_message(VERB_NORMAL,"%d/%d \r", nTotal-nLimit, nTotal); | |
| 1311 | 1378 | fflush(stdout); |
| 1312 | 1379 | } |
| 1313 | 1380 | db_finalize(&q); |
| 1314 | 1381 | fprintf(xCmd, "done\n"); |
| 1315 | 1382 | if( zDebug ){ |
| 1316 | 1383 | if( xCmd!=stdout ) fclose(xCmd); |
| 1317 | 1384 | }else{ |
| 1318 | 1385 | pclose(xCmd); |
| 1319 | 1386 | } |
| 1320 | - fossil_print("%d check-ins added to the mirror\n", nTotal-nLimit); | |
| 1387 | + gitmirror_message(VERB_NORMAL, "%d check-ins added to the %s\n", | |
| 1388 | + nTotal-nLimit, zMirror); | |
| 1321 | 1389 | |
| 1322 | 1390 | /* Read the export-marks file. Transfer the new marks over into |
| 1323 | 1391 | ** the import-marks file. |
| 1324 | 1392 | */ |
| 1325 | 1393 | pOut = fopen(".mirror_state/out", "rb"); |
| @@ -1374,11 +1442,11 @@ | ||
| 1374 | 1442 | const char *zObj = db_column_text(&q,1); |
| 1375 | 1443 | char *zTagCmd; |
| 1376 | 1444 | gitmirror_sanitize_name(zTagname); |
| 1377 | 1445 | zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj); |
| 1378 | 1446 | fossil_free(zTagname); |
| 1379 | - fossil_print("%s\n", zTagCmd); | |
| 1447 | + gitmirror_message(VERB_NORMAL, "%s\n", zTagCmd); | |
| 1380 | 1448 | fossil_system(zTagCmd); |
| 1381 | 1449 | fossil_free(zTagCmd); |
| 1382 | 1450 | } |
| 1383 | 1451 | db_finalize(&q); |
| 1384 | 1452 | |
| @@ -1396,11 +1464,11 @@ | ||
| 1396 | 1464 | if( zPushUrl ){ |
| 1397 | 1465 | char *zPushCmd; |
| 1398 | 1466 | UrlData url; |
| 1399 | 1467 | url_parse_local(zPushUrl, 0, &url); |
| 1400 | 1468 | zPushCmd = mprintf("git push --mirror %s", url.canonical); |
| 1401 | - fossil_print("%s\n", zPushCmd); | |
| 1469 | + gitmirror_message(VERB_NORMAL, "%s\n", zPushCmd); | |
| 1402 | 1470 | fossil_free(zPushCmd); |
| 1403 | 1471 | zPushCmd = mprintf("git push --mirror %s", zPushUrl); |
| 1404 | 1472 | fossil_system(zPushCmd); |
| 1405 | 1473 | fossil_free(zPushCmd); |
| 1406 | 1474 | } |
| @@ -1440,10 +1508,12 @@ | ||
| 1440 | 1508 | ** --debug FILE Write fast-export text to FILE rather than |
| 1441 | 1509 | ** piping it into "git fast-import". |
| 1442 | 1510 | ** --force|-f Do the export even if nothing has changed |
| 1443 | 1511 | ** --limit N Add no more than N new check-ins to MIRROR. |
| 1444 | 1512 | ** Useful for debugging |
| 1513 | +** --quiet|-q Reduce output. Repeat for even less output. | |
| 1514 | +** --verbose|-v More output. | |
| 1445 | 1515 | ** |
| 1446 | 1516 | ** fossil git import MIRROR |
| 1447 | 1517 | ** |
| 1448 | 1518 | ** TBD... |
| 1449 | 1519 | */ |
| 1450 | 1520 |
| --- src/export.c | |
| +++ src/export.c | |
| @@ -842,10 +842,33 @@ | |
| 842 | /*************************************************************************** |
| 843 | ** Implementation of the "fossil git" command follows. We hope that the |
| 844 | ** new code that follows will largely replace the legacy "fossil export" |
| 845 | ** and "fossil import" code above. |
| 846 | */ |
| 847 | |
| 848 | /* |
| 849 | ** Convert characters of z[] that are not allowed to be in branch or |
| 850 | ** tag names into "_". |
| 851 | */ |
| @@ -948,31 +971,49 @@ | |
| 948 | static const char zEmptySha3[] = |
| 949 | "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"; |
| 950 | |
| 951 | /* |
| 952 | ** Export a single file named by zUuid. |
| 953 | */ |
| 954 | static void gitmirror_send_file(FILE *xCmd, const char *zUuid){ |
| 955 | int iMark; |
| 956 | int rid; |
| 957 | int rc; |
| 958 | Blob data; |
| 959 | rid = fast_uuid_to_rid(zUuid); |
| 960 | if( rid<0 ){ |
| 961 | zUuid = zEmptySha3; |
| 962 | }else{ |
| 963 | rc = content_get(rid, &data); |
| 964 | if( rc==0 ){ |
| 965 | blob_init(&data, 0, 0); |
| 966 | zUuid = zEmptySha3; |
| 967 | } |
| 968 | } |
| 969 | iMark = gitmirror_find_mark(zUuid, 1); |
| 970 | fprintf(xCmd, "blob\nmark :%d\ndata %d\n", iMark, blob_size(&data)); |
| 971 | fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd); |
| 972 | fprintf(xCmd, "\n"); |
| 973 | blob_reset(&data); |
| 974 | } |
| 975 | |
| 976 | /* |
| 977 | ** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for |
| 978 | ** the check-in to export. |
| @@ -982,13 +1023,15 @@ | |
| 982 | ** This can only happen on a timewarp, so deep nesting is unlikely. |
| 983 | ** |
| 984 | ** Before sending the check-in, first make sure all associated files |
| 985 | ** have already been exported, and send "blob" records for any that |
| 986 | ** have not been. Update the MIRROR.MMARK table so that it holds the |
| 987 | ** marks for the exported files. |
| 988 | */ |
| 989 | static void gitmirror_send_checkin( |
| 990 | FILE *xCmd, /* Write fast-import text on this pipe */ |
| 991 | int rid, /* BLOB.RID for the check-in to export */ |
| 992 | const char *zUuid, /* BLOB.UUID for the check-in to export */ |
| 993 | int *pnLimit, /* Stop when the counter reaches zero */ |
| 994 | int fManifest /* MFESTFLG_* values */ |
| @@ -999,44 +1042,63 @@ | |
| 999 | Stmt q; /* An SQL query */ |
| 1000 | char *zBranch; /* The branch of the check-in */ |
| 1001 | int iMark; /* The mark for the check-in */ |
| 1002 | Blob sql; /* String of SQL for part of the query */ |
| 1003 | Blob comment; /* The comment text for the check-in */ |
| 1004 | |
| 1005 | pMan = manifest_get(rid, CFTYPE_MANIFEST, 0); |
| 1006 | if( pMan==0 ){ |
| 1007 | /* Must be a phantom. Return without doing anything, and in particular |
| 1008 | ** without creating a mark for this check-in. */ |
| 1009 | return; |
| 1010 | } |
| 1011 | |
| 1012 | /* Check to see if any parent logins have not yet been processed, and |
| 1013 | ** if so, create them */ |
| 1014 | for(i=0; i<pMan->nParent; i++){ |
| 1015 | int iMark = gitmirror_find_mark(pMan->azParent[i], 0); |
| 1016 | if( iMark<=0 ){ |
| 1017 | int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", |
| 1018 | pMan->azParent[i]); |
| 1019 | gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], pnLimit, fManifest); |
| 1020 | if( *pnLimit<=0 ){ |
| 1021 | manifest_destroy(pMan); |
| 1022 | return; |
| 1023 | } |
| 1024 | } |
| 1025 | } |
| 1026 | |
| 1027 | /* Make sure all necessary files have been exported */ |
| 1028 | db_prepare(&q, |
| 1029 | "SELECT uuid FROM files_of_checkin(%Q)" |
| 1030 | " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)", |
| 1031 | zUuid |
| 1032 | ); |
| 1033 | while( db_step(&q)==SQLITE_ROW ){ |
| 1034 | const char *zFUuid = db_column_text(&q, 0); |
| 1035 | gitmirror_send_file(xCmd, zFUuid); |
| 1036 | } |
| 1037 | db_finalize(&q); |
| 1038 | |
| 1039 | /* Figure out which branch this check-in is a member of */ |
| 1040 | zBranch = db_text(0, |
| 1041 | "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d", |
| 1042 | TAG_BRANCH, rid |
| @@ -1141,10 +1203,11 @@ | |
| 1141 | blob_reset(&tagslist); |
| 1142 | } |
| 1143 | |
| 1144 | /* The check-in is finished, so decrement the counter */ |
| 1145 | (*pnLimit)--; |
| 1146 | } |
| 1147 | |
| 1148 | /* |
| 1149 | ** Implementation of the "fossil git export" command. |
| 1150 | */ |
| @@ -1174,10 +1237,13 @@ | |
| 1174 | nLimit = (unsigned int)atoi(zLimit); |
| 1175 | if( nLimit<=0 ) fossil_fatal("--limit must be positive"); |
| 1176 | } |
| 1177 | zAutoPush = find_option("autopush",0,1); |
| 1178 | bForce = find_option("force","f",0)!=0; |
| 1179 | verify_all_options(); |
| 1180 | if( g.argc!=4 && g.argc!=3 ){ usage("export ?MIRROR?"); } |
| 1181 | if( g.argc==4 ){ |
| 1182 | Blob mirror; |
| 1183 | file_canonical_name(g.argv[3], &mirror, 0); |
| @@ -1195,14 +1261,14 @@ | |
| 1195 | |
| 1196 | /* Make sure GIT has been initialized */ |
| 1197 | z = mprintf("%s/.git", zMirror); |
| 1198 | if( !file_isdir(z, ExtFILE) ){ |
| 1199 | zCmd = mprintf("git init '%s'",zMirror); |
| 1200 | fossil_print("%s\n", zCmd); |
| 1201 | rc = fossil_system(zCmd); |
| 1202 | if( rc ){ |
| 1203 | fossil_fatal("command failed: \"%s\"", zCmd); |
| 1204 | } |
| 1205 | fossil_free(zCmd); |
| 1206 | } |
| 1207 | fossil_free(z); |
| 1208 | |
| @@ -1245,11 +1311,11 @@ | |
| 1245 | if( !bForce |
| 1246 | && !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')" |
| 1247 | " AND mtime>coalesce((SELECT value FROM mconfig" |
| 1248 | " WHERE key='start'),0.0)") |
| 1249 | ){ |
| 1250 | fossil_print("no changes\n"); |
| 1251 | db_commit_transaction(); |
| 1252 | return; |
| 1253 | } |
| 1254 | |
| 1255 | /* Do we need to include manifest files in the clone? */ |
| @@ -1271,11 +1337,11 @@ | |
| 1271 | }else{ |
| 1272 | zCmd = mprintf("git fast-import" |
| 1273 | " --import-marks-if-exists=.mirror_state/in" |
| 1274 | " --export-marks=.mirror_state/out" |
| 1275 | " --quiet --done"); |
| 1276 | fossil_print("%s\n", zCmd); |
| 1277 | xCmd = popen(zCmd, "w"); |
| 1278 | if( zCmd==0 ){ |
| 1279 | fossil_fatal("cannot start the \"git fast-import\" command"); |
| 1280 | } |
| 1281 | fossil_free(zCmd); |
| @@ -1304,22 +1370,24 @@ | |
| 1304 | while( nLimit && db_step(&q)==SQLITE_ROW ){ |
| 1305 | int rid = db_column_int(&q, 0); |
| 1306 | double rMTime = db_column_double(&q, 1); |
| 1307 | const char *zUuid = db_column_text(&q, 2); |
| 1308 | if( rMTime>rEnd ) rEnd = rMTime; |
| 1309 | gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); |
| 1310 | printf("\r%d/%d ", nTotal-nLimit, nTotal); |
| 1311 | fflush(stdout); |
| 1312 | } |
| 1313 | db_finalize(&q); |
| 1314 | fprintf(xCmd, "done\n"); |
| 1315 | if( zDebug ){ |
| 1316 | if( xCmd!=stdout ) fclose(xCmd); |
| 1317 | }else{ |
| 1318 | pclose(xCmd); |
| 1319 | } |
| 1320 | fossil_print("%d check-ins added to the mirror\n", nTotal-nLimit); |
| 1321 | |
| 1322 | /* Read the export-marks file. Transfer the new marks over into |
| 1323 | ** the import-marks file. |
| 1324 | */ |
| 1325 | pOut = fopen(".mirror_state/out", "rb"); |
| @@ -1374,11 +1442,11 @@ | |
| 1374 | const char *zObj = db_column_text(&q,1); |
| 1375 | char *zTagCmd; |
| 1376 | gitmirror_sanitize_name(zTagname); |
| 1377 | zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj); |
| 1378 | fossil_free(zTagname); |
| 1379 | fossil_print("%s\n", zTagCmd); |
| 1380 | fossil_system(zTagCmd); |
| 1381 | fossil_free(zTagCmd); |
| 1382 | } |
| 1383 | db_finalize(&q); |
| 1384 | |
| @@ -1396,11 +1464,11 @@ | |
| 1396 | if( zPushUrl ){ |
| 1397 | char *zPushCmd; |
| 1398 | UrlData url; |
| 1399 | url_parse_local(zPushUrl, 0, &url); |
| 1400 | zPushCmd = mprintf("git push --mirror %s", url.canonical); |
| 1401 | fossil_print("%s\n", zPushCmd); |
| 1402 | fossil_free(zPushCmd); |
| 1403 | zPushCmd = mprintf("git push --mirror %s", zPushUrl); |
| 1404 | fossil_system(zPushCmd); |
| 1405 | fossil_free(zPushCmd); |
| 1406 | } |
| @@ -1440,10 +1508,12 @@ | |
| 1440 | ** --debug FILE Write fast-export text to FILE rather than |
| 1441 | ** piping it into "git fast-import". |
| 1442 | ** --force|-f Do the export even if nothing has changed |
| 1443 | ** --limit N Add no more than N new check-ins to MIRROR. |
| 1444 | ** Useful for debugging |
| 1445 | ** |
| 1446 | ** fossil git import MIRROR |
| 1447 | ** |
| 1448 | ** TBD... |
| 1449 | */ |
| 1450 |
| --- src/export.c | |
| +++ src/export.c | |
| @@ -842,10 +842,33 @@ | |
| 842 | /*************************************************************************** |
| 843 | ** Implementation of the "fossil git" command follows. We hope that the |
| 844 | ** new code that follows will largely replace the legacy "fossil export" |
| 845 | ** and "fossil import" code above. |
| 846 | */ |
| 847 | |
| 848 | /* Verbosity level. Higher means more output. |
| 849 | ** |
| 850 | ** 0 print nothing at all |
| 851 | ** 1 Errors only |
| 852 | ** 2 Progress information (This is the default) |
| 853 | ** 3 Extra details |
| 854 | */ |
| 855 | #define VERB_ERROR 1 |
| 856 | #define VERB_NORMAL 2 |
| 857 | #define VERB_EXTRA 3 |
| 858 | static int gitmirror_verbosity = VERB_NORMAL; |
| 859 | |
| 860 | /* |
| 861 | ** Output routine that depends on verbosity |
| 862 | */ |
| 863 | static void gitmirror_message(int iLevel, const char *zFormat, ...){ |
| 864 | va_list ap; |
| 865 | if( iLevel>gitmirror_verbosity ) return; |
| 866 | va_start(ap, zFormat); |
| 867 | fossil_vprint(zFormat, ap); |
| 868 | va_end(ap); |
| 869 | } |
| 870 | |
| 871 | /* |
| 872 | ** Convert characters of z[] that are not allowed to be in branch or |
| 873 | ** tag names into "_". |
| 874 | */ |
| @@ -948,31 +971,49 @@ | |
| 971 | static const char zEmptySha3[] = |
| 972 | "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"; |
| 973 | |
| 974 | /* |
| 975 | ** Export a single file named by zUuid. |
| 976 | ** |
| 977 | ** Return 0 on success and non-zero on any failure. |
| 978 | ** |
| 979 | ** If zUuid is a shunned file, then treat it as if it were any empty file. |
| 980 | ** But files that are missing from the repository but have not been officially |
| 981 | ** shunned cause an error return. Except, if bPhantomOk is true, then missing |
| 982 | ** files are replaced by an empty file. |
| 983 | */ |
| 984 | static int gitmirror_send_file(FILE *xCmd, const char *zUuid, int bPhantomOk){ |
| 985 | int iMark; |
| 986 | int rid; |
| 987 | int rc; |
| 988 | Blob data; |
| 989 | rid = fast_uuid_to_rid(zUuid); |
| 990 | if( rid<0 ){ |
| 991 | if( bPhantomOk || uuid_is_shunned(zUuid) ){ |
| 992 | gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); |
| 993 | zUuid = zEmptySha3; |
| 994 | }else{ |
| 995 | return 1; |
| 996 | } |
| 997 | }else{ |
| 998 | rc = content_get(rid, &data); |
| 999 | if( rc==0 ){ |
| 1000 | if( bPhantomOk ){ |
| 1001 | blob_init(&data, 0, 0); |
| 1002 | gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); |
| 1003 | zUuid = zEmptySha3; |
| 1004 | }else{ |
| 1005 | return 1; |
| 1006 | } |
| 1007 | } |
| 1008 | } |
| 1009 | iMark = gitmirror_find_mark(zUuid, 1); |
| 1010 | fprintf(xCmd, "blob\nmark :%d\ndata %d\n", iMark, blob_size(&data)); |
| 1011 | fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd); |
| 1012 | fprintf(xCmd, "\n"); |
| 1013 | blob_reset(&data); |
| 1014 | return 0; |
| 1015 | } |
| 1016 | |
| 1017 | /* |
| 1018 | ** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for |
| 1019 | ** the check-in to export. |
| @@ -982,13 +1023,15 @@ | |
| 1023 | ** This can only happen on a timewarp, so deep nesting is unlikely. |
| 1024 | ** |
| 1025 | ** Before sending the check-in, first make sure all associated files |
| 1026 | ** have already been exported, and send "blob" records for any that |
| 1027 | ** have not been. Update the MIRROR.MMARK table so that it holds the |
| 1028 | ** marks for the exported files. |
| 1029 | ** |
| 1030 | ** Return zero on success and non-zero if the export should be stopped. |
| 1031 | */ |
| 1032 | static int gitmirror_send_checkin( |
| 1033 | FILE *xCmd, /* Write fast-import text on this pipe */ |
| 1034 | int rid, /* BLOB.RID for the check-in to export */ |
| 1035 | const char *zUuid, /* BLOB.UUID for the check-in to export */ |
| 1036 | int *pnLimit, /* Stop when the counter reaches zero */ |
| 1037 | int fManifest /* MFESTFLG_* values */ |
| @@ -999,44 +1042,63 @@ | |
| 1042 | Stmt q; /* An SQL query */ |
| 1043 | char *zBranch; /* The branch of the check-in */ |
| 1044 | int iMark; /* The mark for the check-in */ |
| 1045 | Blob sql; /* String of SQL for part of the query */ |
| 1046 | Blob comment; /* The comment text for the check-in */ |
| 1047 | int nErr = 0; /* Number of errors */ |
| 1048 | int bPhantomOk; /* True if phantom files should be ignored */ |
| 1049 | |
| 1050 | pMan = manifest_get(rid, CFTYPE_MANIFEST, 0); |
| 1051 | if( pMan==0 ){ |
| 1052 | /* Must be a phantom. Return without doing anything, and in particular |
| 1053 | ** without creating a mark for this check-in. */ |
| 1054 | gitmirror_message(VERB_NORMAL, "missing check-in: %s\n", zUuid); |
| 1055 | return 0; |
| 1056 | } |
| 1057 | |
| 1058 | /* Check to see if any parent logins have not yet been processed, and |
| 1059 | ** if so, create them */ |
| 1060 | for(i=0; i<pMan->nParent; i++){ |
| 1061 | int iMark = gitmirror_find_mark(pMan->azParent[i], 0); |
| 1062 | if( iMark<=0 ){ |
| 1063 | int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", |
| 1064 | pMan->azParent[i]); |
| 1065 | int rc = gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], |
| 1066 | pnLimit, fManifest); |
| 1067 | if( rc || *pnLimit<=0 ){ |
| 1068 | manifest_destroy(pMan); |
| 1069 | return 1; |
| 1070 | } |
| 1071 | } |
| 1072 | } |
| 1073 | |
| 1074 | /* Ignore phantom files on check-ins that are over one year old */ |
| 1075 | bPhantomOk = db_int(0, "SELECT %.6f<julianday('now','-1 year')", |
| 1076 | pMan->rDate); |
| 1077 | |
| 1078 | /* Make sure all necessary files have been exported */ |
| 1079 | db_prepare(&q, |
| 1080 | "SELECT uuid FROM files_of_checkin(%Q)" |
| 1081 | " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)", |
| 1082 | zUuid |
| 1083 | ); |
| 1084 | while( db_step(&q)==SQLITE_ROW ){ |
| 1085 | const char *zFUuid = db_column_text(&q, 0); |
| 1086 | int n = gitmirror_send_file(xCmd, zFUuid, bPhantomOk); |
| 1087 | nErr += n; |
| 1088 | if( n ) gitmirror_message(VERB_ERROR, "missing file: %s\n", zFUuid); |
| 1089 | } |
| 1090 | db_finalize(&q); |
| 1091 | |
| 1092 | /* If some required files could not be exported, abandon the check-in |
| 1093 | ** export */ |
| 1094 | if( nErr ){ |
| 1095 | gitmirror_message(VERB_ERROR, |
| 1096 | "export of %s abandoned due to missing files\n", zUuid); |
| 1097 | *pnLimit = 0; |
| 1098 | return 1; |
| 1099 | } |
| 1100 | |
| 1101 | /* Figure out which branch this check-in is a member of */ |
| 1102 | zBranch = db_text(0, |
| 1103 | "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d", |
| 1104 | TAG_BRANCH, rid |
| @@ -1141,10 +1203,11 @@ | |
| 1203 | blob_reset(&tagslist); |
| 1204 | } |
| 1205 | |
| 1206 | /* The check-in is finished, so decrement the counter */ |
| 1207 | (*pnLimit)--; |
| 1208 | return 0; |
| 1209 | } |
| 1210 | |
| 1211 | /* |
| 1212 | ** Implementation of the "fossil git export" command. |
| 1213 | */ |
| @@ -1174,10 +1237,13 @@ | |
| 1237 | nLimit = (unsigned int)atoi(zLimit); |
| 1238 | if( nLimit<=0 ) fossil_fatal("--limit must be positive"); |
| 1239 | } |
| 1240 | zAutoPush = find_option("autopush",0,1); |
| 1241 | bForce = find_option("force","f",0)!=0; |
| 1242 | gitmirror_verbosity = VERB_NORMAL; |
| 1243 | while( find_option("quiet","q",0)!=0 ){ gitmirror_verbosity--; } |
| 1244 | while( find_option("verbose","v",0)!=0 ){ gitmirror_verbosity++; } |
| 1245 | verify_all_options(); |
| 1246 | if( g.argc!=4 && g.argc!=3 ){ usage("export ?MIRROR?"); } |
| 1247 | if( g.argc==4 ){ |
| 1248 | Blob mirror; |
| 1249 | file_canonical_name(g.argv[3], &mirror, 0); |
| @@ -1195,14 +1261,14 @@ | |
| 1261 | |
| 1262 | /* Make sure GIT has been initialized */ |
| 1263 | z = mprintf("%s/.git", zMirror); |
| 1264 | if( !file_isdir(z, ExtFILE) ){ |
| 1265 | zCmd = mprintf("git init '%s'",zMirror); |
| 1266 | gitmirror_message(VERB_NORMAL, "%s\n", zCmd); |
| 1267 | rc = fossil_system(zCmd); |
| 1268 | if( rc ){ |
| 1269 | fossil_fatal("cannot initialize the git repository using: \"%s\"", zCmd); |
| 1270 | } |
| 1271 | fossil_free(zCmd); |
| 1272 | } |
| 1273 | fossil_free(z); |
| 1274 | |
| @@ -1245,11 +1311,11 @@ | |
| 1311 | if( !bForce |
| 1312 | && !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')" |
| 1313 | " AND mtime>coalesce((SELECT value FROM mconfig" |
| 1314 | " WHERE key='start'),0.0)") |
| 1315 | ){ |
| 1316 | gitmirror_message(VERB_NORMAL, "no changes\n"); |
| 1317 | db_commit_transaction(); |
| 1318 | return; |
| 1319 | } |
| 1320 | |
| 1321 | /* Do we need to include manifest files in the clone? */ |
| @@ -1271,11 +1337,11 @@ | |
| 1337 | }else{ |
| 1338 | zCmd = mprintf("git fast-import" |
| 1339 | " --import-marks-if-exists=.mirror_state/in" |
| 1340 | " --export-marks=.mirror_state/out" |
| 1341 | " --quiet --done"); |
| 1342 | gitmirror_message(VERB_NORMAL, "%s\n", zCmd); |
| 1343 | xCmd = popen(zCmd, "w"); |
| 1344 | if( zCmd==0 ){ |
| 1345 | fossil_fatal("cannot start the \"git fast-import\" command"); |
| 1346 | } |
| 1347 | fossil_free(zCmd); |
| @@ -1304,22 +1370,24 @@ | |
| 1370 | while( nLimit && db_step(&q)==SQLITE_ROW ){ |
| 1371 | int rid = db_column_int(&q, 0); |
| 1372 | double rMTime = db_column_double(&q, 1); |
| 1373 | const char *zUuid = db_column_text(&q, 2); |
| 1374 | if( rMTime>rEnd ) rEnd = rMTime; |
| 1375 | rc = gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); |
| 1376 | if( rc ) break; |
| 1377 | gitmirror_message(VERB_NORMAL,"%d/%d \r", nTotal-nLimit, nTotal); |
| 1378 | fflush(stdout); |
| 1379 | } |
| 1380 | db_finalize(&q); |
| 1381 | fprintf(xCmd, "done\n"); |
| 1382 | if( zDebug ){ |
| 1383 | if( xCmd!=stdout ) fclose(xCmd); |
| 1384 | }else{ |
| 1385 | pclose(xCmd); |
| 1386 | } |
| 1387 | gitmirror_message(VERB_NORMAL, "%d check-ins added to the %s\n", |
| 1388 | nTotal-nLimit, zMirror); |
| 1389 | |
| 1390 | /* Read the export-marks file. Transfer the new marks over into |
| 1391 | ** the import-marks file. |
| 1392 | */ |
| 1393 | pOut = fopen(".mirror_state/out", "rb"); |
| @@ -1374,11 +1442,11 @@ | |
| 1442 | const char *zObj = db_column_text(&q,1); |
| 1443 | char *zTagCmd; |
| 1444 | gitmirror_sanitize_name(zTagname); |
| 1445 | zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj); |
| 1446 | fossil_free(zTagname); |
| 1447 | gitmirror_message(VERB_NORMAL, "%s\n", zTagCmd); |
| 1448 | fossil_system(zTagCmd); |
| 1449 | fossil_free(zTagCmd); |
| 1450 | } |
| 1451 | db_finalize(&q); |
| 1452 | |
| @@ -1396,11 +1464,11 @@ | |
| 1464 | if( zPushUrl ){ |
| 1465 | char *zPushCmd; |
| 1466 | UrlData url; |
| 1467 | url_parse_local(zPushUrl, 0, &url); |
| 1468 | zPushCmd = mprintf("git push --mirror %s", url.canonical); |
| 1469 | gitmirror_message(VERB_NORMAL, "%s\n", zPushCmd); |
| 1470 | fossil_free(zPushCmd); |
| 1471 | zPushCmd = mprintf("git push --mirror %s", zPushUrl); |
| 1472 | fossil_system(zPushCmd); |
| 1473 | fossil_free(zPushCmd); |
| 1474 | } |
| @@ -1440,10 +1508,12 @@ | |
| 1508 | ** --debug FILE Write fast-export text to FILE rather than |
| 1509 | ** piping it into "git fast-import". |
| 1510 | ** --force|-f Do the export even if nothing has changed |
| 1511 | ** --limit N Add no more than N new check-ins to MIRROR. |
| 1512 | ** Useful for debugging |
| 1513 | ** --quiet|-q Reduce output. Repeat for even less output. |
| 1514 | ** --verbose|-v More output. |
| 1515 | ** |
| 1516 | ** fossil git import MIRROR |
| 1517 | ** |
| 1518 | ** TBD... |
| 1519 | */ |
| 1520 |
+10
| --- src/printf.c | ||
| +++ src/printf.c | ||
| @@ -964,10 +964,20 @@ | ||
| 964 | 964 | vxprintf(&b, zFormat, ap); |
| 965 | 965 | fossil_puts(blob_str(&b), 0); |
| 966 | 966 | blob_reset(&b); |
| 967 | 967 | } |
| 968 | 968 | va_end(ap); |
| 969 | +} | |
| 970 | +void fossil_vprint(const char *zFormat, va_list ap){ | |
| 971 | + if( g.cgiOutput ){ | |
| 972 | + cgi_vprintf(zFormat, ap); | |
| 973 | + }else{ | |
| 974 | + Blob b = empty_blob; | |
| 975 | + vxprintf(&b, zFormat, ap); | |
| 976 | + fossil_puts(blob_str(&b), 0); | |
| 977 | + blob_reset(&b); | |
| 978 | + } | |
| 969 | 979 | } |
| 970 | 980 | |
| 971 | 981 | /* |
| 972 | 982 | ** Print a trace message on standard error. |
| 973 | 983 | */ |
| 974 | 984 |
| --- src/printf.c | |
| +++ src/printf.c | |
| @@ -964,10 +964,20 @@ | |
| 964 | vxprintf(&b, zFormat, ap); |
| 965 | fossil_puts(blob_str(&b), 0); |
| 966 | blob_reset(&b); |
| 967 | } |
| 968 | va_end(ap); |
| 969 | } |
| 970 | |
| 971 | /* |
| 972 | ** Print a trace message on standard error. |
| 973 | */ |
| 974 |
| --- src/printf.c | |
| +++ src/printf.c | |
| @@ -964,10 +964,20 @@ | |
| 964 | vxprintf(&b, zFormat, ap); |
| 965 | fossil_puts(blob_str(&b), 0); |
| 966 | blob_reset(&b); |
| 967 | } |
| 968 | va_end(ap); |
| 969 | } |
| 970 | void fossil_vprint(const char *zFormat, va_list ap){ |
| 971 | if( g.cgiOutput ){ |
| 972 | cgi_vprintf(zFormat, ap); |
| 973 | }else{ |
| 974 | Blob b = empty_blob; |
| 975 | vxprintf(&b, zFormat, ap); |
| 976 | fossil_puts(blob_str(&b), 0); |
| 977 | blob_reset(&b); |
| 978 | } |
| 979 | } |
| 980 | |
| 981 | /* |
| 982 | ** Print a trace message on standard error. |
| 983 | */ |
| 984 |