Fossil SCM

Do the export within a transaction for performance. Record a complete two-way map of Git and Fossil names in the mirror.mmark table.

drh 2019-03-15 21:36 mirror-cmd
Commit c4f9b177f4dfa1891ac1aaa817abae821928e3c1e90a997955ca579aac9576d4
Parent 051cd3825ed507b…
1 file changed +29 -33
~ src/export.c
M src/export.c
+29 -33
--- src/export.c
+++ src/export.c
@@ -912,22 +912,10 @@
912912
 zOut[j++] = '"';
913913
 zOut[j] = 0;
914914
 return zOut;
915915
 }
916916
917
-/*
918
-** Transfer a tag over to the mirror. "rid" is the BLOB.RID value for
919
-** the record that describes the tag.
920
-**
921
-** The Git tag mechanism is very limited compared to Fossil. Many Fossil
922
-** tags cannot be exported to Git. If this tag cannot be exported, then
923
-** silently ignore it.
924
-*/
925
-static void gitmirror_send_tag(FILE *xCmd, int rid){
926
- return;
927
-}
928
-
929917
 /*
930918
 ** Locate the mark for a UUID.
931919
 **
932920
 ** If the mark does not exist and if the bCreate flag is false, then
933921
 ** return 0. If the mark does not exist and the bCreate flag is true,
@@ -1210,28 +1198,26 @@
12101198
 if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
12111199
 fossil_free(z);
12121200
12131201
 /* Attach the .mirror_state/db database */
12141202
 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
1203
+ db_begin_write();
12151204
 db_multi_exec(
12161205
 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
12171206
 " key TEXT PRIMARY KEY,\n"
12181207
 " Value ANY\n"
12191208
 ") WITHOUT ROWID;\n"
12201209
 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
12211210
 " id INTEGER PRIMARY KEY,\n"
1222
- " uuid TEXT UNIQUE\n"
1211
+ " uuid TEXT UNIQUE,\n"
1212
+ " githash TEXT UNIQUE\n"
12231213
 ");"
1224
- "CREATE TABLE IF NOT EXISTS mirror.mtag(\n"
1225
- " tagname TEXT PRIMARY KEY,\n"
1226
- " cnt INTEGER DEFAULT 1\n"
1227
- ") WITHOUT ROWID;"
12281214
 );
12291215
12301216
 /* See if there is any work to be done. Exit early if not, before starting
12311217
 ** the "git fast-import" command. */
1232
- if( !db_exists("SELECT 1 FROM event WHERE type IN ('t','ci')"
1218
+ if( !db_exists("SELECT 1 FROM event WHERE type='ci'"
12331219
 " AND mtime>coalesce((SELECT value FROM mconfig"
12341220
 " WHERE key='start'),0.0)")
12351221
 ){
12361222
 fossil_print("no changes\n");
12371223
 return;
@@ -1267,40 +1253,35 @@
12671253
 }
12681254
12691255
 /* Run the export */
12701256
 rEnd = 0.0;
12711257
 db_multi_exec(
1272
- "CREATE TEMP TABLE tomirror(objid INTEGER PRIMARY KEY,type,mtime,uuid);\n"
1258
+ "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n"
12731259
 "INSERT INTO tomirror "
1274
- "SELECT objid, type, mtime, blob.uuid FROM event, blob\n"
1275
- " WHERE type IN ('ci','t')"
1260
+ "SELECT objid, mtime, blob.uuid FROM event, blob\n"
1261
+ " WHERE type='ci'"
12761262
 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
12771263
 " AND blob.rid=event.objid"
12781264
 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
12791265
 );
1280
- nTotal = db_int(0, "SELECT count(*) FROM tomirror WHERE type='ci'");
1266
+ nTotal = db_int(0, "SELECT count(*) FROM tomirror");
12811267
 if( nLimit<nTotal ){
12821268
 nTotal = nLimit;
12831269
 }else if( nLimit>nTotal ){
12841270
 nLimit = nTotal;
12851271
 }
12861272
 db_prepare(&q,
1287
- "SELECT objid, type, mtime, uuid FROM tomirror ORDER BY mtime"
1273
+ "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime"
12881274
 );
12891275
 while( nLimit && db_step(&q)==SQLITE_ROW ){
1290
- double rMTime = db_column_double(&q, 2);
1291
- const char *zType = db_column_text(&q, 1);
12921276
 int rid = db_column_int(&q, 0);
1293
- const char *zUuid = db_column_text(&q, 3);
1277
+ double rMTime = db_column_double(&q, 1);
1278
+ const char *zUuid = db_column_text(&q, 2);
12941279
 if( rMTime>rEnd ) rEnd = rMTime;
1295
- if( zType[0]=='t' ){
1296
- gitmirror_send_tag(xCmd, rid);
1297
- }else{
1298
- gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1299
- printf("\r%d/%d ", nTotal-nLimit, nTotal);
1300
- fflush(stdout);
1301
- }
1280
+ gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1281
+ printf("\r%d/%d ", nTotal-nLimit, nTotal);
1282
+ fflush(stdout);
13021283
 }
13031284
 db_finalize(&q);
13041285
 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
13051286
 db_bind_double(&q, ":x", rEnd);
13061287
 db_step(&q);
@@ -1320,19 +1301,34 @@
13201301
 if( pOut ){
13211302
 pIn = fopen(".mirror_state/in", "ab");
13221303
 if( pIn==0 ){
13231304
 fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
13241305
 }
1306
+ db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id");
13251307
 while( fgets(zLine, sizeof(zLine), pOut) ){
1308
+ int j, k;
1309
+ if( zLine[0]!=':' ) continue;
1310
+ db_bind_int(&q, ":id", atoi(zLine+1));
1311
+ for(j=1; zLine[j] && zLine[j]!=' '; j++){}
1312
+ if( zLine[j]!=' ' ) continue;
1313
+ j++;
1314
+ if( zLine[j]==0 ) continue;
1315
+ for(k=j; fossil_isalnum(zLine[k]); k++){}
1316
+ zLine[k] = 0;
1317
+ db_bind_text(&q, ":githash", &zLine[j]);
1318
+ db_step(&q);
1319
+ db_reset(&q);
13261320
 fputs(zLine, pIn);
13271321
 }
1322
+ db_finalize(&q);
13281323
 fclose(pOut);
13291324
 fclose(pIn);
13301325
 file_delete(".mirror_state/out");
13311326
 }else{
13321327
 fossil_fatal("git fast-import didn't generate a marks file!");
13331328
 }
1329
+ db_commit_transaction();
13341330
13351331
 /* Optionally do a "git push" */
13361332
 }
13371333
13381334
 /*
13391335
--- src/export.c
+++ src/export.c
@@ -912,22 +912,10 @@
912 zOut[j++] = '"';
913 zOut[j] = 0;
914 return zOut;
915 }
916
917 /*
918 ** Transfer a tag over to the mirror. "rid" is the BLOB.RID value for
919 ** the record that describes the tag.
920 **
921 ** The Git tag mechanism is very limited compared to Fossil. Many Fossil
922 ** tags cannot be exported to Git. If this tag cannot be exported, then
923 ** silently ignore it.
924 */
925 static void gitmirror_send_tag(FILE *xCmd, int rid){
926 return;
927 }
928
929 /*
930 ** Locate the mark for a UUID.
931 **
932 ** If the mark does not exist and if the bCreate flag is false, then
933 ** return 0. If the mark does not exist and the bCreate flag is true,
@@ -1210,28 +1198,26 @@
1210 if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
1211 fossil_free(z);
1212
1213 /* Attach the .mirror_state/db database */
1214 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
 
1215 db_multi_exec(
1216 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
1217 " key TEXT PRIMARY KEY,\n"
1218 " Value ANY\n"
1219 ") WITHOUT ROWID;\n"
1220 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
1221 " id INTEGER PRIMARY KEY,\n"
1222 " uuid TEXT UNIQUE\n"
 
1223 ");"
1224 "CREATE TABLE IF NOT EXISTS mirror.mtag(\n"
1225 " tagname TEXT PRIMARY KEY,\n"
1226 " cnt INTEGER DEFAULT 1\n"
1227 ") WITHOUT ROWID;"
1228 );
1229
1230 /* See if there is any work to be done. Exit early if not, before starting
1231 ** the "git fast-import" command. */
1232 if( !db_exists("SELECT 1 FROM event WHERE type IN ('t','ci')"
1233 " AND mtime>coalesce((SELECT value FROM mconfig"
1234 " WHERE key='start'),0.0)")
1235 ){
1236 fossil_print("no changes\n");
1237 return;
@@ -1267,40 +1253,35 @@
1267 }
1268
1269 /* Run the export */
1270 rEnd = 0.0;
1271 db_multi_exec(
1272 "CREATE TEMP TABLE tomirror(objid INTEGER PRIMARY KEY,type,mtime,uuid);\n"
1273 "INSERT INTO tomirror "
1274 "SELECT objid, type, mtime, blob.uuid FROM event, blob\n"
1275 " WHERE type IN ('ci','t')"
1276 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
1277 " AND blob.rid=event.objid"
1278 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
1279 );
1280 nTotal = db_int(0, "SELECT count(*) FROM tomirror WHERE type='ci'");
1281 if( nLimit<nTotal ){
1282 nTotal = nLimit;
1283 }else if( nLimit>nTotal ){
1284 nLimit = nTotal;
1285 }
1286 db_prepare(&q,
1287 "SELECT objid, type, mtime, uuid FROM tomirror ORDER BY mtime"
1288 );
1289 while( nLimit && db_step(&q)==SQLITE_ROW ){
1290 double rMTime = db_column_double(&q, 2);
1291 const char *zType = db_column_text(&q, 1);
1292 int rid = db_column_int(&q, 0);
1293 const char *zUuid = db_column_text(&q, 3);
 
1294 if( rMTime>rEnd ) rEnd = rMTime;
1295 if( zType[0]=='t' ){
1296 gitmirror_send_tag(xCmd, rid);
1297 }else{
1298 gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1299 printf("\r%d/%d ", nTotal-nLimit, nTotal);
1300 fflush(stdout);
1301 }
1302 }
1303 db_finalize(&q);
1304 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
1305 db_bind_double(&q, ":x", rEnd);
1306 db_step(&q);
@@ -1320,19 +1301,34 @@
1320 if( pOut ){
1321 pIn = fopen(".mirror_state/in", "ab");
1322 if( pIn==0 ){
1323 fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
1324 }
 
1325 while( fgets(zLine, sizeof(zLine), pOut) ){
 
 
 
 
 
 
 
 
 
 
 
 
1326 fputs(zLine, pIn);
1327 }
 
1328 fclose(pOut);
1329 fclose(pIn);
1330 file_delete(".mirror_state/out");
1331 }else{
1332 fossil_fatal("git fast-import didn't generate a marks file!");
1333 }
 
1334
1335 /* Optionally do a "git push" */
1336 }
1337
1338 /*
1339
--- src/export.c
+++ src/export.c
@@ -912,22 +912,10 @@
912 zOut[j++] = '"';
913 zOut[j] = 0;
914 return zOut;
915 }
916
 
 
 
 
 
 
 
 
 
 
 
 
917 /*
918 ** Locate the mark for a UUID.
919 **
920 ** If the mark does not exist and if the bCreate flag is false, then
921 ** return 0. If the mark does not exist and the bCreate flag is true,
@@ -1210,28 +1198,26 @@
1198 if( rc ) fossil_fatal("cannot create directory \"%s\"", z);
1199 fossil_free(z);
1200
1201 /* Attach the .mirror_state/db database */
1202 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror);
1203 db_begin_write();
1204 db_multi_exec(
1205 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n"
1206 " key TEXT PRIMARY KEY,\n"
1207 " Value ANY\n"
1208 ") WITHOUT ROWID;\n"
1209 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n"
1210 " id INTEGER PRIMARY KEY,\n"
1211 " uuid TEXT UNIQUE,\n"
1212 " githash TEXT UNIQUE\n"
1213 ");"
 
 
 
 
1214 );
1215
1216 /* See if there is any work to be done. Exit early if not, before starting
1217 ** the "git fast-import" command. */
1218 if( !db_exists("SELECT 1 FROM event WHERE type='ci'"
1219 " AND mtime>coalesce((SELECT value FROM mconfig"
1220 " WHERE key='start'),0.0)")
1221 ){
1222 fossil_print("no changes\n");
1223 return;
@@ -1267,40 +1253,35 @@
1253 }
1254
1255 /* Run the export */
1256 rEnd = 0.0;
1257 db_multi_exec(
1258 "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n"
1259 "INSERT INTO tomirror "
1260 "SELECT objid, mtime, blob.uuid FROM event, blob\n"
1261 " WHERE type='ci'"
1262 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)"
1263 " AND blob.rid=event.objid"
1264 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark);"
1265 );
1266 nTotal = db_int(0, "SELECT count(*) FROM tomirror");
1267 if( nLimit<nTotal ){
1268 nTotal = nLimit;
1269 }else if( nLimit>nTotal ){
1270 nLimit = nTotal;
1271 }
1272 db_prepare(&q,
1273 "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime"
1274 );
1275 while( nLimit && db_step(&q)==SQLITE_ROW ){
 
 
1276 int rid = db_column_int(&q, 0);
1277 double rMTime = db_column_double(&q, 1);
1278 const char *zUuid = db_column_text(&q, 2);
1279 if( rMTime>rEnd ) rEnd = rMTime;
1280 gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest);
1281 printf("\r%d/%d ", nTotal-nLimit, nTotal);
1282 fflush(stdout);
 
 
 
 
1283 }
1284 db_finalize(&q);
1285 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)");
1286 db_bind_double(&q, ":x", rEnd);
1287 db_step(&q);
@@ -1320,19 +1301,34 @@
1301 if( pOut ){
1302 pIn = fopen(".mirror_state/in", "ab");
1303 if( pIn==0 ){
1304 fossil_fatal("cannot open %s/.mirror_state/in for appending", zMirror);
1305 }
1306 db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id");
1307 while( fgets(zLine, sizeof(zLine), pOut) ){
1308 int j, k;
1309 if( zLine[0]!=':' ) continue;
1310 db_bind_int(&q, ":id", atoi(zLine+1));
1311 for(j=1; zLine[j] && zLine[j]!=' '; j++){}
1312 if( zLine[j]!=' ' ) continue;
1313 j++;
1314 if( zLine[j]==0 ) continue;
1315 for(k=j; fossil_isalnum(zLine[k]); k++){}
1316 zLine[k] = 0;
1317 db_bind_text(&q, ":githash", &zLine[j]);
1318 db_step(&q);
1319 db_reset(&q);
1320 fputs(zLine, pIn);
1321 }
1322 db_finalize(&q);
1323 fclose(pOut);
1324 fclose(pIn);
1325 file_delete(".mirror_state/out");
1326 }else{
1327 fossil_fatal("git fast-import didn't generate a marks file!");
1328 }
1329 db_commit_transaction();
1330
1331 /* Optionally do a "git push" */
1332 }
1333
1334 /*
1335

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button