Fossil SCM

Implement the ability to resume a clone that has failed. This is a variation on the attempt made in [ec26471439] that was never completed. The significant difference in the use of the "clone protocol" which uses cfile cards to complete the synchronization rather than the "sync protocol" which takes much longer using the file card. a

andybradford 2023-11-24 21:39 trunk
Commit ee710cc171e6164bfad85e6b38f63f7868e3d2706236b40ad078ae898cac5962
3 files changed +30 -16 +3 +15
+30 -16
--- src/clone.c
+++ src/clone.c
@@ -129,10 +129,11 @@
129129
** check-out
130130
** --nocompress Omit extra delta compression
131131
** --no-open Clone only. Do not open a check-out.
132132
** --once Don't remember the URI.
133133
** --private Also clone private branches
134
+** --resume Resume a failed clone
134135
** --save-http-password Remember the HTTP password without asking
135136
** -c|--ssh-command SSH Use SSH as the "ssh" command
136137
** --ssl-identity FILENAME Use the SSL identity if requested by the server
137138
** --transport-command CMD Use CMD to move messages to the server and back
138139
** -u|--unversioned Also sync unversioned content
@@ -149,10 +150,11 @@
149150
int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
150151
int syncFlags = SYNC_CLONE;
151152
int noCompress = find_option("nocompress",0,0)!=0;
152153
int noOpen = find_option("no-open",0,0)!=0;
153154
int allowNested = find_option("nested",0,0)!=0; /* Used by open */
155
+ int bResume = find_option("resume",0,0)!=0; /* Resume a failed clone */
154156
const char *zRepo = 0; /* Name of the new local repository file */
155157
const char *zWorkDir = 0; /* Open in this directory, if not zero */
156158
157159
158160
/* Also clone private branches */
@@ -197,11 +199,11 @@
197199
if( zWorkDir==0 ){
198200
zWorkDir = mprintf("./%s", zBase);
199201
}
200202
fossil_free(zBase);
201203
}
202
- if( -1 != file_size(zRepo, ExtFILE) ){
204
+ if( -1 != file_size(zRepo, ExtFILE) && bResume==0 ){
203205
fossil_fatal("file already exists: %s", zRepo);
204206
}
205207
/* Fail before clone if open will fail because inside an open check-out */
206208
if( zWorkDir!=0 && zWorkDir[0]!=0 && !noOpen ){
207209
if( db_open_local_v2(0, allowNested) ){
@@ -226,19 +228,27 @@
226228
g.zLogin = db_text(0, "SELECT login FROM user WHERE cap LIKE '%%s%%'");
227229
}
228230
fossil_print("Repository cloned into %s\n", zRepo);
229231
}else{
230232
db_close_config();
231
- db_create_repository(zRepo);
232
- db_open_repository(zRepo);
233
- db_open_config(0,0);
234
- db_begin_transaction();
235
- db_record_repository_filename(zRepo);
236
- db_initial_setup(0, 0, zDefaultUser);
237
- user_select();
238
- db_set("content-schema", CONTENT_SCHEMA, 0);
239
- db_set("aux-schema", AUX_SCHEMA_MAX, 0);
233
+ if( bResume ){
234
+ db_open_repository(zRepo);
235
+ db_open_config(0,0);
236
+ db_begin_transaction();
237
+ user_select();
238
+ }else{
239
+ db_create_repository(zRepo);
240
+ db_open_repository(zRepo);
241
+ db_open_config(0,0);
242
+ db_begin_transaction();
243
+ db_record_repository_filename(zRepo);
244
+ db_initial_setup(0, 0, zDefaultUser);
245
+ user_select();
246
+ db_set("content-schema", CONTENT_SCHEMA, 0);
247
+ db_set("aux-schema", AUX_SCHEMA_MAX, 0);
248
+ db_set_int("aux-clone-seqno", 1, 0);
249
+ }
240250
db_set("rebuilt", get_version(), 0);
241251
db_unset("hash-policy", 0);
242252
remember_or_get_http_auth(zHttpAuth, urlFlags & URL_REMEMBER, g.argv[2]);
243253
url_remember();
244254
if( g.zSSLIdentity!=0 ){
@@ -263,22 +273,26 @@
263273
url_get_password_if_needed();
264274
g.xlinkClusterOnly = 1;
265275
nErr = client_sync(syncFlags,CONFIGSET_ALL,0,0);
266276
g.xlinkClusterOnly = 0;
267277
verify_cancel();
278
+ if( nErr ){
279
+ fossil_warning("server returned an error - clone incomplete");
280
+ }else{
281
+ db_unprotect(PROTECT_CONFIG);
282
+ db_multi_exec("DELETE FROM config WHERE name = 'aux-clone-seqno';");
283
+ db_protect_pop();
284
+ }
268285
db_end_transaction(0);
269286
db_close(1);
270
- if( nErr ){
271
- file_delete(zRepo);
272
- fossil_fatal("server returned an error - clone aborted");
273
- }
274287
db_open_repository(zRepo);
275288
}
276289
db_begin_transaction();
277290
if( db_exists("SELECT 1 FROM delta WHERE srcId IN phantom") ){
278
- fossil_fatal("there are unresolved deltas -"
279
- " the clone is probably incomplete and unusable.");
291
+ fossil_warning("there are unresolved deltas -"
292
+ " the clone is probably incomplete and unusable.\n"
293
+ "It may be possible to continue clone with --resume.");
280294
}
281295
fossil_print("Rebuilding repository meta-data...\n");
282296
rebuild_db(1, 0);
283297
if( !noCompress ){
284298
int nDelta = 0;
285299
--- src/clone.c
+++ src/clone.c
@@ -129,10 +129,11 @@
129 ** check-out
130 ** --nocompress Omit extra delta compression
131 ** --no-open Clone only. Do not open a check-out.
132 ** --once Don't remember the URI.
133 ** --private Also clone private branches
 
134 ** --save-http-password Remember the HTTP password without asking
135 ** -c|--ssh-command SSH Use SSH as the "ssh" command
136 ** --ssl-identity FILENAME Use the SSL identity if requested by the server
137 ** --transport-command CMD Use CMD to move messages to the server and back
138 ** -u|--unversioned Also sync unversioned content
@@ -149,10 +150,11 @@
149 int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
150 int syncFlags = SYNC_CLONE;
151 int noCompress = find_option("nocompress",0,0)!=0;
152 int noOpen = find_option("no-open",0,0)!=0;
153 int allowNested = find_option("nested",0,0)!=0; /* Used by open */
 
154 const char *zRepo = 0; /* Name of the new local repository file */
155 const char *zWorkDir = 0; /* Open in this directory, if not zero */
156
157
158 /* Also clone private branches */
@@ -197,11 +199,11 @@
197 if( zWorkDir==0 ){
198 zWorkDir = mprintf("./%s", zBase);
199 }
200 fossil_free(zBase);
201 }
202 if( -1 != file_size(zRepo, ExtFILE) ){
203 fossil_fatal("file already exists: %s", zRepo);
204 }
205 /* Fail before clone if open will fail because inside an open check-out */
206 if( zWorkDir!=0 && zWorkDir[0]!=0 && !noOpen ){
207 if( db_open_local_v2(0, allowNested) ){
@@ -226,19 +228,27 @@
226 g.zLogin = db_text(0, "SELECT login FROM user WHERE cap LIKE '%%s%%'");
227 }
228 fossil_print("Repository cloned into %s\n", zRepo);
229 }else{
230 db_close_config();
231 db_create_repository(zRepo);
232 db_open_repository(zRepo);
233 db_open_config(0,0);
234 db_begin_transaction();
235 db_record_repository_filename(zRepo);
236 db_initial_setup(0, 0, zDefaultUser);
237 user_select();
238 db_set("content-schema", CONTENT_SCHEMA, 0);
239 db_set("aux-schema", AUX_SCHEMA_MAX, 0);
 
 
 
 
 
 
 
 
240 db_set("rebuilt", get_version(), 0);
241 db_unset("hash-policy", 0);
242 remember_or_get_http_auth(zHttpAuth, urlFlags & URL_REMEMBER, g.argv[2]);
243 url_remember();
244 if( g.zSSLIdentity!=0 ){
@@ -263,22 +273,26 @@
263 url_get_password_if_needed();
264 g.xlinkClusterOnly = 1;
265 nErr = client_sync(syncFlags,CONFIGSET_ALL,0,0);
266 g.xlinkClusterOnly = 0;
267 verify_cancel();
 
 
 
 
 
 
 
268 db_end_transaction(0);
269 db_close(1);
270 if( nErr ){
271 file_delete(zRepo);
272 fossil_fatal("server returned an error - clone aborted");
273 }
274 db_open_repository(zRepo);
275 }
276 db_begin_transaction();
277 if( db_exists("SELECT 1 FROM delta WHERE srcId IN phantom") ){
278 fossil_fatal("there are unresolved deltas -"
279 " the clone is probably incomplete and unusable.");
 
280 }
281 fossil_print("Rebuilding repository meta-data...\n");
282 rebuild_db(1, 0);
283 if( !noCompress ){
284 int nDelta = 0;
285
--- src/clone.c
+++ src/clone.c
@@ -129,10 +129,11 @@
129 ** check-out
130 ** --nocompress Omit extra delta compression
131 ** --no-open Clone only. Do not open a check-out.
132 ** --once Don't remember the URI.
133 ** --private Also clone private branches
134 ** --resume Resume a failed clone
135 ** --save-http-password Remember the HTTP password without asking
136 ** -c|--ssh-command SSH Use SSH as the "ssh" command
137 ** --ssl-identity FILENAME Use the SSL identity if requested by the server
138 ** --transport-command CMD Use CMD to move messages to the server and back
139 ** -u|--unversioned Also sync unversioned content
@@ -149,10 +150,11 @@
150 int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
151 int syncFlags = SYNC_CLONE;
152 int noCompress = find_option("nocompress",0,0)!=0;
153 int noOpen = find_option("no-open",0,0)!=0;
154 int allowNested = find_option("nested",0,0)!=0; /* Used by open */
155 int bResume = find_option("resume",0,0)!=0; /* Resume a failed clone */
156 const char *zRepo = 0; /* Name of the new local repository file */
157 const char *zWorkDir = 0; /* Open in this directory, if not zero */
158
159
160 /* Also clone private branches */
@@ -197,11 +199,11 @@
199 if( zWorkDir==0 ){
200 zWorkDir = mprintf("./%s", zBase);
201 }
202 fossil_free(zBase);
203 }
204 if( -1 != file_size(zRepo, ExtFILE) && bResume==0 ){
205 fossil_fatal("file already exists: %s", zRepo);
206 }
207 /* Fail before clone if open will fail because inside an open check-out */
208 if( zWorkDir!=0 && zWorkDir[0]!=0 && !noOpen ){
209 if( db_open_local_v2(0, allowNested) ){
@@ -226,19 +228,27 @@
228 g.zLogin = db_text(0, "SELECT login FROM user WHERE cap LIKE '%%s%%'");
229 }
230 fossil_print("Repository cloned into %s\n", zRepo);
231 }else{
232 db_close_config();
233 if( bResume ){
234 db_open_repository(zRepo);
235 db_open_config(0,0);
236 db_begin_transaction();
237 user_select();
238 }else{
239 db_create_repository(zRepo);
240 db_open_repository(zRepo);
241 db_open_config(0,0);
242 db_begin_transaction();
243 db_record_repository_filename(zRepo);
244 db_initial_setup(0, 0, zDefaultUser);
245 user_select();
246 db_set("content-schema", CONTENT_SCHEMA, 0);
247 db_set("aux-schema", AUX_SCHEMA_MAX, 0);
248 db_set_int("aux-clone-seqno", 1, 0);
249 }
250 db_set("rebuilt", get_version(), 0);
251 db_unset("hash-policy", 0);
252 remember_or_get_http_auth(zHttpAuth, urlFlags & URL_REMEMBER, g.argv[2]);
253 url_remember();
254 if( g.zSSLIdentity!=0 ){
@@ -263,22 +273,26 @@
273 url_get_password_if_needed();
274 g.xlinkClusterOnly = 1;
275 nErr = client_sync(syncFlags,CONFIGSET_ALL,0,0);
276 g.xlinkClusterOnly = 0;
277 verify_cancel();
278 if( nErr ){
279 fossil_warning("server returned an error - clone incomplete");
280 }else{
281 db_unprotect(PROTECT_CONFIG);
282 db_multi_exec("DELETE FROM config WHERE name = 'aux-clone-seqno';");
283 db_protect_pop();
284 }
285 db_end_transaction(0);
286 db_close(1);
 
 
 
 
287 db_open_repository(zRepo);
288 }
289 db_begin_transaction();
290 if( db_exists("SELECT 1 FROM delta WHERE srcId IN phantom") ){
291 fossil_warning("there are unresolved deltas -"
292 " the clone is probably incomplete and unusable.\n"
293 "It may be possible to continue clone with --resume.");
294 }
295 fossil_print("Rebuilding repository meta-data...\n");
296 rebuild_db(1, 0);
297 if( !noCompress ){
298 int nDelta = 0;
299
+3
--- src/db.c
+++ src/db.c
@@ -4237,10 +4237,13 @@
42374237
"or file:");
42384238
}
42394239
42404240
db_open_config(0,0);
42414241
db_open_repository(zRepo);
4242
+ if( db_get_int("aux-clone-seqno",0)!=0 ){
4243
+ fossil_fatal("This repository appears to be an incomplete clone.");
4244
+ }
42424245
42434246
/* Figure out which revision to open. */
42444247
if( !emptyFlag ){
42454248
if( g.argc==4 ){
42464249
g.zOpenRevision = g.argv[3];
42474250
--- src/db.c
+++ src/db.c
@@ -4237,10 +4237,13 @@
4237 "or file:");
4238 }
4239
4240 db_open_config(0,0);
4241 db_open_repository(zRepo);
 
 
 
4242
4243 /* Figure out which revision to open. */
4244 if( !emptyFlag ){
4245 if( g.argc==4 ){
4246 g.zOpenRevision = g.argv[3];
4247
--- src/db.c
+++ src/db.c
@@ -4237,10 +4237,13 @@
4237 "or file:");
4238 }
4239
4240 db_open_config(0,0);
4241 db_open_repository(zRepo);
4242 if( db_get_int("aux-clone-seqno",0)!=0 ){
4243 fossil_fatal("This repository appears to be an incomplete clone.");
4244 }
4245
4246 /* Figure out which revision to open. */
4247 if( !emptyFlag ){
4248 if( g.argc==4 ){
4249 g.zOpenRevision = g.argv[3];
4250
+15
--- src/xfer.c
+++ src/xfer.c
@@ -1465,10 +1465,14 @@
14651465
blob_is_int(&xfer.aToken[2], &seqno);
14661466
max = db_int(0, "SELECT max(rid) FROM blob");
14671467
while( xfer.mxSend>(int)blob_size(xfer.pOut) && seqno<=max){
14681468
if( time(NULL) >= xfer.maxTime ) break;
14691469
if( iVers>=3 ){
1470
+ if( seqno==1 ){
1471
+ send_all(&xfer);
1472
+ if( xfer.syncPrivate ) send_private(&xfer);
1473
+ }
14701474
send_compressed_file(&xfer, seqno);
14711475
}else{
14721476
send_file(&xfer, seqno, 0, 1);
14731477
}
14741478
seqno++;
@@ -1996,10 +2000,16 @@
19962000
&& configRcvMask==0
19972001
&& configSendMask==0
19982002
){
19992003
return 0; /* Nothing to do */
20002004
}
2005
+
2006
+ if( (syncFlags & SYNC_CLONE)==0 && db_get_int("aux-clone-seqno",0) ){
2007
+ fossil_fatal("Unable to synchronize due to incomplete clone.");
2008
+ }else{
2009
+ cloneSeqno = db_get_int("aux-clone-seqno",1);
2010
+ }
20012011
20022012
/* Compute an appropriate project code. zPCode is the project code
20032013
** for the local repository. zAltPCode will usually be NULL, but might
20042014
** also be an alternative project code to expect on the server. When
20052015
** zAltPCode is not NULL, that means we are doing a cross-project import -
@@ -2810,10 +2820,15 @@
28102820
}
28112821
nUncRcvd += blob_size(&recv);
28122822
blob_reset(&recv);
28132823
nCycle++;
28142824
2825
+ /* Record the current cloneSeqno in the event that clone fails to enable
2826
+ ** the ability to resume from this same point in clone. */
2827
+ if( (syncFlags & SYNC_CLONE)!=0 ){
2828
+ db_set_int("aux-clone-seqno", cloneSeqno, 0);
2829
+ }
28152830
/* Set go to 1 if we need to continue the sync/push/pull/clone for
28162831
** another round. Set go to 0 if it is time to quit. */
28172832
nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
28182833
if( (nFileRecv>0 || newPhantom) && db_exists("SELECT 1 FROM phantom") ){
28192834
go = 1;
28202835
--- src/xfer.c
+++ src/xfer.c
@@ -1465,10 +1465,14 @@
1465 blob_is_int(&xfer.aToken[2], &seqno);
1466 max = db_int(0, "SELECT max(rid) FROM blob");
1467 while( xfer.mxSend>(int)blob_size(xfer.pOut) && seqno<=max){
1468 if( time(NULL) >= xfer.maxTime ) break;
1469 if( iVers>=3 ){
 
 
 
 
1470 send_compressed_file(&xfer, seqno);
1471 }else{
1472 send_file(&xfer, seqno, 0, 1);
1473 }
1474 seqno++;
@@ -1996,10 +2000,16 @@
1996 && configRcvMask==0
1997 && configSendMask==0
1998 ){
1999 return 0; /* Nothing to do */
2000 }
 
 
 
 
 
 
2001
2002 /* Compute an appropriate project code. zPCode is the project code
2003 ** for the local repository. zAltPCode will usually be NULL, but might
2004 ** also be an alternative project code to expect on the server. When
2005 ** zAltPCode is not NULL, that means we are doing a cross-project import -
@@ -2810,10 +2820,15 @@
2810 }
2811 nUncRcvd += blob_size(&recv);
2812 blob_reset(&recv);
2813 nCycle++;
2814
 
 
 
 
 
2815 /* Set go to 1 if we need to continue the sync/push/pull/clone for
2816 ** another round. Set go to 0 if it is time to quit. */
2817 nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
2818 if( (nFileRecv>0 || newPhantom) && db_exists("SELECT 1 FROM phantom") ){
2819 go = 1;
2820
--- src/xfer.c
+++ src/xfer.c
@@ -1465,10 +1465,14 @@
1465 blob_is_int(&xfer.aToken[2], &seqno);
1466 max = db_int(0, "SELECT max(rid) FROM blob");
1467 while( xfer.mxSend>(int)blob_size(xfer.pOut) && seqno<=max){
1468 if( time(NULL) >= xfer.maxTime ) break;
1469 if( iVers>=3 ){
1470 if( seqno==1 ){
1471 send_all(&xfer);
1472 if( xfer.syncPrivate ) send_private(&xfer);
1473 }
1474 send_compressed_file(&xfer, seqno);
1475 }else{
1476 send_file(&xfer, seqno, 0, 1);
1477 }
1478 seqno++;
@@ -1996,10 +2000,16 @@
2000 && configRcvMask==0
2001 && configSendMask==0
2002 ){
2003 return 0; /* Nothing to do */
2004 }
2005
2006 if( (syncFlags & SYNC_CLONE)==0 && db_get_int("aux-clone-seqno",0) ){
2007 fossil_fatal("Unable to synchronize due to incomplete clone.");
2008 }else{
2009 cloneSeqno = db_get_int("aux-clone-seqno",1);
2010 }
2011
2012 /* Compute an appropriate project code. zPCode is the project code
2013 ** for the local repository. zAltPCode will usually be NULL, but might
2014 ** also be an alternative project code to expect on the server. When
2015 ** zAltPCode is not NULL, that means we are doing a cross-project import -
@@ -2810,10 +2820,15 @@
2820 }
2821 nUncRcvd += blob_size(&recv);
2822 blob_reset(&recv);
2823 nCycle++;
2824
2825 /* Record the current cloneSeqno in the event that clone fails to enable
2826 ** the ability to resume from this same point in clone. */
2827 if( (syncFlags & SYNC_CLONE)!=0 ){
2828 db_set_int("aux-clone-seqno", cloneSeqno, 0);
2829 }
2830 /* Set go to 1 if we need to continue the sync/push/pull/clone for
2831 ** another round. Set go to 0 if it is time to quit. */
2832 nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
2833 if( (nFileRecv>0 || newPhantom) && db_exists("SELECT 1 FROM phantom") ){
2834 go = 1;
2835

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button