Fossil SCM

New higher-performance clone algorithm merged in from the experimental branch.

drh 2010-10-28 14:03 trunk merge

Commit fda9b15cfcf738f1a0ecdcff1c0e05db9e65859d

Parent ff79f97a072ee1c…

6 files changed +52 -1 +1 -1 +36 -6 +34 +10 +128 -41

~ src/content.c ~ src/http.c ~ src/manifest.c ~ src/rebuild.c ~ src/schema.c ~ src/xfer.c

M src/content.c

+52 -1

		--- src/content.c
		+++ src/content.c
		@@ -325,31 +325,72 @@
325	325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	326	blob_uncompress(&content, &content);
327	327	blob_write_to_file(&content, zFile);
328	328	}
329	329
	330	+/*
	331	+** The following flag is set to disable the automatic calls to
	332	+** manifest_crosslink() when a record is dephantomized. This
	333	+** flag can be set (for example) when doing a clone when we know
	334	+** that rebuild will be run over all records at the conclusion
	335	+** of the operation.
	336	+*/
	337	+static int ignoreDephantomizations = 0;
	338	+
330	339	/*
331	340	** When a record is converted from a phantom to a real record,
332	341	** if that record has other records that are derived by delta,
333	342	** then call manifest_crosslink() on those other records.
	343	+**
	344	+** If the formerly phantom record or any of the other records
	345	+** derived by delta from the former phantom are a baseline manifest,
	346	+** then also invoke manifest_crosslink() on the delta-manifests
	347	+** associated with that baseline.
334	348	**
335	349	** Tail recursion is used to minimize stack depth.
336	350	*/
337	351	void after_dephantomize(int rid, int linkFlag){
338	352	Stmt q;
339	353	int nChildAlloc = 0;
340	354	int *aChild = 0;
	355	+ Blob content;
341	356
	357	+ if( ignoreDephantomizations ) return;
342	358	while( rid ){
343	359	int nChildUsed = 0;
344	360	int i;
	361	+
	362	+ /* Parse the object rid itself */
345	363	if( linkFlag ){
346		- Blob content;
347	364	content_get(rid, &content);
348	365	manifest_crosslink(rid, &content);
349	366	blob_reset(&content);
350	367	}
	368	+
	369	+ /* Parse all delta-manifests that depend on baseline-manifest rid */
	370	+ db_prepare(&q, "SELECT rid FROM orphan WHERE baseline=%d", rid);
	371	+ while( db_step(&q)==SQLITE_ROW ){
	372	+ int child = db_column_int(&q, 0);
	373	+ if( nChildUsed>=nChildAlloc ){
	374	+ nChildAlloc = nChildAlloc*2 + 10;
	375	+ aChild = fossil_realloc(aChild, nChildAlloc*sizeof(aChild));
	376	+ }
	377	+ aChild[nChildUsed++] = child;
	378	+ }
	379	+ db_finalize(&q);
	380	+ for(i=0; i<nChildUsed; i++){
	381	+ content_get(aChild[i], &content);
	382	+ manifest_crosslink(aChild[i], &content);
	383	+ blob_reset(&content);
	384	+ }
	385	+ if( nChildUsed ){
	386	+ db_multi_exec("DELETE FROM orphan WHERE baseline=%d", rid);
	387	+ }
	388	+
	389	+ /* Recursively dephantomize all artifacts that are derived by
	390	+ ** delta from artifact rid */
	391	+ nChildUsed = 0;
351	392	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
352	393	while( db_step(&q)==SQLITE_ROW ){
353	394	int child = db_column_int(&q, 0);
354	395	if( nChildUsed>=nChildAlloc ){
355	396	nChildAlloc = nChildAlloc*2 + 10;
		@@ -359,15 +400,25 @@
359	400	}
360	401	db_finalize(&q);
361	402	for(i=1; i<nChildUsed; i++){
362	403	after_dephantomize(aChild[i], 1);
363	404	}
	405	+
	406	+ /* Tail recursion for the common case where only a single artifact
	407	+ ** is derived by delta from rid... */
364	408	rid = nChildUsed>0 ? aChild[0] : 0;
365	409	linkFlag = 1;
366	410	}
367	411	free(aChild);
368	412	}
	413	+
	414	+/*
	415	+** Turn dephantomization processing on or off.
	416	+*/
	417	+void content_enable_dephantomize(int onoff){
	418	+ ignoreDephantomizations = !onoff;
	419	+}
369	420
370	421	/*
371	422	** Write content into the database. Return the record ID. If the
372	423	** content is already in the database, just return the record ID.
373	424	**
374	425

	--- src/content.c
	+++ src/content.c
	@@ -325,31 +325,72 @@
325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	blob_uncompress(&content, &content);
327	blob_write_to_file(&content, zFile);
328	}
329









330	/*
331	** When a record is converted from a phantom to a real record,
332	** if that record has other records that are derived by delta,
333	** then call manifest_crosslink() on those other records.





334	**
335	** Tail recursion is used to minimize stack depth.
336	*/
337	void after_dephantomize(int rid, int linkFlag){
338	Stmt q;
339	int nChildAlloc = 0;
340	int *aChild = 0;

341

342	while( rid ){
343	int nChildUsed = 0;
344	int i;


345	if( linkFlag ){
346	Blob content;
347	content_get(rid, &content);
348	manifest_crosslink(rid, &content);
349	blob_reset(&content);
350	}
























351	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
352	while( db_step(&q)==SQLITE_ROW ){
353	int child = db_column_int(&q, 0);
354	if( nChildUsed>=nChildAlloc ){
355	nChildAlloc = nChildAlloc*2 + 10;
	@@ -359,15 +400,25 @@
359	}
360	db_finalize(&q);
361	for(i=1; i<nChildUsed; i++){
362	after_dephantomize(aChild[i], 1);
363	}



364	rid = nChildUsed>0 ? aChild[0] : 0;
365	linkFlag = 1;
366	}
367	free(aChild);
368	}







369
370	/*
371	** Write content into the database. Return the record ID. If the
372	** content is already in the database, just return the record ID.
373	**
374

	--- src/content.c
	+++ src/content.c
	@@ -325,31 +325,72 @@
325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	blob_uncompress(&content, &content);
327	blob_write_to_file(&content, zFile);
328	}
329
330	/*
331	** The following flag is set to disable the automatic calls to
332	** manifest_crosslink() when a record is dephantomized. This
333	** flag can be set (for example) when doing a clone when we know
334	** that rebuild will be run over all records at the conclusion
335	** of the operation.
336	*/
337	static int ignoreDephantomizations = 0;
338
339	/*
340	** When a record is converted from a phantom to a real record,
341	** if that record has other records that are derived by delta,
342	** then call manifest_crosslink() on those other records.
343	**
344	** If the formerly phantom record or any of the other records
345	** derived by delta from the former phantom are a baseline manifest,
346	** then also invoke manifest_crosslink() on the delta-manifests
347	** associated with that baseline.
348	**
349	** Tail recursion is used to minimize stack depth.
350	*/
351	void after_dephantomize(int rid, int linkFlag){
352	Stmt q;
353	int nChildAlloc = 0;
354	int *aChild = 0;
355	Blob content;
356
357	if( ignoreDephantomizations ) return;
358	while( rid ){
359	int nChildUsed = 0;
360	int i;
361
362	/* Parse the object rid itself */
363	if( linkFlag ){

364	content_get(rid, &content);
365	manifest_crosslink(rid, &content);
366	blob_reset(&content);
367	}
368
369	/* Parse all delta-manifests that depend on baseline-manifest rid */
370	db_prepare(&q, "SELECT rid FROM orphan WHERE baseline=%d", rid);
371	while( db_step(&q)==SQLITE_ROW ){
372	int child = db_column_int(&q, 0);
373	if( nChildUsed>=nChildAlloc ){
374	nChildAlloc = nChildAlloc*2 + 10;
375	aChild = fossil_realloc(aChild, nChildAlloc*sizeof(aChild));
376	}
377	aChild[nChildUsed++] = child;
378	}
379	db_finalize(&q);
380	for(i=0; i<nChildUsed; i++){
381	content_get(aChild[i], &content);
382	manifest_crosslink(aChild[i], &content);
383	blob_reset(&content);
384	}
385	if( nChildUsed ){
386	db_multi_exec("DELETE FROM orphan WHERE baseline=%d", rid);
387	}
388
389	/* Recursively dephantomize all artifacts that are derived by
390	** delta from artifact rid */
391	nChildUsed = 0;
392	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
393	while( db_step(&q)==SQLITE_ROW ){
394	int child = db_column_int(&q, 0);
395	if( nChildUsed>=nChildAlloc ){
396	nChildAlloc = nChildAlloc*2 + 10;
	@@ -359,15 +400,25 @@
400	}
401	db_finalize(&q);
402	for(i=1; i<nChildUsed; i++){
403	after_dephantomize(aChild[i], 1);
404	}
405
406	/* Tail recursion for the common case where only a single artifact
407	** is derived by delta from rid... */
408	rid = nChildUsed>0 ? aChild[0] : 0;
409	linkFlag = 1;
410	}
411	free(aChild);
412	}
413
414	/*
415	** Turn dephantomization processing on or off.
416	*/
417	void content_enable_dephantomize(int onoff){
418	ignoreDephantomizations = !onoff;
419	}
420
421	/*
422	** Write content into the database. Return the record ID. If the
423	** content is already in the database, just return the record ID.
424	**
425

M src/http.c

+1 -1

		--- src/http.c
		+++ src/http.c
		@@ -267,11 +267,11 @@
267	267	}
268	268	z[j] = 0;
269	269	fossil_fatal("server sends error: %s", z);
270	270	}
271	271	if( g.fHttpTrace ){
272		- printf("HTTP RECEIVE:\n%s\n=======================\n", blob_str(pReply));
	272	+ /printf("HTTP RECEIVE:\n%s\n=======================\n",blob_str(pReply));/
273	273	}else{
274	274	blob_uncompress(pReply, pReply);
275	275	}
276	276
277	277	/*
278	278

	--- src/http.c
	+++ src/http.c
	@@ -267,11 +267,11 @@
267	}
268	z[j] = 0;
269	fossil_fatal("server sends error: %s", z);
270	}
271	if( g.fHttpTrace ){
272	printf("HTTP RECEIVE:\n%s\n=======================\n", blob_str(pReply));
273	}else{
274	blob_uncompress(pReply, pReply);
275	}
276
277	/*
278

	--- src/http.c
	+++ src/http.c
	@@ -267,11 +267,11 @@
267	}
268	z[j] = 0;
269	fossil_fatal("server sends error: %s", z);
270	}
271	if( g.fHttpTrace ){
272	/printf("HTTP RECEIVE:\n%s\n=======================\n",blob_str(pReply));/
273	}else{
274	blob_uncompress(pReply, pReply);
275	}
276
277	/*
278

M src/manifest.c

+36 -6

		--- src/manifest.c
		+++ src/manifest.c
		@@ -913,28 +913,47 @@
913	913	}
914	914	}
915	915
916	916	/*
917	917	** Fetch the baseline associated with the delta-manifest p.
918		-** Print a fatal-error and quit if unable to load the baseline.
	918	+** Return 0 on success. If unable to parse the baseline,
	919	+** throw an error. If the baseline is a manifest, throw an
	920	+** error if throwError is true, or record that p is an orphan
	921	+** and return 1 throwError is false.
919	922	*/
920		-static void fetch_baseline(Manifest *p){
	923	+static int fetch_baseline(Manifest *p, int throwError){
921	924	if( p->zBaseline!=0 && p->pBaseline==0 ){
922	925	int rid = uuid_to_rid(p->zBaseline, 0);
	926	+ if( rid==0 && !throwError ){
	927	+ rid = content_new(p->zBaseline);
	928	+ db_multi_exec(
	929	+ "INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
	930	+ rid, p->rid
	931	+ );
	932	+ return 1;
	933	+ }
923	934	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
924	935	if( p->pBaseline==0 ){
	936	+ if( !throwError && db_exists("SELECT 1 FROM phantom WHERE rid=%d",rid) ){
	937	+ db_multi_exec(
	938	+ "INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
	939	+ rid, p->rid
	940	+ );
	941	+ return 1;
	942	+ }
925	943	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
926	944	}
927	945	}
	946	+ return 0;
928	947	}
929	948
930	949	/*
931	950	** Rewind a manifest-file iterator back to the beginning of the manifest.
932	951	*/
933	952	void manifest_file_rewind(Manifest *p){
934	953	p->iFile = 0;
935		- fetch_baseline(p);
	954	+ fetch_baseline(p, 1);
936	955	if( p->pBaseline ){
937	956	p->pBaseline->iFile = 0;
938	957	}
939	958	}
940	959
		@@ -1123,11 +1142,11 @@
1123	1142	ManifestFile *pFile;
1124	1143
1125	1144	pFile = manifest_file_seek_base(p, zName);
1126	1145	if( pFile && pFile->zUuid==0 ) return 0;
1127	1146	if( pFile==0 && p->zBaseline ){
1128		- fetch_baseline(p);
	1147	+ fetch_baseline(p, 1);
1129	1148	pFile = manifest_file_seek_base(p->pBaseline, zName);
1130	1149	}
1131	1150	return pFile;
1132	1151	}
1133	1152
		@@ -1181,15 +1200,18 @@
1181	1200	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1182	1201	content_get(otherRid, &otherContent);
1183	1202	if( blob_size(&otherContent)==0 ) return;
1184	1203	*ppOther = manifest_parse(&otherContent, otherRid);
1185	1204	if( *ppOther==0 ) return;
	1205	+ }
	1206	+ if( fetch_baseline(pParent, 0) \|\| fetch_baseline(pChild, 0) ){
	1207	+ manifest_destroy(*ppOther);
	1208	+ return;
1186	1209	}
1187	1210	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1188	1211	content_deltify(pid, cid, 0);
1189	1212	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){
1190		- fetch_baseline(pParent);
1191	1213	content_deltify(pParent->pBaseline->rid, cid, 0);
1192	1214	}
1193	1215
1194	1216	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1195	1217	if( pChildFile->zPrior ){
		@@ -1370,10 +1392,14 @@
1370	1392	return 0;
1371	1393	}
1372	1394	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1373	1395	manifest_destroy(p);
1374	1396	return 0;
	1397	+ }
	1398	+ if( p->type==CFTYPE_MANIFEST && fetch_baseline(p, 0) ){
	1399	+ manifest_destroy(p);
	1400	+ return 0;
1375	1401	}
1376	1402	db_begin_transaction();
1377	1403	if( p->type==CFTYPE_MANIFEST ){
1378	1404	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1379	1405	char *zCom;
		@@ -1427,16 +1453,20 @@
1427	1453	}
1428	1454	}
1429	1455	}
1430	1456	}
1431	1457	if( p->type==CFTYPE_CLUSTER ){
	1458	+ static Stmt del1;
1432	1459	tag_insert("cluster", 1, 0, rid, p->rDate, rid);
	1460	+ db_static_prepare(&del1, "DELETE FROM unclustered WHERE rid=:rid");
1433	1461	for(i=0; i<p->nCChild; i++){
1434	1462	int mid;
1435	1463	mid = uuid_to_rid(p->azCChild[i], 1);
1436	1464	if( mid>0 ){
1437		- db_multi_exec("DELETE FROM unclustered WHERE rid=%d", mid);
	1465	+ db_bind_int(&del1, ":rid", mid);
	1466	+ db_step(&del1);
	1467	+ db_reset(&del1);
1438	1468	}
1439	1469	}
1440	1470	}
1441	1471	if( p->type==CFTYPE_CONTROL
1442	1472	\|\| p->type==CFTYPE_MANIFEST
1443	1473

	--- src/manifest.c
	+++ src/manifest.c
	@@ -913,28 +913,47 @@
913	}
914	}
915
916	/*
917	** Fetch the baseline associated with the delta-manifest p.
918	** Print a fatal-error and quit if unable to load the baseline.



919	*/
920	static void fetch_baseline(Manifest *p){
921	if( p->zBaseline!=0 && p->pBaseline==0 ){
922	int rid = uuid_to_rid(p->zBaseline, 0);








923	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
924	if( p->pBaseline==0 ){







925	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
926	}
927	}

928	}
929
930	/*
931	** Rewind a manifest-file iterator back to the beginning of the manifest.
932	*/
933	void manifest_file_rewind(Manifest *p){
934	p->iFile = 0;
935	fetch_baseline(p);
936	if( p->pBaseline ){
937	p->pBaseline->iFile = 0;
938	}
939	}
940
	@@ -1123,11 +1142,11 @@
1123	ManifestFile *pFile;
1124
1125	pFile = manifest_file_seek_base(p, zName);
1126	if( pFile && pFile->zUuid==0 ) return 0;
1127	if( pFile==0 && p->zBaseline ){
1128	fetch_baseline(p);
1129	pFile = manifest_file_seek_base(p->pBaseline, zName);
1130	}
1131	return pFile;
1132	}
1133
	@@ -1181,15 +1200,18 @@
1181	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1182	content_get(otherRid, &otherContent);
1183	if( blob_size(&otherContent)==0 ) return;
1184	*ppOther = manifest_parse(&otherContent, otherRid);
1185	if( *ppOther==0 ) return;




1186	}
1187	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1188	content_deltify(pid, cid, 0);
1189	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){
1190	fetch_baseline(pParent);
1191	content_deltify(pParent->pBaseline->rid, cid, 0);
1192	}
1193
1194	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1195	if( pChildFile->zPrior ){
	@@ -1370,10 +1392,14 @@
1370	return 0;
1371	}
1372	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1373	manifest_destroy(p);
1374	return 0;




1375	}
1376	db_begin_transaction();
1377	if( p->type==CFTYPE_MANIFEST ){
1378	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1379	char *zCom;
	@@ -1427,16 +1453,20 @@
1427	}
1428	}
1429	}
1430	}
1431	if( p->type==CFTYPE_CLUSTER ){

1432	tag_insert("cluster", 1, 0, rid, p->rDate, rid);

1433	for(i=0; i<p->nCChild; i++){
1434	int mid;
1435	mid = uuid_to_rid(p->azCChild[i], 1);
1436	if( mid>0 ){
1437	db_multi_exec("DELETE FROM unclustered WHERE rid=%d", mid);


1438	}
1439	}
1440	}
1441	if( p->type==CFTYPE_CONTROL
1442	\|\| p->type==CFTYPE_MANIFEST
1443

	--- src/manifest.c
	+++ src/manifest.c
	@@ -913,28 +913,47 @@
913	}
914	}
915
916	/*
917	** Fetch the baseline associated with the delta-manifest p.
918	** Return 0 on success. If unable to parse the baseline,
919	** throw an error. If the baseline is a manifest, throw an
920	** error if throwError is true, or record that p is an orphan
921	** and return 1 throwError is false.
922	*/
923	static int fetch_baseline(Manifest *p, int throwError){
924	if( p->zBaseline!=0 && p->pBaseline==0 ){
925	int rid = uuid_to_rid(p->zBaseline, 0);
926	if( rid==0 && !throwError ){
927	rid = content_new(p->zBaseline);
928	db_multi_exec(
929	"INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
930	rid, p->rid
931	);
932	return 1;
933	}
934	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
935	if( p->pBaseline==0 ){
936	if( !throwError && db_exists("SELECT 1 FROM phantom WHERE rid=%d",rid) ){
937	db_multi_exec(
938	"INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
939	rid, p->rid
940	);
941	return 1;
942	}
943	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
944	}
945	}
946	return 0;
947	}
948
949	/*
950	** Rewind a manifest-file iterator back to the beginning of the manifest.
951	*/
952	void manifest_file_rewind(Manifest *p){
953	p->iFile = 0;
954	fetch_baseline(p, 1);
955	if( p->pBaseline ){
956	p->pBaseline->iFile = 0;
957	}
958	}
959
	@@ -1123,11 +1142,11 @@
1142	ManifestFile *pFile;
1143
1144	pFile = manifest_file_seek_base(p, zName);
1145	if( pFile && pFile->zUuid==0 ) return 0;
1146	if( pFile==0 && p->zBaseline ){
1147	fetch_baseline(p, 1);
1148	pFile = manifest_file_seek_base(p->pBaseline, zName);
1149	}
1150	return pFile;
1151	}
1152
	@@ -1181,15 +1200,18 @@
1200	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1201	content_get(otherRid, &otherContent);
1202	if( blob_size(&otherContent)==0 ) return;
1203	*ppOther = manifest_parse(&otherContent, otherRid);
1204	if( *ppOther==0 ) return;
1205	}
1206	if( fetch_baseline(pParent, 0) \|\| fetch_baseline(pChild, 0) ){
1207	manifest_destroy(*ppOther);
1208	return;
1209	}
1210	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1211	content_deltify(pid, cid, 0);
1212	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){

1213	content_deltify(pParent->pBaseline->rid, cid, 0);
1214	}
1215
1216	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1217	if( pChildFile->zPrior ){
	@@ -1370,10 +1392,14 @@
1392	return 0;
1393	}
1394	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1395	manifest_destroy(p);
1396	return 0;
1397	}
1398	if( p->type==CFTYPE_MANIFEST && fetch_baseline(p, 0) ){
1399	manifest_destroy(p);
1400	return 0;
1401	}
1402	db_begin_transaction();
1403	if( p->type==CFTYPE_MANIFEST ){
1404	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1405	char *zCom;
	@@ -1427,16 +1453,20 @@
1453	}
1454	}
1455	}
1456	}
1457	if( p->type==CFTYPE_CLUSTER ){
1458	static Stmt del1;
1459	tag_insert("cluster", 1, 0, rid, p->rDate, rid);
1460	db_static_prepare(&del1, "DELETE FROM unclustered WHERE rid=:rid");
1461	for(i=0; i<p->nCChild; i++){
1462	int mid;
1463	mid = uuid_to_rid(p->azCChild[i], 1);
1464	if( mid>0 ){
1465	db_bind_int(&del1, ":rid", mid);
1466	db_step(&del1);
1467	db_reset(&del1);
1468	}
1469	}
1470	}
1471	if( p->type==CFTYPE_CONTROL
1472	\|\| p->type==CFTYPE_MANIFEST
1473

M src/rebuild.c

+34

		--- src/rebuild.c
		+++ src/rebuild.c
		@@ -247,10 +247,11 @@
247	247	*/
248	248	int rebuild_db(int randomize, int doOut){
249	249	Stmt s;
250	250	int errCnt = 0;
251	251	char *zTable;
	252	+ int incrSize;
252	253
253	254	bag_init(&bagDone);
254	255	ttyOutput = doOut;
255	256	processCnt = 0;
256	257	if (!g.fQuiet) {
		@@ -284,10 +285,12 @@
284	285	);
285	286	db_multi_exec(
286	287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	288	);
288	289	totalSize = db_int(0, "SELECT count(*) FROM blob");
	290	+ incrSize = totalSize/100;
	291	+ totalSize += incrSize*2;
289	292	db_prepare(&s,
290	293	"SELECT rid, size FROM blob /scan/"
291	294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	296	);
		@@ -321,10 +324,19 @@
321	324	}
322	325	}
323	326	db_finalize(&s);
324	327	manifest_crosslink_end();
325	328	rebuild_tag_trunk();
	329	+ if (!g.fQuiet) {
	330	+ processCnt += incrSize;
	331	+ percent_complete((processCnt*1000)/totalSize);
	332	+ }
	333	+ create_cluster();
	334	+ if (!g.fQuiet) {
	335	+ processCnt += incrSize;
	336	+ percent_complete((processCnt*1000)/totalSize);
	337	+ }
326	338	if(!g.fQuiet && ttyOutput ){
327	339	printf("\n");
328	340	}
329	341	return errCnt;
330	342	}
		@@ -385,10 +397,32 @@
385	397	"UPDATE config SET value='detached-' \|\| value"
386	398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	399	);
388	400	db_end_transaction(0);
389	401	}
	402	+
	403	+/*
	404	+** COMMAND: test-create-clusters
	405	+**
	406	+** Create clusters for all unclustered artifacts if the number of unclustered
	407	+** artifacts exceeds the current clustering threshold.
	408	+*/
	409	+void test_createcluster_cmd(void){
	410	+ if( g.argc==3 ){
	411	+ db_open_repository(g.argv[2]);
	412	+ }else{
	413	+ db_find_and_open_repository(1);
	414	+ if( g.argc!=2 ){
	415	+ usage("?REPOSITORY-FILENAME?");
	416	+ }
	417	+ db_close();
	418	+ db_open_repository(g.zRepositoryName);
	419	+ }
	420	+ db_begin_transaction();
	421	+ create_cluster();
	422	+ db_end_transaction(0);
	423	+}
390	424
391	425	/*
392	426	** COMMAND: scrub
393	427	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	428	**
395	429

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;

252
253	bag_init(&bagDone);
254	ttyOutput = doOut;
255	processCnt = 0;
256	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
284	);
285	db_multi_exec(
286	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	);
288	totalSize = db_int(0, "SELECT count(*) FROM blob");


289	db_prepare(&s,
290	"SELECT rid, size FROM blob /scan/"
291	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	);
	@@ -321,10 +324,19 @@
321	}
322	}
323	db_finalize(&s);
324	manifest_crosslink_end();
325	rebuild_tag_trunk();









326	if(!g.fQuiet && ttyOutput ){
327	printf("\n");
328	}
329	return errCnt;
330	}
	@@ -385,10 +397,32 @@
385	"UPDATE config SET value='detached-' \|\| value"
386	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	);
388	db_end_transaction(0);
389	}






















390
391	/*
392	** COMMAND: scrub
393	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	**
395

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;
252	int incrSize;
253
254	bag_init(&bagDone);
255	ttyOutput = doOut;
256	processCnt = 0;
257	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
285	);
286	db_multi_exec(
287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
288	);
289	totalSize = db_int(0, "SELECT count(*) FROM blob");
290	incrSize = totalSize/100;
291	totalSize += incrSize*2;
292	db_prepare(&s,
293	"SELECT rid, size FROM blob /scan/"
294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
296	);
	@@ -321,10 +324,19 @@
324	}
325	}
326	db_finalize(&s);
327	manifest_crosslink_end();
328	rebuild_tag_trunk();
329	if (!g.fQuiet) {
330	processCnt += incrSize;
331	percent_complete((processCnt*1000)/totalSize);
332	}
333	create_cluster();
334	if (!g.fQuiet) {
335	processCnt += incrSize;
336	percent_complete((processCnt*1000)/totalSize);
337	}
338	if(!g.fQuiet && ttyOutput ){
339	printf("\n");
340	}
341	return errCnt;
342	}
	@@ -385,10 +397,32 @@
397	"UPDATE config SET value='detached-' \|\| value"
398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
399	);
400	db_end_transaction(0);
401	}
402
403	/*
404	** COMMAND: test-create-clusters
405	**
406	** Create clusters for all unclustered artifacts if the number of unclustered
407	** artifacts exceeds the current clustering threshold.
408	*/
409	void test_createcluster_cmd(void){
410	if( g.argc==3 ){
411	db_open_repository(g.argv[2]);
412	}else{
413	db_find_and_open_repository(1);
414	if( g.argc!=2 ){
415	usage("?REPOSITORY-FILENAME?");
416	}
417	db_close();
418	db_open_repository(g.zRepositoryName);
419	}
420	db_begin_transaction();
421	create_cluster();
422	db_end_transaction(0);
423	}
424
425	/*
426	** COMMAND: scrub
427	** %fossil scrub [--verily] [--force] [REPOSITORY]
428	**
429

M src/schema.c

+10

		--- src/schema.c
		+++ src/schema.c
		@@ -241,10 +241,20 @@
241	241	@ -- UUID but we do not (yet) know the file content.
242	242	@ --
243	243	@ CREATE TABLE phantom(
244	244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	245	@ );
	246	+@
	247	+@ -- A record of orphaned delta-manifests. An orphan is a delta-manifest
	248	+@ -- for which we have content, but its baseline-manifest is a phantom.
	249	+@ -- We have to track all orphan maniftests so that when the baseline arrives,
	250	+@ -- we know to process the orphaned deltas.
	251	+@ CREATE TABLE orphan(
	252	+@ rid INTEGER PRIMARY KEY, -- Delta manifest with a phantom baseline
	253	+@ baseline INTEGER -- Phantom baseline of this orphan
	254	+@ );
	255	+@ CREATE INDEX orphan_baseline ON orphan(baseline);
246	256	@
247	257	@ -- Unclustered records. An unclustered record is a record (including
248	258	@ -- a cluster records themselves) that is not mentioned by some other
249	259	@ -- cluster.
250	260	@ --
251	261

	--- src/schema.c
	+++ src/schema.c
	@@ -241,10 +241,20 @@
241	@ -- UUID but we do not (yet) know the file content.
242	@ --
243	@ CREATE TABLE phantom(
244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	@ );










246	@
247	@ -- Unclustered records. An unclustered record is a record (including
248	@ -- a cluster records themselves) that is not mentioned by some other
249	@ -- cluster.
250	@ --
251

	--- src/schema.c
	+++ src/schema.c
	@@ -241,10 +241,20 @@
241	@ -- UUID but we do not (yet) know the file content.
242	@ --
243	@ CREATE TABLE phantom(
244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	@ );
246	@
247	@ -- A record of orphaned delta-manifests. An orphan is a delta-manifest
248	@ -- for which we have content, but its baseline-manifest is a phantom.
249	@ -- We have to track all orphan maniftests so that when the baseline arrives,
250	@ -- we know to process the orphaned deltas.
251	@ CREATE TABLE orphan(
252	@ rid INTEGER PRIMARY KEY, -- Delta manifest with a phantom baseline
253	@ baseline INTEGER -- Phantom baseline of this orphan
254	@ );
255	@ CREATE INDEX orphan_baseline ON orphan(baseline);
256	@
257	@ -- Unclustered records. An unclustered record is a record (including
258	@ -- a cluster records themselves) that is not mentioned by some other
259	@ -- cluster.
260	@ --
261

M src/xfer.c

+128 -41

		--- src/xfer.c
		+++ src/xfer.c
		@@ -72,11 +72,17 @@
72	72	/*
73	73	** Remember that the other side of the connection already has a copy
74	74	** of the file rid.
75	75	*/
76	76	static void remote_has(int rid){
77		- if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
	77	+ if( rid ){
	78	+ static Stmt q;
	79	+ db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
	80	+ db_bind_int(&q, ":r", rid);
	81	+ db_step(&q);
	82	+ db_reset(&q);
	83	+ }
78	84	}
79	85
80	86	/*
81	87	** The aToken[0..nToken-1] blob array is a parse of a "file" line
82	88	** message. This routine finishes parsing that message and does
		@@ -95,11 +101,11 @@
95	101	** be initialized to an empty string.
96	102	**
97	103	** Any artifact successfully received by this routine is considered to
98	104	** be public and is therefore removed from the "private" table.
99	105	*/
100		-static void xfer_accept_file(Xfer *pXfer){
	106	+static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
101	107	int n;
102	108	int rid;
103	109	int srcid = 0;
104	110	Blob content, hash;
105	111
		@@ -114,13 +120,26 @@
114	120	return;
115	121	}
116	122	blob_zero(&content);
117	123	blob_zero(&hash);
118	124	blob_extract(pXfer->pIn, n, &content);
119		- if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
	125	+ if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
120	126	/* Ignore files that have been shunned */
121	127	return;
	128	+ }
	129	+ if( cloneFlag ){
	130	+ if( pXfer->nToken==4 ){
	131	+ srcid = rid_from_uuid(&pXfer->aToken[2], 1);
	132	+ pXfer->nDeltaRcvd++;
	133	+ }else{
	134	+ srcid = 0;
	135	+ pXfer->nFileRcvd++;
	136	+ }
	137	+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
	138	+ remote_has(rid);
	139	+ blob_reset(&content);
	140	+ return;
122	141	}
123	142	if( pXfer->nToken==4 ){
124	143	Blob src, next;
125	144	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
126	145	if( content_get(srcid, &src)==0 ){
		@@ -467,14 +486,15 @@
467	486	** Check to see if the number of unclustered entries is greater than
468	487	** 100 and if it is, form a new cluster. Unclustered phantoms do not
469	488	** count toward the 100 total. And phantoms are never added to a new
470	489	** cluster.
471	490	*/
472		-static void create_cluster(void){
	491	+void create_cluster(void){
473	492	Blob cluster, cksum;
474	493	Stmt q;
475	494	int nUncl;
	495	+ int nRow = 0;
476	496
477	497	/* We should not ever get any private artifacts in the unclustered table.
478	498	** But if we do (because of a bug) now is a good time to delete them. */
479	499	db_multi_exec(
480	500	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
		@@ -481,30 +501,41 @@
481	501	);
482	502
483	503	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
484	504	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
485	505	" WHERE rid=unclustered.rid)");
486		- if( nUncl<100 ){
487		- return;
488		- }
489		- blob_zero(&cluster);
490		- db_prepare(&q, "SELECT uuid FROM unclustered, blob"
491		- " WHERE NOT EXISTS(SELECT 1 FROM phantom"
492		- " WHERE rid=unclustered.rid)"
493		- " AND unclustered.rid=blob.rid"
494		- " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
495		- " ORDER BY 1");
496		- while( db_step(&q)==SQLITE_ROW ){
497		- blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
498		- }
499		- db_finalize(&q);
500		- md5sum_blob(&cluster, &cksum);
501		- blob_appendf(&cluster, "Z %b\n", &cksum);
502		- blob_reset(&cksum);
503		- db_multi_exec("DELETE FROM unclustered");
504		- content_put(&cluster, 0, 0);
505		- blob_reset(&cluster);
	506	+ if( nUncl>=100 ){
	507	+ blob_zero(&cluster);
	508	+ db_prepare(&q, "SELECT uuid FROM unclustered, blob"
	509	+ " WHERE NOT EXISTS(SELECT 1 FROM phantom"
	510	+ " WHERE rid=unclustered.rid)"
	511	+ " AND unclustered.rid=blob.rid"
	512	+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
	513	+ " ORDER BY 1");
	514	+ while( db_step(&q)==SQLITE_ROW ){
	515	+ blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
	516	+ nRow++;
	517	+ if( nRow>=800 && nUncl>nRow+100 ){
	518	+ md5sum_blob(&cluster, &cksum);
	519	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	520	+ blob_reset(&cksum);
	521	+ content_put(&cluster, 0, 0);
	522	+ blob_reset(&cluster);
	523	+ nUncl -= nRow;
	524	+ nRow = 0;
	525	+ }
	526	+ }
	527	+ db_finalize(&q);
	528	+ db_multi_exec("DELETE FROM unclustered");
	529	+ if( nRow>0 ){
	530	+ md5sum_blob(&cluster, &cksum);
	531	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	532	+ blob_reset(&cksum);
	533	+ content_put(&cluster, 0, 0);
	534	+ blob_reset(&cluster);
	535	+ }
	536	+ }
506	537	}
507	538
508	539	/*
509	540	** Send an igot message for every entry in unclustered table.
510	541	** Return the number of cards sent.
		@@ -606,19 +637,17 @@
606	637	blobarray_zero(xfer.aToken, count(xfer.aToken));
607	638	cgi_set_content_type(g.zContentType);
608	639	blob_zero(&xfer.err);
609	640	xfer.pIn = &g.cgiIn;
610	641	xfer.pOut = cgi_output_blob();
611		- xfer.mxSend = db_get_int("max-download", 5000000);
	642	+ xfer.mxSend = db_get_int("max-download", 20000000);
612	643	g.xferPanic = 1;
613	644
614	645	db_begin_transaction();
615	646	db_multi_exec(
616	647	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
617	648	);
618		- zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
619		- @ # timestamp %s(zNow)
620	649	manifest_crosslink_begin();
621	650	while( blob_line(xfer.pIn, &xfer.line) ){
622	651	if( blob_buffer(&xfer.line)[0]=='#' ) continue;
623	652	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
624	653
		@@ -632,11 +661,11 @@
632	661	cgi_reset_content();
633	662	@ error not\sauthorized\sto\swrite
634	663	nErr++;
635	664	break;
636	665	}
637		- xfer_accept_file(&xfer);
	666	+ xfer_accept_file(&xfer, 0);
638	667	if( blob_size(&xfer.err) ){
639	668	cgi_reset_content();
640	669	@ error %T(blob_str(&xfer.err))
641	670	nErr++;
642	671	break;
		@@ -718,26 +747,42 @@
718	747	isPush = 1;
719	748	}
720	749	}
721	750	}else
722	751
723		- /* clone
	752	+ /* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
724	753	**
725	754	** The client knows nothing. Tell all.
726	755	*/
727	756	if( blob_eq(&xfer.aToken[0], "clone") ){
	757	+ int iVers;
728	758	login_check_credentials();
729	759	if( !g.okClone ){
730	760	cgi_reset_content();
731	761	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
732	762	@ error not\sauthorized\sto\sclone
733	763	nErr++;
734	764	break;
735	765	}
736		- isClone = 1;
737		- isPull = 1;
738		- deltaFlag = 1;
	766	+ if( xfer.nToken==3
	767	+ && blob_is_int(&xfer.aToken[1], &iVers)
	768	+ && iVers>=2
	769	+ ){
	770	+ int seqno, max;
	771	+ blob_is_int(&xfer.aToken[2], &seqno);
	772	+ max = db_int(0, "SELECT max(rid) FROM blob");
	773	+ while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
	774	+ send_file(&xfer, seqno, 0, 1);
	775	+ seqno++;
	776	+ }
	777	+ if( seqno>=max ) seqno = 0;
	778	+ @ clone_seqno %d(seqno)
	779	+ }else{
	780	+ isClone = 1;
	781	+ isPull = 1;
	782	+ deltaFlag = 1;
	783	+ }
739	784	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
740	785	}else
741	786
742	787	/* login USER NONCE SIGNATURE
743	788	**
		@@ -874,10 +919,18 @@
874	919	}
875	920	if( recvConfig ){
876	921	configure_finalize_receive();
877	922	}
878	923	manifest_crosslink_end();
	924	+
	925	+ /* Send the server timestamp last, in case prior processing happened
	926	+ ** to use up a significant fraction of our time window.
	927	+ */
	928	+ zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
	929	+ @ # timestamp %s(zNow)
	930	+ free(zNow);
	931	+
879	932	db_end_transaction(0);
880	933	}
881	934
882	935	/*
883	936	** COMMAND: test-xfer
		@@ -943,13 +996,17 @@
943	996	int origConfigRcvMask; /* Original value of configRcvMask */
944	997	int nFileRecv; /* Number of files received */
945	998	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
946	999	const char zCookie; / Server cookie */
947	1000	int nSent, nRcvd; /* Bytes sent and received (after compression) */
	1001	+ int cloneSeqno = 1; /* Sequence number for clones */
948	1002	Blob send; /* Text we are sending to the server */
949	1003	Blob recv; /* Reply we got back from the server */
950	1004	Xfer xfer; /* Transfer data */
	1005	+ int pctDone; /* Percentage done with a message */
	1006	+ int lastPctDone = -1; /* Last displayed pctDone */
	1007	+ double rArrivalTime; /* Time at which a message arrived */
951	1008	const char *zSCode = db_get("server-code", "x");
952	1009	const char *zPCode = db_get("project-code", 0);
953	1010
954	1011	if( db_get_boolean("dont-push", 0) ) pushFlag = 0;
955	1012	if( pushFlag + pullFlag + cloneFlag == 0
		@@ -977,15 +1034,16 @@
977	1034
978	1035	/*
979	1036	** Always begin with a clone, pull, or push message
980	1037	*/
981	1038	if( cloneFlag ){
982		- blob_appendf(&send, "clone\n");
	1039	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
983	1040	pushFlag = 0;
984	1041	pullFlag = 0;
985	1042	nCardSent++;
986	1043	/* TBD: Request all transferable configuration values */
	1044	+ content_enable_dephantomize(0);
987	1045	}else if( pullFlag ){
988	1046	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
989	1047	nCardSent++;
990	1048	}
991	1049	if( pushFlag ){
		@@ -1009,11 +1067,11 @@
1009	1067	}
1010	1068
1011	1069	/* Generate gimme cards for phantoms and leaf cards
1012	1070	** for all leaves.
1013	1071	*/
1014		- if( pullFlag \|\| cloneFlag ){
	1072	+ if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1015	1073	request_phantoms(&xfer, mxPhantomReq);
1016	1074	}
1017	1075	if( pushFlag ){
1018	1076	send_unsent(&xfer);
1019	1077	nCardSent += send_unclustered(&xfer);
		@@ -1058,22 +1116,27 @@
1058	1116	blob_appendf(&send, "# %s\n", zRandomness);
1059	1117	free(zRandomness);
1060	1118
1061	1119	/* Exchange messages with the server */
1062	1120	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1063		- fossil_print(zValueFormat, "Send:",
	1121	+ fossil_print(zValueFormat, "Sent:",
1064	1122	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1065	1123	xfer.nFileSent, xfer.nDeltaSent);
1066	1124	nCardSent = 0;
1067	1125	nCardRcvd = 0;
1068	1126	xfer.nFileSent = 0;
1069	1127	xfer.nDeltaSent = 0;
1070	1128	xfer.nGimmeSent = 0;
1071	1129	xfer.nIGotSent = 0;
	1130	+ if( !g.cgiOutput && !g.fQuiet ){
	1131	+ printf("waiting for server...");
	1132	+ }
1072	1133	fflush(stdout);
1073	1134	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
	1135	+ lastPctDone = -1;
1074	1136	blob_reset(&send);
	1137	+ rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1075	1138
1076	1139	/* Begin constructing the next message (which might never be
1077	1140	** sent) by beginning with the pull or push cards
1078	1141	*/
1079	1142	if( pullFlag ){
		@@ -1086,18 +1149,21 @@
1086	1149	}
1087	1150	go = 0;
1088	1151
1089	1152	/* Process the reply that came back from the server */
1090	1153	while( blob_line(&recv, &xfer.line) ){
	1154	+ if( g.fHttpTrace ){
	1155	+ printf("\rGOT: %.*s", blob_size(&xfer.line), blob_buffer(&xfer.line));
	1156	+ }
1091	1157	if( blob_buffer(&xfer.line)[0]=='#' ){
1092	1158	const char *zLine = blob_buffer(&xfer.line);
1093	1159	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1094	1160	char zTime[20];
1095	1161	double rDiff;
1096	1162	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1097		- rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1098		- zTime);
	1163	+ rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
	1164	+ zTime, rArrivalTime);
1099	1165	if( rDiff<0.0 ) rDiff = -rDiff;
1100	1166	if( rDiff>9e98 ) rDiff = 0.0;
1101	1167	if( (rDiff24.03600.0)>=60.0 ){
1102	1168	fossil_warning("* time skew * server time differs by %s",
1103	1169	db_timespan_name(rDiff));
		@@ -1107,21 +1173,25 @@
1107	1173	continue;
1108	1174	}
1109	1175	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1110	1176	nCardRcvd++;
1111	1177	if( !g.cgiOutput && !g.fQuiet ){
1112		- printf("\r%d", nCardRcvd);
1113		- fflush(stdout);
	1178	+ pctDone = (recv.iCursor*100)/recv.nUsed;
	1179	+ if( pctDone!=lastPctDone ){
	1180	+ printf("\rprocessed: %d%% ", pctDone);
	1181	+ lastPctDone = pctDone;
	1182	+ fflush(stdout);
	1183	+ }
1114	1184	}
1115	1185
1116	1186	/* file UUID SIZE \n CONTENT
1117	1187	** file UUID DELTASRC SIZE \n CONTENT
1118	1188	**
1119	1189	** Receive a file transmitted from the server.
1120	1190	*/
1121	1191	if( blob_eq(&xfer.aToken[0],"file") ){
1122		- xfer_accept_file(&xfer);
	1192	+ xfer_accept_file(&xfer, cloneFlag);
1123	1193	}else
1124	1194
1125	1195	/* gimme UUID
1126	1196	**
1127	1197	** Server is requesting a file. If the file is a manifest, assume
		@@ -1177,11 +1247,11 @@
1177	1247	}
1178	1248	if( zPCode==0 ){
1179	1249	zPCode = mprintf("%b", &xfer.aToken[2]);
1180	1250	db_set("project-code", zPCode, 0);
1181	1251	}
1182		- blob_appendf(&send, "clone\n");
	1252	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1183	1253	nCardSent++;
1184	1254	}else
1185	1255
1186	1256	/* config NAME SIZE \n CONTENT
1187	1257	**
		@@ -1236,10 +1306,21 @@
1236	1306	** same server.
1237	1307	*/
1238	1308	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1239	1309	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1240	1310	}else
	1311	+
	1312	+ /* clone_seqno N
	1313	+ **
	1314	+ ** When doing a clone, the server tries to send all of its artifacts
	1315	+ ** in sequence. This card indicates the sequence number of the next
	1316	+ ** blob that needs to be sent. If N<=0 that indicates that all blobs
	1317	+ ** have been sent.
	1318	+ */
	1319	+ if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
	1320	+ blob_is_int(&xfer.aToken[1], &cloneSeqno);
	1321	+ }else
1241	1322
1242	1323	/* message MESSAGE
1243	1324	**
1244	1325	** Print a message. Similar to "error" but does not stop processing.
1245	1326	**
		@@ -1314,10 +1395,12 @@
1314	1395	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1315	1396	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1316	1397	go = 1;
1317	1398	mxPhantomReq = nFileRecv*2;
1318	1399	if( mxPhantomReq<200 ) mxPhantomReq = 200;
	1400	+ }else if( cloneFlag && nFileRecv>0 ){
	1401	+ go = 1;
1319	1402	}
1320	1403	nCardRcvd = 0;
1321	1404	xfer.nFileRcvd = 0;
1322	1405	xfer.nDeltaRcvd = 0;
1323	1406	xfer.nDanglingFile = 0;
		@@ -1329,15 +1412,19 @@
1329	1412	go = 1;
1330	1413	}
1331	1414
1332	1415	/* If this is a clone, the go at least two rounds */
1333	1416	if( cloneFlag && nCycle==1 ) go = 1;
	1417	+
	1418	+ /* Stop the cycle if the server sends a "clone_seqno 0" card */
	1419	+ if( cloneSeqno<=0 ) go = 0;
1334	1420	};
1335	1421	transport_stats(&nSent, &nRcvd, 1);
1336	1422	fossil_print("Total network traffic: %d bytes sent, %d bytes received\n",
1337	1423	nSent, nRcvd);
1338	1424	transport_close();
1339	1425	transport_global_shutdown();
1340	1426	db_multi_exec("DROP TABLE onremote");
1341	1427	manifest_crosslink_end();
	1428	+ content_enable_dephantomize(1);
1342	1429	db_end_transaction(0);
1343	1430	}
1344	1431

	--- src/xfer.c
	+++ src/xfer.c
	@@ -72,11 +72,17 @@
72	/*
73	** Remember that the other side of the connection already has a copy
74	** of the file rid.
75	*/
76	static void remote_has(int rid){
77	if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);






78	}
79
80	/*
81	** The aToken[0..nToken-1] blob array is a parse of a "file" line
82	** message. This routine finishes parsing that message and does
	@@ -95,11 +101,11 @@
95	** be initialized to an empty string.
96	**
97	** Any artifact successfully received by this routine is considered to
98	** be public and is therefore removed from the "private" table.
99	*/
100	static void xfer_accept_file(Xfer *pXfer){
101	int n;
102	int rid;
103	int srcid = 0;
104	Blob content, hash;
105
	@@ -114,13 +120,26 @@
114	return;
115	}
116	blob_zero(&content);
117	blob_zero(&hash);
118	blob_extract(pXfer->pIn, n, &content);
119	if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
120	/* Ignore files that have been shunned */
121	return;













122	}
123	if( pXfer->nToken==4 ){
124	Blob src, next;
125	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
126	if( content_get(srcid, &src)==0 ){
	@@ -467,14 +486,15 @@
467	** Check to see if the number of unclustered entries is greater than
468	** 100 and if it is, form a new cluster. Unclustered phantoms do not
469	** count toward the 100 total. And phantoms are never added to a new
470	** cluster.
471	*/
472	static void create_cluster(void){
473	Blob cluster, cksum;
474	Stmt q;
475	int nUncl;

476
477	/* We should not ever get any private artifacts in the unclustered table.
478	** But if we do (because of a bug) now is a good time to delete them. */
479	db_multi_exec(
480	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -481,30 +501,41 @@
481	);
482
483	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
484	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
485	" WHERE rid=unclustered.rid)");
486	if( nUncl<100 ){
487	return;
488	}
489	blob_zero(&cluster);
490	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
491	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
492	" WHERE rid=unclustered.rid)"
493	" AND unclustered.rid=blob.rid"
494	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
495	" ORDER BY 1");
496	while( db_step(&q)==SQLITE_ROW ){
497	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
498	}
499	db_finalize(&q);
500	md5sum_blob(&cluster, &cksum);
501	blob_appendf(&cluster, "Z %b\n", &cksum);
502	blob_reset(&cksum);
503	db_multi_exec("DELETE FROM unclustered");
504	content_put(&cluster, 0, 0);
505	blob_reset(&cluster);











506	}
507
508	/*
509	** Send an igot message for every entry in unclustered table.
510	** Return the number of cards sent.
	@@ -606,19 +637,17 @@
606	blobarray_zero(xfer.aToken, count(xfer.aToken));
607	cgi_set_content_type(g.zContentType);
608	blob_zero(&xfer.err);
609	xfer.pIn = &g.cgiIn;
610	xfer.pOut = cgi_output_blob();
611	xfer.mxSend = db_get_int("max-download", 5000000);
612	g.xferPanic = 1;
613
614	db_begin_transaction();
615	db_multi_exec(
616	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
617	);
618	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
619	@ # timestamp %s(zNow)
620	manifest_crosslink_begin();
621	while( blob_line(xfer.pIn, &xfer.line) ){
622	if( blob_buffer(&xfer.line)[0]=='#' ) continue;
623	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
624
	@@ -632,11 +661,11 @@
632	cgi_reset_content();
633	@ error not\sauthorized\sto\swrite
634	nErr++;
635	break;
636	}
637	xfer_accept_file(&xfer);
638	if( blob_size(&xfer.err) ){
639	cgi_reset_content();
640	@ error %T(blob_str(&xfer.err))
641	nErr++;
642	break;
	@@ -718,26 +747,42 @@
718	isPush = 1;
719	}
720	}
721	}else
722
723	/* clone
724	**
725	** The client knows nothing. Tell all.
726	*/
727	if( blob_eq(&xfer.aToken[0], "clone") ){

728	login_check_credentials();
729	if( !g.okClone ){
730	cgi_reset_content();
731	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
732	@ error not\sauthorized\sto\sclone
733	nErr++;
734	break;
735	}
736	isClone = 1;
737	isPull = 1;
738	deltaFlag = 1;















739	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
740	}else
741
742	/* login USER NONCE SIGNATURE
743	**
	@@ -874,10 +919,18 @@
874	}
875	if( recvConfig ){
876	configure_finalize_receive();
877	}
878	manifest_crosslink_end();








879	db_end_transaction(0);
880	}
881
882	/*
883	** COMMAND: test-xfer
	@@ -943,13 +996,17 @@
943	int origConfigRcvMask; /* Original value of configRcvMask */
944	int nFileRecv; /* Number of files received */
945	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
946	const char zCookie; / Server cookie */
947	int nSent, nRcvd; /* Bytes sent and received (after compression) */

948	Blob send; /* Text we are sending to the server */
949	Blob recv; /* Reply we got back from the server */
950	Xfer xfer; /* Transfer data */



951	const char *zSCode = db_get("server-code", "x");
952	const char *zPCode = db_get("project-code", 0);
953
954	if( db_get_boolean("dont-push", 0) ) pushFlag = 0;
955	if( pushFlag + pullFlag + cloneFlag == 0
	@@ -977,15 +1034,16 @@
977
978	/*
979	** Always begin with a clone, pull, or push message
980	*/
981	if( cloneFlag ){
982	blob_appendf(&send, "clone\n");
983	pushFlag = 0;
984	pullFlag = 0;
985	nCardSent++;
986	/* TBD: Request all transferable configuration values */

987	}else if( pullFlag ){
988	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
989	nCardSent++;
990	}
991	if( pushFlag ){
	@@ -1009,11 +1067,11 @@
1009	}
1010
1011	/* Generate gimme cards for phantoms and leaf cards
1012	** for all leaves.
1013	*/
1014	if( pullFlag \|\| cloneFlag ){
1015	request_phantoms(&xfer, mxPhantomReq);
1016	}
1017	if( pushFlag ){
1018	send_unsent(&xfer);
1019	nCardSent += send_unclustered(&xfer);
	@@ -1058,22 +1116,27 @@
1058	blob_appendf(&send, "# %s\n", zRandomness);
1059	free(zRandomness);
1060
1061	/* Exchange messages with the server */
1062	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1063	fossil_print(zValueFormat, "Send:",
1064	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1065	xfer.nFileSent, xfer.nDeltaSent);
1066	nCardSent = 0;
1067	nCardRcvd = 0;
1068	xfer.nFileSent = 0;
1069	xfer.nDeltaSent = 0;
1070	xfer.nGimmeSent = 0;
1071	xfer.nIGotSent = 0;



1072	fflush(stdout);
1073	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);

1074	blob_reset(&send);

1075
1076	/* Begin constructing the next message (which might never be
1077	** sent) by beginning with the pull or push cards
1078	*/
1079	if( pullFlag ){
	@@ -1086,18 +1149,21 @@
1086	}
1087	go = 0;
1088
1089	/* Process the reply that came back from the server */
1090	while( blob_line(&recv, &xfer.line) ){



1091	if( blob_buffer(&xfer.line)[0]=='#' ){
1092	const char *zLine = blob_buffer(&xfer.line);
1093	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1094	char zTime[20];
1095	double rDiff;
1096	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1097	rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1098	zTime);
1099	if( rDiff<0.0 ) rDiff = -rDiff;
1100	if( rDiff>9e98 ) rDiff = 0.0;
1101	if( (rDiff24.03600.0)>=60.0 ){
1102	fossil_warning("* time skew * server time differs by %s",
1103	db_timespan_name(rDiff));
	@@ -1107,21 +1173,25 @@
1107	continue;
1108	}
1109	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1110	nCardRcvd++;
1111	if( !g.cgiOutput && !g.fQuiet ){
1112	printf("\r%d", nCardRcvd);
1113	fflush(stdout);




1114	}
1115
1116	/* file UUID SIZE \n CONTENT
1117	** file UUID DELTASRC SIZE \n CONTENT
1118	**
1119	** Receive a file transmitted from the server.
1120	*/
1121	if( blob_eq(&xfer.aToken[0],"file") ){
1122	xfer_accept_file(&xfer);
1123	}else
1124
1125	/* gimme UUID
1126	**
1127	** Server is requesting a file. If the file is a manifest, assume
	@@ -1177,11 +1247,11 @@
1177	}
1178	if( zPCode==0 ){
1179	zPCode = mprintf("%b", &xfer.aToken[2]);
1180	db_set("project-code", zPCode, 0);
1181	}
1182	blob_appendf(&send, "clone\n");
1183	nCardSent++;
1184	}else
1185
1186	/* config NAME SIZE \n CONTENT
1187	**
	@@ -1236,10 +1306,21 @@
1236	** same server.
1237	*/
1238	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1239	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1240	}else











1241
1242	/* message MESSAGE
1243	**
1244	** Print a message. Similar to "error" but does not stop processing.
1245	**
	@@ -1314,10 +1395,12 @@
1314	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1315	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1316	go = 1;
1317	mxPhantomReq = nFileRecv*2;
1318	if( mxPhantomReq<200 ) mxPhantomReq = 200;


1319	}
1320	nCardRcvd = 0;
1321	xfer.nFileRcvd = 0;
1322	xfer.nDeltaRcvd = 0;
1323	xfer.nDanglingFile = 0;
	@@ -1329,15 +1412,19 @@
1329	go = 1;
1330	}
1331
1332	/* If this is a clone, the go at least two rounds */
1333	if( cloneFlag && nCycle==1 ) go = 1;



1334	};
1335	transport_stats(&nSent, &nRcvd, 1);
1336	fossil_print("Total network traffic: %d bytes sent, %d bytes received\n",
1337	nSent, nRcvd);
1338	transport_close();
1339	transport_global_shutdown();
1340	db_multi_exec("DROP TABLE onremote");
1341	manifest_crosslink_end();

1342	db_end_transaction(0);
1343	}
1344

	--- src/xfer.c
	+++ src/xfer.c
	@@ -72,11 +72,17 @@
72	/*
73	** Remember that the other side of the connection already has a copy
74	** of the file rid.
75	*/
76	static void remote_has(int rid){
77	if( rid ){
78	static Stmt q;
79	db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
80	db_bind_int(&q, ":r", rid);
81	db_step(&q);
82	db_reset(&q);
83	}
84	}
85
86	/*
87	** The aToken[0..nToken-1] blob array is a parse of a "file" line
88	** message. This routine finishes parsing that message and does
	@@ -95,11 +101,11 @@
101	** be initialized to an empty string.
102	**
103	** Any artifact successfully received by this routine is considered to
104	** be public and is therefore removed from the "private" table.
105	*/
106	static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
107	int n;
108	int rid;
109	int srcid = 0;
110	Blob content, hash;
111
	@@ -114,13 +120,26 @@
120	return;
121	}
122	blob_zero(&content);
123	blob_zero(&hash);
124	blob_extract(pXfer->pIn, n, &content);
125	if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
126	/* Ignore files that have been shunned */
127	return;
128	}
129	if( cloneFlag ){
130	if( pXfer->nToken==4 ){
131	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
132	pXfer->nDeltaRcvd++;
133	}else{
134	srcid = 0;
135	pXfer->nFileRcvd++;
136	}
137	rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
138	remote_has(rid);
139	blob_reset(&content);
140	return;
141	}
142	if( pXfer->nToken==4 ){
143	Blob src, next;
144	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
145	if( content_get(srcid, &src)==0 ){
	@@ -467,14 +486,15 @@
486	** Check to see if the number of unclustered entries is greater than
487	** 100 and if it is, form a new cluster. Unclustered phantoms do not
488	** count toward the 100 total. And phantoms are never added to a new
489	** cluster.
490	*/
491	void create_cluster(void){
492	Blob cluster, cksum;
493	Stmt q;
494	int nUncl;
495	int nRow = 0;
496
497	/* We should not ever get any private artifacts in the unclustered table.
498	** But if we do (because of a bug) now is a good time to delete them. */
499	db_multi_exec(
500	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -481,30 +501,41 @@
501	);
502
503	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
504	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
505	" WHERE rid=unclustered.rid)");
506	if( nUncl>=100 ){
507	blob_zero(&cluster);
508	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
509	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
510	" WHERE rid=unclustered.rid)"
511	" AND unclustered.rid=blob.rid"
512	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
513	" ORDER BY 1");
514	while( db_step(&q)==SQLITE_ROW ){
515	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
516	nRow++;
517	if( nRow>=800 && nUncl>nRow+100 ){
518	md5sum_blob(&cluster, &cksum);
519	blob_appendf(&cluster, "Z %b\n", &cksum);
520	blob_reset(&cksum);
521	content_put(&cluster, 0, 0);
522	blob_reset(&cluster);
523	nUncl -= nRow;
524	nRow = 0;
525	}
526	}
527	db_finalize(&q);
528	db_multi_exec("DELETE FROM unclustered");
529	if( nRow>0 ){
530	md5sum_blob(&cluster, &cksum);
531	blob_appendf(&cluster, "Z %b\n", &cksum);
532	blob_reset(&cksum);
533	content_put(&cluster, 0, 0);
534	blob_reset(&cluster);
535	}
536	}
537	}
538
539	/*
540	** Send an igot message for every entry in unclustered table.
541	** Return the number of cards sent.
	@@ -606,19 +637,17 @@
637	blobarray_zero(xfer.aToken, count(xfer.aToken));
638	cgi_set_content_type(g.zContentType);
639	blob_zero(&xfer.err);
640	xfer.pIn = &g.cgiIn;
641	xfer.pOut = cgi_output_blob();
642	xfer.mxSend = db_get_int("max-download", 20000000);
643	g.xferPanic = 1;
644
645	db_begin_transaction();
646	db_multi_exec(
647	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
648	);


649	manifest_crosslink_begin();
650	while( blob_line(xfer.pIn, &xfer.line) ){
651	if( blob_buffer(&xfer.line)[0]=='#' ) continue;
652	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
653
	@@ -632,11 +661,11 @@
661	cgi_reset_content();
662	@ error not\sauthorized\sto\swrite
663	nErr++;
664	break;
665	}
666	xfer_accept_file(&xfer, 0);
667	if( blob_size(&xfer.err) ){
668	cgi_reset_content();
669	@ error %T(blob_str(&xfer.err))
670	nErr++;
671	break;
	@@ -718,26 +747,42 @@
747	isPush = 1;
748	}
749	}
750	}else
751
752	/* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
753	**
754	** The client knows nothing. Tell all.
755	*/
756	if( blob_eq(&xfer.aToken[0], "clone") ){
757	int iVers;
758	login_check_credentials();
759	if( !g.okClone ){
760	cgi_reset_content();
761	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
762	@ error not\sauthorized\sto\sclone
763	nErr++;
764	break;
765	}
766	if( xfer.nToken==3
767	&& blob_is_int(&xfer.aToken[1], &iVers)
768	&& iVers>=2
769	){
770	int seqno, max;
771	blob_is_int(&xfer.aToken[2], &seqno);
772	max = db_int(0, "SELECT max(rid) FROM blob");
773	while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
774	send_file(&xfer, seqno, 0, 1);
775	seqno++;
776	}
777	if( seqno>=max ) seqno = 0;
778	@ clone_seqno %d(seqno)
779	}else{
780	isClone = 1;
781	isPull = 1;
782	deltaFlag = 1;
783	}
784	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
785	}else
786
787	/* login USER NONCE SIGNATURE
788	**
	@@ -874,10 +919,18 @@
919	}
920	if( recvConfig ){
921	configure_finalize_receive();
922	}
923	manifest_crosslink_end();
924
925	/* Send the server timestamp last, in case prior processing happened
926	** to use up a significant fraction of our time window.
927	*/
928	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
929	@ # timestamp %s(zNow)
930	free(zNow);
931
932	db_end_transaction(0);
933	}
934
935	/*
936	** COMMAND: test-xfer
	@@ -943,13 +996,17 @@
996	int origConfigRcvMask; /* Original value of configRcvMask */
997	int nFileRecv; /* Number of files received */
998	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
999	const char zCookie; / Server cookie */
1000	int nSent, nRcvd; /* Bytes sent and received (after compression) */
1001	int cloneSeqno = 1; /* Sequence number for clones */
1002	Blob send; /* Text we are sending to the server */
1003	Blob recv; /* Reply we got back from the server */
1004	Xfer xfer; /* Transfer data */
1005	int pctDone; /* Percentage done with a message */
1006	int lastPctDone = -1; /* Last displayed pctDone */
1007	double rArrivalTime; /* Time at which a message arrived */
1008	const char *zSCode = db_get("server-code", "x");
1009	const char *zPCode = db_get("project-code", 0);
1010
1011	if( db_get_boolean("dont-push", 0) ) pushFlag = 0;
1012	if( pushFlag + pullFlag + cloneFlag == 0
	@@ -977,15 +1034,16 @@
1034
1035	/*
1036	** Always begin with a clone, pull, or push message
1037	*/
1038	if( cloneFlag ){
1039	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1040	pushFlag = 0;
1041	pullFlag = 0;
1042	nCardSent++;
1043	/* TBD: Request all transferable configuration values */
1044	content_enable_dephantomize(0);
1045	}else if( pullFlag ){
1046	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1047	nCardSent++;
1048	}
1049	if( pushFlag ){
	@@ -1009,11 +1067,11 @@
1067	}
1068
1069	/* Generate gimme cards for phantoms and leaf cards
1070	** for all leaves.
1071	*/
1072	if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1073	request_phantoms(&xfer, mxPhantomReq);
1074	}
1075	if( pushFlag ){
1076	send_unsent(&xfer);
1077	nCardSent += send_unclustered(&xfer);
	@@ -1058,22 +1116,27 @@
1116	blob_appendf(&send, "# %s\n", zRandomness);
1117	free(zRandomness);
1118
1119	/* Exchange messages with the server */
1120	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1121	fossil_print(zValueFormat, "Sent:",
1122	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1123	xfer.nFileSent, xfer.nDeltaSent);
1124	nCardSent = 0;
1125	nCardRcvd = 0;
1126	xfer.nFileSent = 0;
1127	xfer.nDeltaSent = 0;
1128	xfer.nGimmeSent = 0;
1129	xfer.nIGotSent = 0;
1130	if( !g.cgiOutput && !g.fQuiet ){
1131	printf("waiting for server...");
1132	}
1133	fflush(stdout);
1134	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
1135	lastPctDone = -1;
1136	blob_reset(&send);
1137	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1138
1139	/* Begin constructing the next message (which might never be
1140	** sent) by beginning with the pull or push cards
1141	*/
1142	if( pullFlag ){
	@@ -1086,18 +1149,21 @@
1149	}
1150	go = 0;
1151
1152	/* Process the reply that came back from the server */
1153	while( blob_line(&recv, &xfer.line) ){
1154	if( g.fHttpTrace ){
1155	printf("\rGOT: %.*s", blob_size(&xfer.line), blob_buffer(&xfer.line));
1156	}
1157	if( blob_buffer(&xfer.line)[0]=='#' ){
1158	const char *zLine = blob_buffer(&xfer.line);
1159	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1160	char zTime[20];
1161	double rDiff;
1162	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1163	rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
1164	zTime, rArrivalTime);
1165	if( rDiff<0.0 ) rDiff = -rDiff;
1166	if( rDiff>9e98 ) rDiff = 0.0;
1167	if( (rDiff24.03600.0)>=60.0 ){
1168	fossil_warning("* time skew * server time differs by %s",
1169	db_timespan_name(rDiff));
	@@ -1107,21 +1173,25 @@
1173	continue;
1174	}
1175	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1176	nCardRcvd++;
1177	if( !g.cgiOutput && !g.fQuiet ){
1178	pctDone = (recv.iCursor*100)/recv.nUsed;
1179	if( pctDone!=lastPctDone ){
1180	printf("\rprocessed: %d%% ", pctDone);
1181	lastPctDone = pctDone;
1182	fflush(stdout);
1183	}
1184	}
1185
1186	/* file UUID SIZE \n CONTENT
1187	** file UUID DELTASRC SIZE \n CONTENT
1188	**
1189	** Receive a file transmitted from the server.
1190	*/
1191	if( blob_eq(&xfer.aToken[0],"file") ){
1192	xfer_accept_file(&xfer, cloneFlag);
1193	}else
1194
1195	/* gimme UUID
1196	**
1197	** Server is requesting a file. If the file is a manifest, assume
	@@ -1177,11 +1247,11 @@
1247	}
1248	if( zPCode==0 ){
1249	zPCode = mprintf("%b", &xfer.aToken[2]);
1250	db_set("project-code", zPCode, 0);
1251	}
1252	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1253	nCardSent++;
1254	}else
1255
1256	/* config NAME SIZE \n CONTENT
1257	**
	@@ -1236,10 +1306,21 @@
1306	** same server.
1307	*/
1308	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1309	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1310	}else
1311
1312	/* clone_seqno N
1313	**
1314	** When doing a clone, the server tries to send all of its artifacts
1315	** in sequence. This card indicates the sequence number of the next
1316	** blob that needs to be sent. If N<=0 that indicates that all blobs
1317	** have been sent.
1318	*/
1319	if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
1320	blob_is_int(&xfer.aToken[1], &cloneSeqno);
1321	}else
1322
1323	/* message MESSAGE
1324	**
1325	** Print a message. Similar to "error" but does not stop processing.
1326	**
	@@ -1314,10 +1395,12 @@
1395	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1396	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1397	go = 1;
1398	mxPhantomReq = nFileRecv*2;
1399	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1400	}else if( cloneFlag && nFileRecv>0 ){
1401	go = 1;
1402	}
1403	nCardRcvd = 0;
1404	xfer.nFileRcvd = 0;
1405	xfer.nDeltaRcvd = 0;
1406	xfer.nDanglingFile = 0;
	@@ -1329,15 +1412,19 @@
1412	go = 1;
1413	}
1414
1415	/* If this is a clone, the go at least two rounds */
1416	if( cloneFlag && nCycle==1 ) go = 1;
1417
1418	/* Stop the cycle if the server sends a "clone_seqno 0" card */
1419	if( cloneSeqno<=0 ) go = 0;
1420	};
1421	transport_stats(&nSent, &nRcvd, 1);
1422	fossil_print("Total network traffic: %d bytes sent, %d bytes received\n",
1423	nSent, nRcvd);
1424	transport_close();
1425	transport_global_shutdown();
1426	db_multi_exec("DROP TABLE onremote");
1427	manifest_crosslink_end();
1428	content_enable_dephantomize(1);
1429	db_end_transaction(0);
1430	}
1431

Fossil SCM

Keyboard Shortcuts