Fossil SCM

merge from trunk

wolfgang 2010-10-28 17:41 StvPrivateHook2 merge

Commit e6dce6a16afe21887e104db5fe8e66984014450d

Parent 304c71a09cc05e2…

9 files changed +1 -1 +52 -1 +1 -1 +38 -8 +34 +34 +10 +129 -41 +129 -41

~ src/branch.c ~ src/content.c ~ src/http.c ~ src/manifest.c ~ src/rebuild.c ~ src/rebuild.c ~ src/schema.c ~ src/xfer.c ~ src/xfer.c

M src/branch.c

+1 -1

		--- src/branch.c
		+++ src/branch.c
		@@ -92,11 +92,11 @@
92	92	/* Copy all of the content from the parent into the branch */
93	93	for(i=0; i<pParent->nFile; ++i){
94	94	blob_appendf(&branch, "F %F", pParent->aFile[i].zName);
95	95	if( pParent->aFile[i].zUuid ){
96	96	blob_appendf(&branch, " %s", pParent->aFile[i].zUuid);
97		- if( pParent->aFile[i].zPerm[0] ){
	97	+ if( pParent->aFile[i].zPerm && pParent->aFile[i].zPerm[0] ){
98	98	blob_appendf(&branch, " %s", pParent->aFile[i].zPerm);
99	99	}
100	100	}
101	101	blob_append(&branch, "\n", 1);
102	102	}
103	103

	--- src/branch.c
	+++ src/branch.c
	@@ -92,11 +92,11 @@
92	/* Copy all of the content from the parent into the branch */
93	for(i=0; i<pParent->nFile; ++i){
94	blob_appendf(&branch, "F %F", pParent->aFile[i].zName);
95	if( pParent->aFile[i].zUuid ){
96	blob_appendf(&branch, " %s", pParent->aFile[i].zUuid);
97	if( pParent->aFile[i].zPerm[0] ){
98	blob_appendf(&branch, " %s", pParent->aFile[i].zPerm);
99	}
100	}
101	blob_append(&branch, "\n", 1);
102	}
103

	--- src/branch.c
	+++ src/branch.c
	@@ -92,11 +92,11 @@
92	/* Copy all of the content from the parent into the branch */
93	for(i=0; i<pParent->nFile; ++i){
94	blob_appendf(&branch, "F %F", pParent->aFile[i].zName);
95	if( pParent->aFile[i].zUuid ){
96	blob_appendf(&branch, " %s", pParent->aFile[i].zUuid);
97	if( pParent->aFile[i].zPerm && pParent->aFile[i].zPerm[0] ){
98	blob_appendf(&branch, " %s", pParent->aFile[i].zPerm);
99	}
100	}
101	blob_append(&branch, "\n", 1);
102	}
103

M src/content.c

+52 -1

		--- src/content.c
		+++ src/content.c
		@@ -325,31 +325,72 @@
325	325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	326	blob_uncompress(&content, &content);
327	327	blob_write_to_file(&content, zFile);
328	328	}
329	329
	330	+/*
	331	+** The following flag is set to disable the automatic calls to
	332	+** manifest_crosslink() when a record is dephantomized. This
	333	+** flag can be set (for example) when doing a clone when we know
	334	+** that rebuild will be run over all records at the conclusion
	335	+** of the operation.
	336	+*/
	337	+static int ignoreDephantomizations = 0;
	338	+
330	339	/*
331	340	** When a record is converted from a phantom to a real record,
332	341	** if that record has other records that are derived by delta,
333	342	** then call manifest_crosslink() on those other records.
	343	+**
	344	+** If the formerly phantom record or any of the other records
	345	+** derived by delta from the former phantom are a baseline manifest,
	346	+** then also invoke manifest_crosslink() on the delta-manifests
	347	+** associated with that baseline.
334	348	**
335	349	** Tail recursion is used to minimize stack depth.
336	350	*/
337	351	void after_dephantomize(int rid, int linkFlag){
338	352	Stmt q;
339	353	int nChildAlloc = 0;
340	354	int *aChild = 0;
	355	+ Blob content;
341	356
	357	+ if( ignoreDephantomizations ) return;
342	358	while( rid ){
343	359	int nChildUsed = 0;
344	360	int i;
	361	+
	362	+ /* Parse the object rid itself */
345	363	if( linkFlag ){
346		- Blob content;
347	364	content_get(rid, &content);
348	365	manifest_crosslink(rid, &content);
349	366	blob_reset(&content);
350	367	}
	368	+
	369	+ /* Parse all delta-manifests that depend on baseline-manifest rid */
	370	+ db_prepare(&q, "SELECT rid FROM orphan WHERE baseline=%d", rid);
	371	+ while( db_step(&q)==SQLITE_ROW ){
	372	+ int child = db_column_int(&q, 0);
	373	+ if( nChildUsed>=nChildAlloc ){
	374	+ nChildAlloc = nChildAlloc*2 + 10;
	375	+ aChild = fossil_realloc(aChild, nChildAlloc*sizeof(aChild));
	376	+ }
	377	+ aChild[nChildUsed++] = child;
	378	+ }
	379	+ db_finalize(&q);
	380	+ for(i=0; i<nChildUsed; i++){
	381	+ content_get(aChild[i], &content);
	382	+ manifest_crosslink(aChild[i], &content);
	383	+ blob_reset(&content);
	384	+ }
	385	+ if( nChildUsed ){
	386	+ db_multi_exec("DELETE FROM orphan WHERE baseline=%d", rid);
	387	+ }
	388	+
	389	+ /* Recursively dephantomize all artifacts that are derived by
	390	+ ** delta from artifact rid */
	391	+ nChildUsed = 0;
351	392	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
352	393	while( db_step(&q)==SQLITE_ROW ){
353	394	int child = db_column_int(&q, 0);
354	395	if( nChildUsed>=nChildAlloc ){
355	396	nChildAlloc = nChildAlloc*2 + 10;
		@@ -359,15 +400,25 @@
359	400	}
360	401	db_finalize(&q);
361	402	for(i=1; i<nChildUsed; i++){
362	403	after_dephantomize(aChild[i], 1);
363	404	}
	405	+
	406	+ /* Tail recursion for the common case where only a single artifact
	407	+ ** is derived by delta from rid... */
364	408	rid = nChildUsed>0 ? aChild[0] : 0;
365	409	linkFlag = 1;
366	410	}
367	411	free(aChild);
368	412	}
	413	+
	414	+/*
	415	+** Turn dephantomization processing on or off.
	416	+*/
	417	+void content_enable_dephantomize(int onoff){
	418	+ ignoreDephantomizations = !onoff;
	419	+}
369	420
370	421	/*
371	422	** Write content into the database. Return the record ID. If the
372	423	** content is already in the database, just return the record ID.
373	424	**
374	425

	--- src/content.c
	+++ src/content.c
	@@ -325,31 +325,72 @@
325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	blob_uncompress(&content, &content);
327	blob_write_to_file(&content, zFile);
328	}
329









330	/*
331	** When a record is converted from a phantom to a real record,
332	** if that record has other records that are derived by delta,
333	** then call manifest_crosslink() on those other records.





334	**
335	** Tail recursion is used to minimize stack depth.
336	*/
337	void after_dephantomize(int rid, int linkFlag){
338	Stmt q;
339	int nChildAlloc = 0;
340	int *aChild = 0;

341

342	while( rid ){
343	int nChildUsed = 0;
344	int i;


345	if( linkFlag ){
346	Blob content;
347	content_get(rid, &content);
348	manifest_crosslink(rid, &content);
349	blob_reset(&content);
350	}
























351	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
352	while( db_step(&q)==SQLITE_ROW ){
353	int child = db_column_int(&q, 0);
354	if( nChildUsed>=nChildAlloc ){
355	nChildAlloc = nChildAlloc*2 + 10;
	@@ -359,15 +400,25 @@
359	}
360	db_finalize(&q);
361	for(i=1; i<nChildUsed; i++){
362	after_dephantomize(aChild[i], 1);
363	}



364	rid = nChildUsed>0 ? aChild[0] : 0;
365	linkFlag = 1;
366	}
367	free(aChild);
368	}







369
370	/*
371	** Write content into the database. Return the record ID. If the
372	** content is already in the database, just return the record ID.
373	**
374

	--- src/content.c
	+++ src/content.c
	@@ -325,31 +325,72 @@
325	db_blob(&content, "SELECT content FROM blob WHERE rid=%d", rid);
326	blob_uncompress(&content, &content);
327	blob_write_to_file(&content, zFile);
328	}
329
330	/*
331	** The following flag is set to disable the automatic calls to
332	** manifest_crosslink() when a record is dephantomized. This
333	** flag can be set (for example) when doing a clone when we know
334	** that rebuild will be run over all records at the conclusion
335	** of the operation.
336	*/
337	static int ignoreDephantomizations = 0;
338
339	/*
340	** When a record is converted from a phantom to a real record,
341	** if that record has other records that are derived by delta,
342	** then call manifest_crosslink() on those other records.
343	**
344	** If the formerly phantom record or any of the other records
345	** derived by delta from the former phantom are a baseline manifest,
346	** then also invoke manifest_crosslink() on the delta-manifests
347	** associated with that baseline.
348	**
349	** Tail recursion is used to minimize stack depth.
350	*/
351	void after_dephantomize(int rid, int linkFlag){
352	Stmt q;
353	int nChildAlloc = 0;
354	int *aChild = 0;
355	Blob content;
356
357	if( ignoreDephantomizations ) return;
358	while( rid ){
359	int nChildUsed = 0;
360	int i;
361
362	/* Parse the object rid itself */
363	if( linkFlag ){

364	content_get(rid, &content);
365	manifest_crosslink(rid, &content);
366	blob_reset(&content);
367	}
368
369	/* Parse all delta-manifests that depend on baseline-manifest rid */
370	db_prepare(&q, "SELECT rid FROM orphan WHERE baseline=%d", rid);
371	while( db_step(&q)==SQLITE_ROW ){
372	int child = db_column_int(&q, 0);
373	if( nChildUsed>=nChildAlloc ){
374	nChildAlloc = nChildAlloc*2 + 10;
375	aChild = fossil_realloc(aChild, nChildAlloc*sizeof(aChild));
376	}
377	aChild[nChildUsed++] = child;
378	}
379	db_finalize(&q);
380	for(i=0; i<nChildUsed; i++){
381	content_get(aChild[i], &content);
382	manifest_crosslink(aChild[i], &content);
383	blob_reset(&content);
384	}
385	if( nChildUsed ){
386	db_multi_exec("DELETE FROM orphan WHERE baseline=%d", rid);
387	}
388
389	/* Recursively dephantomize all artifacts that are derived by
390	** delta from artifact rid */
391	nChildUsed = 0;
392	db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
393	while( db_step(&q)==SQLITE_ROW ){
394	int child = db_column_int(&q, 0);
395	if( nChildUsed>=nChildAlloc ){
396	nChildAlloc = nChildAlloc*2 + 10;
	@@ -359,15 +400,25 @@
400	}
401	db_finalize(&q);
402	for(i=1; i<nChildUsed; i++){
403	after_dephantomize(aChild[i], 1);
404	}
405
406	/* Tail recursion for the common case where only a single artifact
407	** is derived by delta from rid... */
408	rid = nChildUsed>0 ? aChild[0] : 0;
409	linkFlag = 1;
410	}
411	free(aChild);
412	}
413
414	/*
415	** Turn dephantomization processing on or off.
416	*/
417	void content_enable_dephantomize(int onoff){
418	ignoreDephantomizations = !onoff;
419	}
420
421	/*
422	** Write content into the database. Return the record ID. If the
423	** content is already in the database, just return the record ID.
424	**
425

M src/http.c

+1 -1

		--- src/http.c
		+++ src/http.c
		@@ -267,11 +267,11 @@
267	267	}
268	268	z[j] = 0;
269	269	fossil_fatal("server sends error: %s", z);
270	270	}
271	271	if( g.fHttpTrace ){
272		- printf("HTTP RECEIVE:\n%s\n=======================\n", blob_str(pReply));
	272	+ /printf("HTTP RECEIVE:\n%s\n=======================\n",blob_str(pReply));/
273	273	}else{
274	274	blob_uncompress(pReply, pReply);
275	275	}
276	276
277	277	/*
278	278

	--- src/http.c
	+++ src/http.c
	@@ -267,11 +267,11 @@
267	}
268	z[j] = 0;
269	fossil_fatal("server sends error: %s", z);
270	}
271	if( g.fHttpTrace ){
272	printf("HTTP RECEIVE:\n%s\n=======================\n", blob_str(pReply));
273	}else{
274	blob_uncompress(pReply, pReply);
275	}
276
277	/*
278

	--- src/http.c
	+++ src/http.c
	@@ -267,11 +267,11 @@
267	}
268	z[j] = 0;
269	fossil_fatal("server sends error: %s", z);
270	}
271	if( g.fHttpTrace ){
272	/printf("HTTP RECEIVE:\n%s\n=======================\n",blob_str(pReply));/
273	}else{
274	blob_uncompress(pReply, pReply);
275	}
276
277	/*
278

M src/manifest.c

+38 -8

		--- src/manifest.c
		+++ src/manifest.c
		@@ -314,11 +314,11 @@
314	314	char cPrevType = 0;
315	315	char cType;
316	316	char *z;
317	317	int n;
318	318	char *zUuid;
319		- int sz;
	319	+ int sz = 0;
320	320
321	321	/* Every control artifact ends with a '\n' character. Exit early
322	322	** if that is not the case for this artifact.
323	323	*/
324	324	z = blob_buffer(pContent);
		@@ -370,11 +370,11 @@
370	370	** is omitted to delete an attachment. <target> is the name of
371	371	** a wiki page or ticket to which that attachment is connected.
372	372	*/
373	373	case 'A': {
374	374	char zName, zTarget, *zSrc;
375		- int nTarget, nSrc;
	375	+ int nTarget = 0, nSrc = 0;
376	376	zName = next_token(&x, 0);
377	377	zTarget = next_token(&x, &nTarget);
378	378	zSrc = next_token(&x, &nSrc);
379	379	if( zName==0 \|\| zTarget==0 ) goto manifest_syntax_error;
380	380	if( p->zAttachName!=0 ) goto manifest_syntax_error;
		@@ -913,28 +913,47 @@
913	913	}
914	914	}
915	915
916	916	/*
917	917	** Fetch the baseline associated with the delta-manifest p.
918		-** Print a fatal-error and quit if unable to load the baseline.
	918	+** Return 0 on success. If unable to parse the baseline,
	919	+** throw an error. If the baseline is a manifest, throw an
	920	+** error if throwError is true, or record that p is an orphan
	921	+** and return 1 throwError is false.
919	922	*/
920		-static void fetch_baseline(Manifest *p){
	923	+static int fetch_baseline(Manifest *p, int throwError){
921	924	if( p->zBaseline!=0 && p->pBaseline==0 ){
922	925	int rid = uuid_to_rid(p->zBaseline, 0);
	926	+ if( rid==0 && !throwError ){
	927	+ rid = content_new(p->zBaseline);
	928	+ db_multi_exec(
	929	+ "INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
	930	+ rid, p->rid
	931	+ );
	932	+ return 1;
	933	+ }
923	934	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
924	935	if( p->pBaseline==0 ){
	936	+ if( !throwError && db_exists("SELECT 1 FROM phantom WHERE rid=%d",rid) ){
	937	+ db_multi_exec(
	938	+ "INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
	939	+ rid, p->rid
	940	+ );
	941	+ return 1;
	942	+ }
925	943	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
926	944	}
927	945	}
	946	+ return 0;
928	947	}
929	948
930	949	/*
931	950	** Rewind a manifest-file iterator back to the beginning of the manifest.
932	951	*/
933	952	void manifest_file_rewind(Manifest *p){
934	953	p->iFile = 0;
935		- fetch_baseline(p);
	954	+ fetch_baseline(p, 1);
936	955	if( p->pBaseline ){
937	956	p->pBaseline->iFile = 0;
938	957	}
939	958	}
940	959
		@@ -1123,11 +1142,11 @@
1123	1142	ManifestFile *pFile;
1124	1143
1125	1144	pFile = manifest_file_seek_base(p, zName);
1126	1145	if( pFile && pFile->zUuid==0 ) return 0;
1127	1146	if( pFile==0 && p->zBaseline ){
1128		- fetch_baseline(p);
	1147	+ fetch_baseline(p, 1);
1129	1148	pFile = manifest_file_seek_base(p->pBaseline, zName);
1130	1149	}
1131	1150	return pFile;
1132	1151	}
1133	1152
		@@ -1181,15 +1200,18 @@
1181	1200	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1182	1201	content_get(otherRid, &otherContent);
1183	1202	if( blob_size(&otherContent)==0 ) return;
1184	1203	*ppOther = manifest_parse(&otherContent, otherRid);
1185	1204	if( *ppOther==0 ) return;
	1205	+ }
	1206	+ if( fetch_baseline(pParent, 0) \|\| fetch_baseline(pChild, 0) ){
	1207	+ manifest_destroy(*ppOther);
	1208	+ return;
1186	1209	}
1187	1210	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1188	1211	content_deltify(pid, cid, 0);
1189	1212	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){
1190		- fetch_baseline(pParent);
1191	1213	content_deltify(pParent->pBaseline->rid, cid, 0);
1192	1214	}
1193	1215
1194	1216	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1195	1217	if( pChildFile->zPrior ){
		@@ -1370,10 +1392,14 @@
1370	1392	return 0;
1371	1393	}
1372	1394	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1373	1395	manifest_destroy(p);
1374	1396	return 0;
	1397	+ }
	1398	+ if( p->type==CFTYPE_MANIFEST && fetch_baseline(p, 0) ){
	1399	+ manifest_destroy(p);
	1400	+ return 0;
1375	1401	}
1376	1402	db_begin_transaction();
1377	1403	if( p->type==CFTYPE_MANIFEST ){
1378	1404	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1379	1405	char *zCom;
		@@ -1427,16 +1453,20 @@
1427	1453	}
1428	1454	}
1429	1455	}
1430	1456	}
1431	1457	if( p->type==CFTYPE_CLUSTER ){
	1458	+ static Stmt del1;
1432	1459	tag_insert("cluster", 1, 0, rid, p->rDate, rid);
	1460	+ db_static_prepare(&del1, "DELETE FROM unclustered WHERE rid=:rid");
1433	1461	for(i=0; i<p->nCChild; i++){
1434	1462	int mid;
1435	1463	mid = uuid_to_rid(p->azCChild[i], 1);
1436	1464	if( mid>0 ){
1437		- db_multi_exec("DELETE FROM unclustered WHERE rid=%d", mid);
	1465	+ db_bind_int(&del1, ":rid", mid);
	1466	+ db_step(&del1);
	1467	+ db_reset(&del1);
1438	1468	}
1439	1469	}
1440	1470	}
1441	1471	if( p->type==CFTYPE_CONTROL
1442	1472	\|\| p->type==CFTYPE_MANIFEST
1443	1473

	--- src/manifest.c
	+++ src/manifest.c
	@@ -314,11 +314,11 @@
314	char cPrevType = 0;
315	char cType;
316	char *z;
317	int n;
318	char *zUuid;
319	int sz;
320
321	/* Every control artifact ends with a '\n' character. Exit early
322	** if that is not the case for this artifact.
323	*/
324	z = blob_buffer(pContent);
	@@ -370,11 +370,11 @@
370	** is omitted to delete an attachment. <target> is the name of
371	** a wiki page or ticket to which that attachment is connected.
372	*/
373	case 'A': {
374	char zName, zTarget, *zSrc;
375	int nTarget, nSrc;
376	zName = next_token(&x, 0);
377	zTarget = next_token(&x, &nTarget);
378	zSrc = next_token(&x, &nSrc);
379	if( zName==0 \|\| zTarget==0 ) goto manifest_syntax_error;
380	if( p->zAttachName!=0 ) goto manifest_syntax_error;
	@@ -913,28 +913,47 @@
913	}
914	}
915
916	/*
917	** Fetch the baseline associated with the delta-manifest p.
918	** Print a fatal-error and quit if unable to load the baseline.



919	*/
920	static void fetch_baseline(Manifest *p){
921	if( p->zBaseline!=0 && p->pBaseline==0 ){
922	int rid = uuid_to_rid(p->zBaseline, 0);








923	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
924	if( p->pBaseline==0 ){







925	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
926	}
927	}

928	}
929
930	/*
931	** Rewind a manifest-file iterator back to the beginning of the manifest.
932	*/
933	void manifest_file_rewind(Manifest *p){
934	p->iFile = 0;
935	fetch_baseline(p);
936	if( p->pBaseline ){
937	p->pBaseline->iFile = 0;
938	}
939	}
940
	@@ -1123,11 +1142,11 @@
1123	ManifestFile *pFile;
1124
1125	pFile = manifest_file_seek_base(p, zName);
1126	if( pFile && pFile->zUuid==0 ) return 0;
1127	if( pFile==0 && p->zBaseline ){
1128	fetch_baseline(p);
1129	pFile = manifest_file_seek_base(p->pBaseline, zName);
1130	}
1131	return pFile;
1132	}
1133
	@@ -1181,15 +1200,18 @@
1181	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1182	content_get(otherRid, &otherContent);
1183	if( blob_size(&otherContent)==0 ) return;
1184	*ppOther = manifest_parse(&otherContent, otherRid);
1185	if( *ppOther==0 ) return;




1186	}
1187	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1188	content_deltify(pid, cid, 0);
1189	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){
1190	fetch_baseline(pParent);
1191	content_deltify(pParent->pBaseline->rid, cid, 0);
1192	}
1193
1194	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1195	if( pChildFile->zPrior ){
	@@ -1370,10 +1392,14 @@
1370	return 0;
1371	}
1372	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1373	manifest_destroy(p);
1374	return 0;




1375	}
1376	db_begin_transaction();
1377	if( p->type==CFTYPE_MANIFEST ){
1378	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1379	char *zCom;
	@@ -1427,16 +1453,20 @@
1427	}
1428	}
1429	}
1430	}
1431	if( p->type==CFTYPE_CLUSTER ){

1432	tag_insert("cluster", 1, 0, rid, p->rDate, rid);

1433	for(i=0; i<p->nCChild; i++){
1434	int mid;
1435	mid = uuid_to_rid(p->azCChild[i], 1);
1436	if( mid>0 ){
1437	db_multi_exec("DELETE FROM unclustered WHERE rid=%d", mid);


1438	}
1439	}
1440	}
1441	if( p->type==CFTYPE_CONTROL
1442	\|\| p->type==CFTYPE_MANIFEST
1443

	--- src/manifest.c
	+++ src/manifest.c
	@@ -314,11 +314,11 @@
314	char cPrevType = 0;
315	char cType;
316	char *z;
317	int n;
318	char *zUuid;
319	int sz = 0;
320
321	/* Every control artifact ends with a '\n' character. Exit early
322	** if that is not the case for this artifact.
323	*/
324	z = blob_buffer(pContent);
	@@ -370,11 +370,11 @@
370	** is omitted to delete an attachment. <target> is the name of
371	** a wiki page or ticket to which that attachment is connected.
372	*/
373	case 'A': {
374	char zName, zTarget, *zSrc;
375	int nTarget = 0, nSrc = 0;
376	zName = next_token(&x, 0);
377	zTarget = next_token(&x, &nTarget);
378	zSrc = next_token(&x, &nSrc);
379	if( zName==0 \|\| zTarget==0 ) goto manifest_syntax_error;
380	if( p->zAttachName!=0 ) goto manifest_syntax_error;
	@@ -913,28 +913,47 @@
913	}
914	}
915
916	/*
917	** Fetch the baseline associated with the delta-manifest p.
918	** Return 0 on success. If unable to parse the baseline,
919	** throw an error. If the baseline is a manifest, throw an
920	** error if throwError is true, or record that p is an orphan
921	** and return 1 throwError is false.
922	*/
923	static int fetch_baseline(Manifest *p, int throwError){
924	if( p->zBaseline!=0 && p->pBaseline==0 ){
925	int rid = uuid_to_rid(p->zBaseline, 0);
926	if( rid==0 && !throwError ){
927	rid = content_new(p->zBaseline);
928	db_multi_exec(
929	"INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
930	rid, p->rid
931	);
932	return 1;
933	}
934	p->pBaseline = manifest_get(rid, CFTYPE_MANIFEST);
935	if( p->pBaseline==0 ){
936	if( !throwError && db_exists("SELECT 1 FROM phantom WHERE rid=%d",rid) ){
937	db_multi_exec(
938	"INSERT OR IGNORE INTO orphan(rid, baseline) VALUES(%d,%d)",
939	rid, p->rid
940	);
941	return 1;
942	}
943	fossil_fatal("cannot access baseline manifest %S", p->zBaseline);
944	}
945	}
946	return 0;
947	}
948
949	/*
950	** Rewind a manifest-file iterator back to the beginning of the manifest.
951	*/
952	void manifest_file_rewind(Manifest *p){
953	p->iFile = 0;
954	fetch_baseline(p, 1);
955	if( p->pBaseline ){
956	p->pBaseline->iFile = 0;
957	}
958	}
959
	@@ -1123,11 +1142,11 @@
1142	ManifestFile *pFile;
1143
1144	pFile = manifest_file_seek_base(p, zName);
1145	if( pFile && pFile->zUuid==0 ) return 0;
1146	if( pFile==0 && p->zBaseline ){
1147	fetch_baseline(p, 1);
1148	pFile = manifest_file_seek_base(p->pBaseline, zName);
1149	}
1150	return pFile;
1151	}
1152
	@@ -1181,15 +1200,18 @@
1200	if( (*ppOther = manifest_cache_find(otherRid))==0 ){
1201	content_get(otherRid, &otherContent);
1202	if( blob_size(&otherContent)==0 ) return;
1203	*ppOther = manifest_parse(&otherContent, otherRid);
1204	if( *ppOther==0 ) return;
1205	}
1206	if( fetch_baseline(pParent, 0) \|\| fetch_baseline(pChild, 0) ){
1207	manifest_destroy(*ppOther);
1208	return;
1209	}
1210	if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
1211	content_deltify(pid, cid, 0);
1212	}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){

1213	content_deltify(pParent->pBaseline->rid, cid, 0);
1214	}
1215
1216	for(i=0, pChildFile=pChild->aFile; i<pChild->nFile; i++, pChildFile++){
1217	if( pChildFile->zPrior ){
	@@ -1370,10 +1392,14 @@
1392	return 0;
1393	}
1394	if( g.xlinkClusterOnly && p->type!=CFTYPE_CLUSTER ){
1395	manifest_destroy(p);
1396	return 0;
1397	}
1398	if( p->type==CFTYPE_MANIFEST && fetch_baseline(p, 0) ){
1399	manifest_destroy(p);
1400	return 0;
1401	}
1402	db_begin_transaction();
1403	if( p->type==CFTYPE_MANIFEST ){
1404	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
1405	char *zCom;
	@@ -1427,16 +1453,20 @@
1453	}
1454	}
1455	}
1456	}
1457	if( p->type==CFTYPE_CLUSTER ){
1458	static Stmt del1;
1459	tag_insert("cluster", 1, 0, rid, p->rDate, rid);
1460	db_static_prepare(&del1, "DELETE FROM unclustered WHERE rid=:rid");
1461	for(i=0; i<p->nCChild; i++){
1462	int mid;
1463	mid = uuid_to_rid(p->azCChild[i], 1);
1464	if( mid>0 ){
1465	db_bind_int(&del1, ":rid", mid);
1466	db_step(&del1);
1467	db_reset(&del1);
1468	}
1469	}
1470	}
1471	if( p->type==CFTYPE_CONTROL
1472	\|\| p->type==CFTYPE_MANIFEST
1473

M src/rebuild.c

+34

		--- src/rebuild.c
		+++ src/rebuild.c
		@@ -247,10 +247,11 @@
247	247	*/
248	248	int rebuild_db(int randomize, int doOut){
249	249	Stmt s;
250	250	int errCnt = 0;
251	251	char *zTable;
	252	+ int incrSize;
252	253
253	254	bag_init(&bagDone);
254	255	ttyOutput = doOut;
255	256	processCnt = 0;
256	257	if (!g.fQuiet) {
		@@ -284,10 +285,12 @@
284	285	);
285	286	db_multi_exec(
286	287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	288	);
288	289	totalSize = db_int(0, "SELECT count(*) FROM blob");
	290	+ incrSize = totalSize/100;
	291	+ totalSize += incrSize*2;
289	292	db_prepare(&s,
290	293	"SELECT rid, size FROM blob /scan/"
291	294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	296	);
		@@ -321,10 +324,19 @@
321	324	}
322	325	}
323	326	db_finalize(&s);
324	327	manifest_crosslink_end();
325	328	rebuild_tag_trunk();
	329	+ if (!g.fQuiet) {
	330	+ processCnt += incrSize;
	331	+ percent_complete((processCnt*1000)/totalSize);
	332	+ }
	333	+ create_cluster();
	334	+ if (!g.fQuiet) {
	335	+ processCnt += incrSize;
	336	+ percent_complete((processCnt*1000)/totalSize);
	337	+ }
326	338	if(!g.fQuiet && ttyOutput ){
327	339	printf("\n");
328	340	}
329	341	return errCnt;
330	342	}
		@@ -385,10 +397,32 @@
385	397	"UPDATE config SET value='detached-' \|\| value"
386	398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	399	);
388	400	db_end_transaction(0);
389	401	}
	402	+
	403	+/*
	404	+** COMMAND: test-create-clusters
	405	+**
	406	+** Create clusters for all unclustered artifacts if the number of unclustered
	407	+** artifacts exceeds the current clustering threshold.
	408	+*/
	409	+void test_createcluster_cmd(void){
	410	+ if( g.argc==3 ){
	411	+ db_open_repository(g.argv[2]);
	412	+ }else{
	413	+ db_find_and_open_repository(1);
	414	+ if( g.argc!=2 ){
	415	+ usage("?REPOSITORY-FILENAME?");
	416	+ }
	417	+ db_close();
	418	+ db_open_repository(g.zRepositoryName);
	419	+ }
	420	+ db_begin_transaction();
	421	+ create_cluster();
	422	+ db_end_transaction(0);
	423	+}
390	424
391	425	/*
392	426	** COMMAND: scrub
393	427	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	428	**
395	429

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;

252
253	bag_init(&bagDone);
254	ttyOutput = doOut;
255	processCnt = 0;
256	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
284	);
285	db_multi_exec(
286	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	);
288	totalSize = db_int(0, "SELECT count(*) FROM blob");


289	db_prepare(&s,
290	"SELECT rid, size FROM blob /scan/"
291	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	);
	@@ -321,10 +324,19 @@
321	}
322	}
323	db_finalize(&s);
324	manifest_crosslink_end();
325	rebuild_tag_trunk();









326	if(!g.fQuiet && ttyOutput ){
327	printf("\n");
328	}
329	return errCnt;
330	}
	@@ -385,10 +397,32 @@
385	"UPDATE config SET value='detached-' \|\| value"
386	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	);
388	db_end_transaction(0);
389	}






















390
391	/*
392	** COMMAND: scrub
393	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	**
395

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;
252	int incrSize;
253
254	bag_init(&bagDone);
255	ttyOutput = doOut;
256	processCnt = 0;
257	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
285	);
286	db_multi_exec(
287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
288	);
289	totalSize = db_int(0, "SELECT count(*) FROM blob");
290	incrSize = totalSize/100;
291	totalSize += incrSize*2;
292	db_prepare(&s,
293	"SELECT rid, size FROM blob /scan/"
294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
296	);
	@@ -321,10 +324,19 @@
324	}
325	}
326	db_finalize(&s);
327	manifest_crosslink_end();
328	rebuild_tag_trunk();
329	if (!g.fQuiet) {
330	processCnt += incrSize;
331	percent_complete((processCnt*1000)/totalSize);
332	}
333	create_cluster();
334	if (!g.fQuiet) {
335	processCnt += incrSize;
336	percent_complete((processCnt*1000)/totalSize);
337	}
338	if(!g.fQuiet && ttyOutput ){
339	printf("\n");
340	}
341	return errCnt;
342	}
	@@ -385,10 +397,32 @@
397	"UPDATE config SET value='detached-' \|\| value"
398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
399	);
400	db_end_transaction(0);
401	}
402
403	/*
404	** COMMAND: test-create-clusters
405	**
406	** Create clusters for all unclustered artifacts if the number of unclustered
407	** artifacts exceeds the current clustering threshold.
408	*/
409	void test_createcluster_cmd(void){
410	if( g.argc==3 ){
411	db_open_repository(g.argv[2]);
412	}else{
413	db_find_and_open_repository(1);
414	if( g.argc!=2 ){
415	usage("?REPOSITORY-FILENAME?");
416	}
417	db_close();
418	db_open_repository(g.zRepositoryName);
419	}
420	db_begin_transaction();
421	create_cluster();
422	db_end_transaction(0);
423	}
424
425	/*
426	** COMMAND: scrub
427	** %fossil scrub [--verily] [--force] [REPOSITORY]
428	**
429

M src/rebuild.c

+34

		--- src/rebuild.c
		+++ src/rebuild.c
		@@ -247,10 +247,11 @@
247	247	*/
248	248	int rebuild_db(int randomize, int doOut){
249	249	Stmt s;
250	250	int errCnt = 0;
251	251	char *zTable;
	252	+ int incrSize;
252	253
253	254	bag_init(&bagDone);
254	255	ttyOutput = doOut;
255	256	processCnt = 0;
256	257	if (!g.fQuiet) {
		@@ -284,10 +285,12 @@
284	285	);
285	286	db_multi_exec(
286	287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	288	);
288	289	totalSize = db_int(0, "SELECT count(*) FROM blob");
	290	+ incrSize = totalSize/100;
	291	+ totalSize += incrSize*2;
289	292	db_prepare(&s,
290	293	"SELECT rid, size FROM blob /scan/"
291	294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	296	);
		@@ -321,10 +324,19 @@
321	324	}
322	325	}
323	326	db_finalize(&s);
324	327	manifest_crosslink_end();
325	328	rebuild_tag_trunk();
	329	+ if (!g.fQuiet) {
	330	+ processCnt += incrSize;
	331	+ percent_complete((processCnt*1000)/totalSize);
	332	+ }
	333	+ create_cluster();
	334	+ if (!g.fQuiet) {
	335	+ processCnt += incrSize;
	336	+ percent_complete((processCnt*1000)/totalSize);
	337	+ }
326	338	if(!g.fQuiet && ttyOutput ){
327	339	printf("\n");
328	340	}
329	341	return errCnt;
330	342	}
		@@ -385,10 +397,32 @@
385	397	"UPDATE config SET value='detached-' \|\| value"
386	398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	399	);
388	400	db_end_transaction(0);
389	401	}
	402	+
	403	+/*
	404	+** COMMAND: test-create-clusters
	405	+**
	406	+** Create clusters for all unclustered artifacts if the number of unclustered
	407	+** artifacts exceeds the current clustering threshold.
	408	+*/
	409	+void test_createcluster_cmd(void){
	410	+ if( g.argc==3 ){
	411	+ db_open_repository(g.argv[2]);
	412	+ }else{
	413	+ db_find_and_open_repository(1);
	414	+ if( g.argc!=2 ){
	415	+ usage("?REPOSITORY-FILENAME?");
	416	+ }
	417	+ db_close();
	418	+ db_open_repository(g.zRepositoryName);
	419	+ }
	420	+ db_begin_transaction();
	421	+ create_cluster();
	422	+ db_end_transaction(0);
	423	+}
390	424
391	425	/*
392	426	** COMMAND: scrub
393	427	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	428	**
395	429

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;

252
253	bag_init(&bagDone);
254	ttyOutput = doOut;
255	processCnt = 0;
256	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
284	);
285	db_multi_exec(
286	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
287	);
288	totalSize = db_int(0, "SELECT count(*) FROM blob");


289	db_prepare(&s,
290	"SELECT rid, size FROM blob /scan/"
291	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
292	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
293	);
	@@ -321,10 +324,19 @@
321	}
322	}
323	db_finalize(&s);
324	manifest_crosslink_end();
325	rebuild_tag_trunk();









326	if(!g.fQuiet && ttyOutput ){
327	printf("\n");
328	}
329	return errCnt;
330	}
	@@ -385,10 +397,32 @@
385	"UPDATE config SET value='detached-' \|\| value"
386	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
387	);
388	db_end_transaction(0);
389	}






















390
391	/*
392	** COMMAND: scrub
393	** %fossil scrub [--verily] [--force] [REPOSITORY]
394	**
395

	--- src/rebuild.c
	+++ src/rebuild.c
	@@ -247,10 +247,11 @@
247	*/
248	int rebuild_db(int randomize, int doOut){
249	Stmt s;
250	int errCnt = 0;
251	char *zTable;
252	int incrSize;
253
254	bag_init(&bagDone);
255	ttyOutput = doOut;
256	processCnt = 0;
257	if (!g.fQuiet) {
	@@ -284,10 +285,12 @@
285	);
286	db_multi_exec(
287	"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
288	);
289	totalSize = db_int(0, "SELECT count(*) FROM blob");
290	incrSize = totalSize/100;
291	totalSize += incrSize*2;
292	db_prepare(&s,
293	"SELECT rid, size FROM blob /scan/"
294	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
295	" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
296	);
	@@ -321,10 +324,19 @@
324	}
325	}
326	db_finalize(&s);
327	manifest_crosslink_end();
328	rebuild_tag_trunk();
329	if (!g.fQuiet) {
330	processCnt += incrSize;
331	percent_complete((processCnt*1000)/totalSize);
332	}
333	create_cluster();
334	if (!g.fQuiet) {
335	processCnt += incrSize;
336	percent_complete((processCnt*1000)/totalSize);
337	}
338	if(!g.fQuiet && ttyOutput ){
339	printf("\n");
340	}
341	return errCnt;
342	}
	@@ -385,10 +397,32 @@
397	"UPDATE config SET value='detached-' \|\| value"
398	" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
399	);
400	db_end_transaction(0);
401	}
402
403	/*
404	** COMMAND: test-create-clusters
405	**
406	** Create clusters for all unclustered artifacts if the number of unclustered
407	** artifacts exceeds the current clustering threshold.
408	*/
409	void test_createcluster_cmd(void){
410	if( g.argc==3 ){
411	db_open_repository(g.argv[2]);
412	}else{
413	db_find_and_open_repository(1);
414	if( g.argc!=2 ){
415	usage("?REPOSITORY-FILENAME?");
416	}
417	db_close();
418	db_open_repository(g.zRepositoryName);
419	}
420	db_begin_transaction();
421	create_cluster();
422	db_end_transaction(0);
423	}
424
425	/*
426	** COMMAND: scrub
427	** %fossil scrub [--verily] [--force] [REPOSITORY]
428	**
429

M src/schema.c

+10

		--- src/schema.c
		+++ src/schema.c
		@@ -241,10 +241,20 @@
241	241	@ -- UUID but we do not (yet) know the file content.
242	242	@ --
243	243	@ CREATE TABLE phantom(
244	244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	245	@ );
	246	+@
	247	+@ -- A record of orphaned delta-manifests. An orphan is a delta-manifest
	248	+@ -- for which we have content, but its baseline-manifest is a phantom.
	249	+@ -- We have to track all orphan maniftests so that when the baseline arrives,
	250	+@ -- we know to process the orphaned deltas.
	251	+@ CREATE TABLE orphan(
	252	+@ rid INTEGER PRIMARY KEY, -- Delta manifest with a phantom baseline
	253	+@ baseline INTEGER -- Phantom baseline of this orphan
	254	+@ );
	255	+@ CREATE INDEX orphan_baseline ON orphan(baseline);
246	256	@
247	257	@ -- Unclustered records. An unclustered record is a record (including
248	258	@ -- a cluster records themselves) that is not mentioned by some other
249	259	@ -- cluster.
250	260	@ --
251	261

	--- src/schema.c
	+++ src/schema.c
	@@ -241,10 +241,20 @@
241	@ -- UUID but we do not (yet) know the file content.
242	@ --
243	@ CREATE TABLE phantom(
244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	@ );










246	@
247	@ -- Unclustered records. An unclustered record is a record (including
248	@ -- a cluster records themselves) that is not mentioned by some other
249	@ -- cluster.
250	@ --
251

	--- src/schema.c
	+++ src/schema.c
	@@ -241,10 +241,20 @@
241	@ -- UUID but we do not (yet) know the file content.
242	@ --
243	@ CREATE TABLE phantom(
244	@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
245	@ );
246	@
247	@ -- A record of orphaned delta-manifests. An orphan is a delta-manifest
248	@ -- for which we have content, but its baseline-manifest is a phantom.
249	@ -- We have to track all orphan maniftests so that when the baseline arrives,
250	@ -- we know to process the orphaned deltas.
251	@ CREATE TABLE orphan(
252	@ rid INTEGER PRIMARY KEY, -- Delta manifest with a phantom baseline
253	@ baseline INTEGER -- Phantom baseline of this orphan
254	@ );
255	@ CREATE INDEX orphan_baseline ON orphan(baseline);
256	@
257	@ -- Unclustered records. An unclustered record is a record (including
258	@ -- a cluster records themselves) that is not mentioned by some other
259	@ -- cluster.
260	@ --
261

M src/xfer.c

+129 -41

		--- src/xfer.c
		+++ src/xfer.c
		@@ -192,11 +192,17 @@
192	192	/*
193	193	** Remember that the other side of the connection already has a copy
194	194	** of the file rid.
195	195	*/
196	196	static void remote_has(int rid){
197		- if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
	197	+ if( rid ){
	198	+ static Stmt q;
	199	+ db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
	200	+ db_bind_int(&q, ":r", rid);
	201	+ db_step(&q);
	202	+ db_reset(&q);
	203	+ }
198	204	}
199	205
200	206	/*
201	207	** The aToken[0..nToken-1] blob array is a parse of a "file" line
202	208	** message. This routine finishes parsing that message and does
		@@ -215,11 +221,11 @@
215	221	** be initialized to an empty string.
216	222	**
217	223	** Any artifact successfully received by this routine is considered to
218	224	** be public and is therefore removed from the "private" table.
219	225	*/
220		-static void xfer_accept_file(Xfer *pXfer){
	226	+static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
221	227	int n;
222	228	int rid;
223	229	int srcid = 0;
224	230	Blob content, hash;
225	231
		@@ -234,13 +240,26 @@
234	240	return;
235	241	}
236	242	blob_zero(&content);
237	243	blob_zero(&hash);
238	244	blob_extract(pXfer->pIn, n, &content);
239		- if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
	245	+ if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
240	246	/* Ignore files that have been shunned */
241	247	return;
	248	+ }
	249	+ if( cloneFlag ){
	250	+ if( pXfer->nToken==4 ){
	251	+ srcid = rid_from_uuid(&pXfer->aToken[2], 1);
	252	+ pXfer->nDeltaRcvd++;
	253	+ }else{
	254	+ srcid = 0;
	255	+ pXfer->nFileRcvd++;
	256	+ }
	257	+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
	258	+ remote_has(rid);
	259	+ blob_reset(&content);
	260	+ return;
242	261	}
243	262	if( pXfer->nToken==4 ){
244	263	Blob src, next;
245	264	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
246	265	if( content_get(srcid, &src)==0 ){
		@@ -587,14 +606,15 @@
587	606	** Check to see if the number of unclustered entries is greater than
588	607	** 100 and if it is, form a new cluster. Unclustered phantoms do not
589	608	** count toward the 100 total. And phantoms are never added to a new
590	609	** cluster.
591	610	*/
592		-static void create_cluster(void){
	611	+void create_cluster(void){
593	612	Blob cluster, cksum;
594	613	Stmt q;
595	614	int nUncl;
	615	+ int nRow = 0;
596	616
597	617	/* We should not ever get any private artifacts in the unclustered table.
598	618	** But if we do (because of a bug) now is a good time to delete them. */
599	619	db_multi_exec(
600	620	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
		@@ -601,30 +621,41 @@
601	621	);
602	622
603	623	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
604	624	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
605	625	" WHERE rid=unclustered.rid)");
606		- if( nUncl<100 ){
607		- return;
608		- }
609		- blob_zero(&cluster);
610		- db_prepare(&q, "SELECT uuid FROM unclustered, blob"
611		- " WHERE NOT EXISTS(SELECT 1 FROM phantom"
612		- " WHERE rid=unclustered.rid)"
613		- " AND unclustered.rid=blob.rid"
614		- " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
615		- " ORDER BY 1");
616		- while( db_step(&q)==SQLITE_ROW ){
617		- blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
618		- }
619		- db_finalize(&q);
620		- md5sum_blob(&cluster, &cksum);
621		- blob_appendf(&cluster, "Z %b\n", &cksum);
622		- blob_reset(&cksum);
623		- db_multi_exec("DELETE FROM unclustered");
624		- content_put(&cluster, 0, 0);
625		- blob_reset(&cluster);
	626	+ if( nUncl>=100 ){
	627	+ blob_zero(&cluster);
	628	+ db_prepare(&q, "SELECT uuid FROM unclustered, blob"
	629	+ " WHERE NOT EXISTS(SELECT 1 FROM phantom"
	630	+ " WHERE rid=unclustered.rid)"
	631	+ " AND unclustered.rid=blob.rid"
	632	+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
	633	+ " ORDER BY 1");
	634	+ while( db_step(&q)==SQLITE_ROW ){
	635	+ blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
	636	+ nRow++;
	637	+ if( nRow>=800 && nUncl>nRow+100 ){
	638	+ md5sum_blob(&cluster, &cksum);
	639	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	640	+ blob_reset(&cksum);
	641	+ content_put(&cluster, 0, 0);
	642	+ blob_reset(&cluster);
	643	+ nUncl -= nRow;
	644	+ nRow = 0;
	645	+ }
	646	+ }
	647	+ db_finalize(&q);
	648	+ db_multi_exec("DELETE FROM unclustered");
	649	+ if( nRow>0 ){
	650	+ md5sum_blob(&cluster, &cksum);
	651	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	652	+ blob_reset(&cksum);
	653	+ content_put(&cluster, 0, 0);
	654	+ blob_reset(&cluster);
	655	+ }
	656	+ }
626	657	}
627	658
628	659	/*
629	660	** Send an igot message for every entry in unclustered table.
630	661	** Return the number of cards sent.
		@@ -729,19 +760,17 @@
729	760	blobarray_zero(xfer.aToken, count(xfer.aToken));
730	761	cgi_set_content_type(g.zContentType);
731	762	blob_zero(&xfer.err);
732	763	xfer.pIn = &g.cgiIn;
733	764	xfer.pOut = cgi_output_blob();
734		- xfer.mxSend = db_get_int("max-download", 5000000);
	765	+ xfer.mxSend = db_get_int("max-download", 20000000);
735	766	g.xferPanic = 1;
736	767
737	768	db_begin_transaction();
738	769	db_multi_exec(
739	770	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
740	771	);
741		- zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
742		- @ # timestamp %s(zNow)
743	772	manifest_crosslink_begin();
744	773	while( blob_line(xfer.pIn, &xfer.line) ){
745	774	if( blob_buffer(&xfer.line)[0]=='#' ){
746	775	if( lenPushHookPattern
747	776	&& blob_buffer(&xfer.line)[1]
		@@ -765,11 +794,11 @@
765	794	cgi_reset_content();
766	795	@ error not\sauthorized\sto\swrite
767	796	nErr++;
768	797	break;
769	798	}
770		- xfer_accept_file(&xfer);
	799	+ xfer_accept_file(&xfer, 0);
771	800	if( blob_size(&xfer.err) ){
772	801	cgi_reset_content();
773	802	@ error %T(blob_str(&xfer.err))
774	803	nErr++;
775	804	break;
		@@ -851,26 +880,42 @@
851	880	isPush = 1;
852	881	}
853	882	}
854	883	}else
855	884
856		- /* clone
	885	+ /* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
857	886	**
858	887	** The client knows nothing. Tell all.
859	888	*/
860	889	if( blob_eq(&xfer.aToken[0], "clone") ){
	890	+ int iVers;
861	891	login_check_credentials();
862	892	if( !g.okClone ){
863	893	cgi_reset_content();
864	894	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
865	895	@ error not\sauthorized\sto\sclone
866	896	nErr++;
867	897	break;
868	898	}
869		- isClone = 1;
870		- isPull = 1;
871		- deltaFlag = 1;
	899	+ if( xfer.nToken==3
	900	+ && blob_is_int(&xfer.aToken[1], &iVers)
	901	+ && iVers>=2
	902	+ ){
	903	+ int seqno, max;
	904	+ blob_is_int(&xfer.aToken[2], &seqno);
	905	+ max = db_int(0, "SELECT max(rid) FROM blob");
	906	+ while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
	907	+ send_file(&xfer, seqno, 0, 1);
	908	+ seqno++;
	909	+ }
	910	+ if( seqno>=max ) seqno = 0;
	911	+ @ clone_seqno %d(seqno)
	912	+ }else{
	913	+ isClone = 1;
	914	+ isPull = 1;
	915	+ deltaFlag = 1;
	916	+ }
872	917	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
873	918	}else
874	919
875	920	/* login USER NONCE SIGNATURE
876	921	**
		@@ -1007,10 +1052,18 @@
1007	1052	}
1008	1053	if( recvConfig ){
1009	1054	configure_finalize_receive();
1010	1055	}
1011	1056	manifest_crosslink_end();
	1057	+
	1058	+ /* Send the server timestamp last, in case prior processing happened
	1059	+ ** to use up a significant fraction of our time window.
	1060	+ */
	1061	+ zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
	1062	+ @ # timestamp %s(zNow)
	1063	+ free(zNow);
	1064	+
1012	1065	db_end_transaction(0);
1013	1066	}
1014	1067
1015	1068	/*
1016	1069	** COMMAND: test-xfer
		@@ -1076,13 +1129,17 @@
1076	1129	int origConfigRcvMask; /* Original value of configRcvMask */
1077	1130	int nFileRecv; /* Number of files received */
1078	1131	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1079	1132	const char zCookie; / Server cookie */
1080	1133	int nSent, nRcvd; /* Bytes sent and received (after compression) */
	1134	+ int cloneSeqno = 1; /* Sequence number for clones */
1081	1135	Blob send; /* Text we are sending to the server */
1082	1136	Blob recv; /* Reply we got back from the server */
1083	1137	Xfer xfer; /* Transfer data */
	1138	+ int pctDone; /* Percentage done with a message */
	1139	+ int lastPctDone = -1; /* Last displayed pctDone */
	1140	+ double rArrivalTime; /* Time at which a message arrived */
1084	1141	const char *zSCode = db_get("server-code", "x");
1085	1142	const char *zPCode = db_get("project-code", 0);
1086	1143	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1087	1144	int allowForced = db_get_boolean("push-hook-force", 0);
1088	1145
		@@ -1113,15 +1170,16 @@
1113	1170
1114	1171	/*
1115	1172	** Always begin with a clone, pull, or push message
1116	1173	*/
1117	1174	if( cloneFlag ){
1118		- blob_appendf(&send, "clone\n");
	1175	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1119	1176	pushFlag = 0;
1120	1177	pullFlag = 0;
1121	1178	nCardSent++;
1122	1179	/* TBD: Request all transferable configuration values */
	1180	+ content_enable_dephantomize(0);
1123	1181	}else if( pullFlag ){
1124	1182	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1125	1183	nCardSent++;
1126	1184	}
1127	1185	if( pushFlag ){
		@@ -1145,11 +1203,11 @@
1145	1203	}
1146	1204
1147	1205	/* Generate gimme cards for phantoms and leaf cards
1148	1206	** for all leaves.
1149	1207	*/
1150		- if( pullFlag \|\| cloneFlag ){
	1208	+ if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1151	1209	request_phantoms(&xfer, mxPhantomReq);
1152	1210	}
1153	1211	if( pushFlag ){
1154	1212	send_unsent(&xfer);
1155	1213	nCardSent += send_unclustered(&xfer);
		@@ -1194,22 +1252,27 @@
1194	1252	blob_appendf(&send, "# %s\n", zRandomness);
1195	1253	free(zRandomness);
1196	1254
1197	1255	/* Exchange messages with the server */
1198	1256	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1199		- fossil_print(zValueFormat, "Send:",
	1257	+ fossil_print(zValueFormat, "Sent:",
1200	1258	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1201	1259	xfer.nFileSent, xfer.nDeltaSent);
1202	1260	nCardSent = 0;
1203	1261	nCardRcvd = 0;
1204	1262	xfer.nFileSent = 0;
1205	1263	xfer.nDeltaSent = 0;
1206	1264	xfer.nGimmeSent = 0;
1207	1265	xfer.nIGotSent = 0;
	1266	+ if( !g.cgiOutput && !g.fQuiet ){
	1267	+ printf("waiting for server...");
	1268	+ }
1208	1269	fflush(stdout);
1209	1270	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
	1271	+ lastPctDone = -1;
1210	1272	blob_reset(&send);
	1273	+ rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1211	1274
1212	1275	/* Begin constructing the next message (which might never be
1213	1276	** sent) by beginning with the pull or push cards
1214	1277	*/
1215	1278	if( pullFlag ){
		@@ -1222,18 +1285,22 @@
1222	1285	}
1223	1286	go = 0;
1224	1287
1225	1288	/* Process the reply that came back from the server */
1226	1289	while( blob_line(&recv, &xfer.line) ){
	1290	+ if( g.fHttpTrace ){
	1291	+ printf("\rGOT: %.*s", (int)blob_size(&xfer.line),
	1292	+ blob_buffer(&xfer.line));
	1293	+ }
1227	1294	if( blob_buffer(&xfer.line)[0]=='#' ){
1228	1295	const char *zLine = blob_buffer(&xfer.line);
1229	1296	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1230	1297	char zTime[20];
1231	1298	double rDiff;
1232	1299	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1233		- rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1234		- zTime);
	1300	+ rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
	1301	+ zTime, rArrivalTime);
1235	1302	if( rDiff<0.0 ) rDiff = -rDiff;
1236	1303	if( rDiff>9e98 ) rDiff = 0.0;
1237	1304	if( (rDiff24.03600.0)>=60.0 ){
1238	1305	fossil_warning("* time skew * server time differs by %s",
1239	1306	db_timespan_name(rDiff));
		@@ -1243,21 +1310,25 @@
1243	1310	continue;
1244	1311	}
1245	1312	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1246	1313	nCardRcvd++;
1247	1314	if( !g.cgiOutput && !g.fQuiet ){
1248		- printf("\r%d", nCardRcvd);
1249		- fflush(stdout);
	1315	+ pctDone = (recv.iCursor*100)/recv.nUsed;
	1316	+ if( pctDone!=lastPctDone ){
	1317	+ printf("\rprocessed: %d%% ", pctDone);
	1318	+ lastPctDone = pctDone;
	1319	+ fflush(stdout);
	1320	+ }
1250	1321	}
1251	1322
1252	1323	/* file UUID SIZE \n CONTENT
1253	1324	** file UUID DELTASRC SIZE \n CONTENT
1254	1325	**
1255	1326	** Receive a file transmitted from the server.
1256	1327	*/
1257	1328	if( blob_eq(&xfer.aToken[0],"file") ){
1258		- xfer_accept_file(&xfer);
	1329	+ xfer_accept_file(&xfer, cloneFlag);
1259	1330	}else
1260	1331
1261	1332	/* gimme UUID
1262	1333	**
1263	1334	** Server is requesting a file. If the file is a manifest, assume
		@@ -1313,11 +1384,11 @@
1313	1384	}
1314	1385	if( zPCode==0 ){
1315	1386	zPCode = mprintf("%b", &xfer.aToken[2]);
1316	1387	db_set("project-code", zPCode, 0);
1317	1388	}
1318		- blob_appendf(&send, "clone\n");
	1389	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1319	1390	nCardSent++;
1320	1391	}else
1321	1392
1322	1393	/* config NAME SIZE \n CONTENT
1323	1394	**
		@@ -1372,10 +1443,21 @@
1372	1443	** same server.
1373	1444	*/
1374	1445	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1375	1446	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1376	1447	}else
	1448	+
	1449	+ /* clone_seqno N
	1450	+ **
	1451	+ ** When doing a clone, the server tries to send all of its artifacts
	1452	+ ** in sequence. This card indicates the sequence number of the next
	1453	+ ** blob that needs to be sent. If N<=0 that indicates that all blobs
	1454	+ ** have been sent.
	1455	+ */
	1456	+ if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
	1457	+ blob_is_int(&xfer.aToken[1], &cloneSeqno);
	1458	+ }else
1377	1459
1378	1460	/* message MESSAGE
1379	1461	**
1380	1462	** Print a message. Similar to "error" but does not stop processing.
1381	1463	**
		@@ -1450,10 +1532,12 @@
1450	1532	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1451	1533	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1452	1534	go = 1;
1453	1535	mxPhantomReq = nFileRecv*2;
1454	1536	if( mxPhantomReq<200 ) mxPhantomReq = 200;
	1537	+ }else if( cloneFlag && nFileRecv>0 ){
	1538	+ go = 1;
1455	1539	}
1456	1540	nCardRcvd = 0;
1457	1541	xfer.nFileRcvd = 0;
1458	1542	xfer.nDeltaRcvd = 0;
1459	1543	xfer.nDanglingFile = 0;
		@@ -1465,10 +1549,13 @@
1465	1549	go = 1;
1466	1550	}
1467	1551
1468	1552	/* If this is a clone, the go at least two rounds */
1469	1553	if( cloneFlag && nCycle==1 ) go = 1;
	1554	+
	1555	+ /* Stop the cycle if the server sends a "clone_seqno 0" card */
	1556	+ if( cloneSeqno<=0 ) go = 0;
1470	1557	};
1471	1558	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1472	1559	if( zPushHookPattern && zPushHookPattern[0] ){
1473	1560	blob_appendf(&send, "#%s%s\n",
1474	1561	((nFileSend > 0)?"P":"F"), zPushHookPattern);
		@@ -1484,7 +1571,8 @@
1484	1571	nSent, nRcvd);
1485	1572	transport_close();
1486	1573	transport_global_shutdown();
1487	1574	db_multi_exec("DROP TABLE onremote");
1488	1575	manifest_crosslink_end();
	1576	+ content_enable_dephantomize(1);
1489	1577	db_end_transaction(0);
1490	1578	}
1491	1579

	--- src/xfer.c
	+++ src/xfer.c
	@@ -192,11 +192,17 @@
192	/*
193	** Remember that the other side of the connection already has a copy
194	** of the file rid.
195	*/
196	static void remote_has(int rid){
197	if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);






198	}
199
200	/*
201	** The aToken[0..nToken-1] blob array is a parse of a "file" line
202	** message. This routine finishes parsing that message and does
	@@ -215,11 +221,11 @@
215	** be initialized to an empty string.
216	**
217	** Any artifact successfully received by this routine is considered to
218	** be public and is therefore removed from the "private" table.
219	*/
220	static void xfer_accept_file(Xfer *pXfer){
221	int n;
222	int rid;
223	int srcid = 0;
224	Blob content, hash;
225
	@@ -234,13 +240,26 @@
234	return;
235	}
236	blob_zero(&content);
237	blob_zero(&hash);
238	blob_extract(pXfer->pIn, n, &content);
239	if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
240	/* Ignore files that have been shunned */
241	return;













242	}
243	if( pXfer->nToken==4 ){
244	Blob src, next;
245	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
246	if( content_get(srcid, &src)==0 ){
	@@ -587,14 +606,15 @@
587	** Check to see if the number of unclustered entries is greater than
588	** 100 and if it is, form a new cluster. Unclustered phantoms do not
589	** count toward the 100 total. And phantoms are never added to a new
590	** cluster.
591	*/
592	static void create_cluster(void){
593	Blob cluster, cksum;
594	Stmt q;
595	int nUncl;

596
597	/* We should not ever get any private artifacts in the unclustered table.
598	** But if we do (because of a bug) now is a good time to delete them. */
599	db_multi_exec(
600	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -601,30 +621,41 @@
601	);
602
603	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
604	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
605	" WHERE rid=unclustered.rid)");
606	if( nUncl<100 ){
607	return;
608	}
609	blob_zero(&cluster);
610	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
611	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
612	" WHERE rid=unclustered.rid)"
613	" AND unclustered.rid=blob.rid"
614	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
615	" ORDER BY 1");
616	while( db_step(&q)==SQLITE_ROW ){
617	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
618	}
619	db_finalize(&q);
620	md5sum_blob(&cluster, &cksum);
621	blob_appendf(&cluster, "Z %b\n", &cksum);
622	blob_reset(&cksum);
623	db_multi_exec("DELETE FROM unclustered");
624	content_put(&cluster, 0, 0);
625	blob_reset(&cluster);











626	}
627
628	/*
629	** Send an igot message for every entry in unclustered table.
630	** Return the number of cards sent.
	@@ -729,19 +760,17 @@
729	blobarray_zero(xfer.aToken, count(xfer.aToken));
730	cgi_set_content_type(g.zContentType);
731	blob_zero(&xfer.err);
732	xfer.pIn = &g.cgiIn;
733	xfer.pOut = cgi_output_blob();
734	xfer.mxSend = db_get_int("max-download", 5000000);
735	g.xferPanic = 1;
736
737	db_begin_transaction();
738	db_multi_exec(
739	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
740	);
741	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
742	@ # timestamp %s(zNow)
743	manifest_crosslink_begin();
744	while( blob_line(xfer.pIn, &xfer.line) ){
745	if( blob_buffer(&xfer.line)[0]=='#' ){
746	if( lenPushHookPattern
747	&& blob_buffer(&xfer.line)[1]
	@@ -765,11 +794,11 @@
765	cgi_reset_content();
766	@ error not\sauthorized\sto\swrite
767	nErr++;
768	break;
769	}
770	xfer_accept_file(&xfer);
771	if( blob_size(&xfer.err) ){
772	cgi_reset_content();
773	@ error %T(blob_str(&xfer.err))
774	nErr++;
775	break;
	@@ -851,26 +880,42 @@
851	isPush = 1;
852	}
853	}
854	}else
855
856	/* clone
857	**
858	** The client knows nothing. Tell all.
859	*/
860	if( blob_eq(&xfer.aToken[0], "clone") ){

861	login_check_credentials();
862	if( !g.okClone ){
863	cgi_reset_content();
864	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
865	@ error not\sauthorized\sto\sclone
866	nErr++;
867	break;
868	}
869	isClone = 1;
870	isPull = 1;
871	deltaFlag = 1;















872	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
873	}else
874
875	/* login USER NONCE SIGNATURE
876	**
	@@ -1007,10 +1052,18 @@
1007	}
1008	if( recvConfig ){
1009	configure_finalize_receive();
1010	}
1011	manifest_crosslink_end();








1012	db_end_transaction(0);
1013	}
1014
1015	/*
1016	** COMMAND: test-xfer
	@@ -1076,13 +1129,17 @@
1076	int origConfigRcvMask; /* Original value of configRcvMask */
1077	int nFileRecv; /* Number of files received */
1078	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1079	const char zCookie; / Server cookie */
1080	int nSent, nRcvd; /* Bytes sent and received (after compression) */

1081	Blob send; /* Text we are sending to the server */
1082	Blob recv; /* Reply we got back from the server */
1083	Xfer xfer; /* Transfer data */



1084	const char *zSCode = db_get("server-code", "x");
1085	const char *zPCode = db_get("project-code", 0);
1086	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1087	int allowForced = db_get_boolean("push-hook-force", 0);
1088
	@@ -1113,15 +1170,16 @@
1113
1114	/*
1115	** Always begin with a clone, pull, or push message
1116	*/
1117	if( cloneFlag ){
1118	blob_appendf(&send, "clone\n");
1119	pushFlag = 0;
1120	pullFlag = 0;
1121	nCardSent++;
1122	/* TBD: Request all transferable configuration values */

1123	}else if( pullFlag ){
1124	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1125	nCardSent++;
1126	}
1127	if( pushFlag ){
	@@ -1145,11 +1203,11 @@
1145	}
1146
1147	/* Generate gimme cards for phantoms and leaf cards
1148	** for all leaves.
1149	*/
1150	if( pullFlag \|\| cloneFlag ){
1151	request_phantoms(&xfer, mxPhantomReq);
1152	}
1153	if( pushFlag ){
1154	send_unsent(&xfer);
1155	nCardSent += send_unclustered(&xfer);
	@@ -1194,22 +1252,27 @@
1194	blob_appendf(&send, "# %s\n", zRandomness);
1195	free(zRandomness);
1196
1197	/* Exchange messages with the server */
1198	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1199	fossil_print(zValueFormat, "Send:",
1200	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1201	xfer.nFileSent, xfer.nDeltaSent);
1202	nCardSent = 0;
1203	nCardRcvd = 0;
1204	xfer.nFileSent = 0;
1205	xfer.nDeltaSent = 0;
1206	xfer.nGimmeSent = 0;
1207	xfer.nIGotSent = 0;



1208	fflush(stdout);
1209	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);

1210	blob_reset(&send);

1211
1212	/* Begin constructing the next message (which might never be
1213	** sent) by beginning with the pull or push cards
1214	*/
1215	if( pullFlag ){
	@@ -1222,18 +1285,22 @@
1222	}
1223	go = 0;
1224
1225	/* Process the reply that came back from the server */
1226	while( blob_line(&recv, &xfer.line) ){




1227	if( blob_buffer(&xfer.line)[0]=='#' ){
1228	const char *zLine = blob_buffer(&xfer.line);
1229	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1230	char zTime[20];
1231	double rDiff;
1232	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1233	rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1234	zTime);
1235	if( rDiff<0.0 ) rDiff = -rDiff;
1236	if( rDiff>9e98 ) rDiff = 0.0;
1237	if( (rDiff24.03600.0)>=60.0 ){
1238	fossil_warning("* time skew * server time differs by %s",
1239	db_timespan_name(rDiff));
	@@ -1243,21 +1310,25 @@
1243	continue;
1244	}
1245	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1246	nCardRcvd++;
1247	if( !g.cgiOutput && !g.fQuiet ){
1248	printf("\r%d", nCardRcvd);
1249	fflush(stdout);




1250	}
1251
1252	/* file UUID SIZE \n CONTENT
1253	** file UUID DELTASRC SIZE \n CONTENT
1254	**
1255	** Receive a file transmitted from the server.
1256	*/
1257	if( blob_eq(&xfer.aToken[0],"file") ){
1258	xfer_accept_file(&xfer);
1259	}else
1260
1261	/* gimme UUID
1262	**
1263	** Server is requesting a file. If the file is a manifest, assume
	@@ -1313,11 +1384,11 @@
1313	}
1314	if( zPCode==0 ){
1315	zPCode = mprintf("%b", &xfer.aToken[2]);
1316	db_set("project-code", zPCode, 0);
1317	}
1318	blob_appendf(&send, "clone\n");
1319	nCardSent++;
1320	}else
1321
1322	/* config NAME SIZE \n CONTENT
1323	**
	@@ -1372,10 +1443,21 @@
1372	** same server.
1373	*/
1374	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1375	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1376	}else











1377
1378	/* message MESSAGE
1379	**
1380	** Print a message. Similar to "error" but does not stop processing.
1381	**
	@@ -1450,10 +1532,12 @@
1450	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1451	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1452	go = 1;
1453	mxPhantomReq = nFileRecv*2;
1454	if( mxPhantomReq<200 ) mxPhantomReq = 200;


1455	}
1456	nCardRcvd = 0;
1457	xfer.nFileRcvd = 0;
1458	xfer.nDeltaRcvd = 0;
1459	xfer.nDanglingFile = 0;
	@@ -1465,10 +1549,13 @@
1465	go = 1;
1466	}
1467
1468	/* If this is a clone, the go at least two rounds */
1469	if( cloneFlag && nCycle==1 ) go = 1;



1470	};
1471	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1472	if( zPushHookPattern && zPushHookPattern[0] ){
1473	blob_appendf(&send, "#%s%s\n",
1474	((nFileSend > 0)?"P":"F"), zPushHookPattern);
	@@ -1484,7 +1571,8 @@
1484	nSent, nRcvd);
1485	transport_close();
1486	transport_global_shutdown();
1487	db_multi_exec("DROP TABLE onremote");
1488	manifest_crosslink_end();

1489	db_end_transaction(0);
1490	}
1491

	--- src/xfer.c
	+++ src/xfer.c
	@@ -192,11 +192,17 @@
192	/*
193	** Remember that the other side of the connection already has a copy
194	** of the file rid.
195	*/
196	static void remote_has(int rid){
197	if( rid ){
198	static Stmt q;
199	db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
200	db_bind_int(&q, ":r", rid);
201	db_step(&q);
202	db_reset(&q);
203	}
204	}
205
206	/*
207	** The aToken[0..nToken-1] blob array is a parse of a "file" line
208	** message. This routine finishes parsing that message and does
	@@ -215,11 +221,11 @@
221	** be initialized to an empty string.
222	**
223	** Any artifact successfully received by this routine is considered to
224	** be public and is therefore removed from the "private" table.
225	*/
226	static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
227	int n;
228	int rid;
229	int srcid = 0;
230	Blob content, hash;
231
	@@ -234,13 +240,26 @@
240	return;
241	}
242	blob_zero(&content);
243	blob_zero(&hash);
244	blob_extract(pXfer->pIn, n, &content);
245	if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
246	/* Ignore files that have been shunned */
247	return;
248	}
249	if( cloneFlag ){
250	if( pXfer->nToken==4 ){
251	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
252	pXfer->nDeltaRcvd++;
253	}else{
254	srcid = 0;
255	pXfer->nFileRcvd++;
256	}
257	rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
258	remote_has(rid);
259	blob_reset(&content);
260	return;
261	}
262	if( pXfer->nToken==4 ){
263	Blob src, next;
264	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
265	if( content_get(srcid, &src)==0 ){
	@@ -587,14 +606,15 @@
606	** Check to see if the number of unclustered entries is greater than
607	** 100 and if it is, form a new cluster. Unclustered phantoms do not
608	** count toward the 100 total. And phantoms are never added to a new
609	** cluster.
610	*/
611	void create_cluster(void){
612	Blob cluster, cksum;
613	Stmt q;
614	int nUncl;
615	int nRow = 0;
616
617	/* We should not ever get any private artifacts in the unclustered table.
618	** But if we do (because of a bug) now is a good time to delete them. */
619	db_multi_exec(
620	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -601,30 +621,41 @@
621	);
622
623	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
624	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
625	" WHERE rid=unclustered.rid)");
626	if( nUncl>=100 ){
627	blob_zero(&cluster);
628	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
629	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
630	" WHERE rid=unclustered.rid)"
631	" AND unclustered.rid=blob.rid"
632	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
633	" ORDER BY 1");
634	while( db_step(&q)==SQLITE_ROW ){
635	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
636	nRow++;
637	if( nRow>=800 && nUncl>nRow+100 ){
638	md5sum_blob(&cluster, &cksum);
639	blob_appendf(&cluster, "Z %b\n", &cksum);
640	blob_reset(&cksum);
641	content_put(&cluster, 0, 0);
642	blob_reset(&cluster);
643	nUncl -= nRow;
644	nRow = 0;
645	}
646	}
647	db_finalize(&q);
648	db_multi_exec("DELETE FROM unclustered");
649	if( nRow>0 ){
650	md5sum_blob(&cluster, &cksum);
651	blob_appendf(&cluster, "Z %b\n", &cksum);
652	blob_reset(&cksum);
653	content_put(&cluster, 0, 0);
654	blob_reset(&cluster);
655	}
656	}
657	}
658
659	/*
660	** Send an igot message for every entry in unclustered table.
661	** Return the number of cards sent.
	@@ -729,19 +760,17 @@
760	blobarray_zero(xfer.aToken, count(xfer.aToken));
761	cgi_set_content_type(g.zContentType);
762	blob_zero(&xfer.err);
763	xfer.pIn = &g.cgiIn;
764	xfer.pOut = cgi_output_blob();
765	xfer.mxSend = db_get_int("max-download", 20000000);
766	g.xferPanic = 1;
767
768	db_begin_transaction();
769	db_multi_exec(
770	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
771	);


772	manifest_crosslink_begin();
773	while( blob_line(xfer.pIn, &xfer.line) ){
774	if( blob_buffer(&xfer.line)[0]=='#' ){
775	if( lenPushHookPattern
776	&& blob_buffer(&xfer.line)[1]
	@@ -765,11 +794,11 @@
794	cgi_reset_content();
795	@ error not\sauthorized\sto\swrite
796	nErr++;
797	break;
798	}
799	xfer_accept_file(&xfer, 0);
800	if( blob_size(&xfer.err) ){
801	cgi_reset_content();
802	@ error %T(blob_str(&xfer.err))
803	nErr++;
804	break;
	@@ -851,26 +880,42 @@
880	isPush = 1;
881	}
882	}
883	}else
884
885	/* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
886	**
887	** The client knows nothing. Tell all.
888	*/
889	if( blob_eq(&xfer.aToken[0], "clone") ){
890	int iVers;
891	login_check_credentials();
892	if( !g.okClone ){
893	cgi_reset_content();
894	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
895	@ error not\sauthorized\sto\sclone
896	nErr++;
897	break;
898	}
899	if( xfer.nToken==3
900	&& blob_is_int(&xfer.aToken[1], &iVers)
901	&& iVers>=2
902	){
903	int seqno, max;
904	blob_is_int(&xfer.aToken[2], &seqno);
905	max = db_int(0, "SELECT max(rid) FROM blob");
906	while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
907	send_file(&xfer, seqno, 0, 1);
908	seqno++;
909	}
910	if( seqno>=max ) seqno = 0;
911	@ clone_seqno %d(seqno)
912	}else{
913	isClone = 1;
914	isPull = 1;
915	deltaFlag = 1;
916	}
917	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
918	}else
919
920	/* login USER NONCE SIGNATURE
921	**
	@@ -1007,10 +1052,18 @@
1052	}
1053	if( recvConfig ){
1054	configure_finalize_receive();
1055	}
1056	manifest_crosslink_end();
1057
1058	/* Send the server timestamp last, in case prior processing happened
1059	** to use up a significant fraction of our time window.
1060	*/
1061	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
1062	@ # timestamp %s(zNow)
1063	free(zNow);
1064
1065	db_end_transaction(0);
1066	}
1067
1068	/*
1069	** COMMAND: test-xfer
	@@ -1076,13 +1129,17 @@
1129	int origConfigRcvMask; /* Original value of configRcvMask */
1130	int nFileRecv; /* Number of files received */
1131	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1132	const char zCookie; / Server cookie */
1133	int nSent, nRcvd; /* Bytes sent and received (after compression) */
1134	int cloneSeqno = 1; /* Sequence number for clones */
1135	Blob send; /* Text we are sending to the server */
1136	Blob recv; /* Reply we got back from the server */
1137	Xfer xfer; /* Transfer data */
1138	int pctDone; /* Percentage done with a message */
1139	int lastPctDone = -1; /* Last displayed pctDone */
1140	double rArrivalTime; /* Time at which a message arrived */
1141	const char *zSCode = db_get("server-code", "x");
1142	const char *zPCode = db_get("project-code", 0);
1143	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1144	int allowForced = db_get_boolean("push-hook-force", 0);
1145
	@@ -1113,15 +1170,16 @@
1170
1171	/*
1172	** Always begin with a clone, pull, or push message
1173	*/
1174	if( cloneFlag ){
1175	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1176	pushFlag = 0;
1177	pullFlag = 0;
1178	nCardSent++;
1179	/* TBD: Request all transferable configuration values */
1180	content_enable_dephantomize(0);
1181	}else if( pullFlag ){
1182	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1183	nCardSent++;
1184	}
1185	if( pushFlag ){
	@@ -1145,11 +1203,11 @@
1203	}
1204
1205	/* Generate gimme cards for phantoms and leaf cards
1206	** for all leaves.
1207	*/
1208	if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1209	request_phantoms(&xfer, mxPhantomReq);
1210	}
1211	if( pushFlag ){
1212	send_unsent(&xfer);
1213	nCardSent += send_unclustered(&xfer);
	@@ -1194,22 +1252,27 @@
1252	blob_appendf(&send, "# %s\n", zRandomness);
1253	free(zRandomness);
1254
1255	/* Exchange messages with the server */
1256	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1257	fossil_print(zValueFormat, "Sent:",
1258	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1259	xfer.nFileSent, xfer.nDeltaSent);
1260	nCardSent = 0;
1261	nCardRcvd = 0;
1262	xfer.nFileSent = 0;
1263	xfer.nDeltaSent = 0;
1264	xfer.nGimmeSent = 0;
1265	xfer.nIGotSent = 0;
1266	if( !g.cgiOutput && !g.fQuiet ){
1267	printf("waiting for server...");
1268	}
1269	fflush(stdout);
1270	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
1271	lastPctDone = -1;
1272	blob_reset(&send);
1273	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1274
1275	/* Begin constructing the next message (which might never be
1276	** sent) by beginning with the pull or push cards
1277	*/
1278	if( pullFlag ){
	@@ -1222,18 +1285,22 @@
1285	}
1286	go = 0;
1287
1288	/* Process the reply that came back from the server */
1289	while( blob_line(&recv, &xfer.line) ){
1290	if( g.fHttpTrace ){
1291	printf("\rGOT: %.*s", (int)blob_size(&xfer.line),
1292	blob_buffer(&xfer.line));
1293	}
1294	if( blob_buffer(&xfer.line)[0]=='#' ){
1295	const char *zLine = blob_buffer(&xfer.line);
1296	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1297	char zTime[20];
1298	double rDiff;
1299	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1300	rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
1301	zTime, rArrivalTime);
1302	if( rDiff<0.0 ) rDiff = -rDiff;
1303	if( rDiff>9e98 ) rDiff = 0.0;
1304	if( (rDiff24.03600.0)>=60.0 ){
1305	fossil_warning("* time skew * server time differs by %s",
1306	db_timespan_name(rDiff));
	@@ -1243,21 +1310,25 @@
1310	continue;
1311	}
1312	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1313	nCardRcvd++;
1314	if( !g.cgiOutput && !g.fQuiet ){
1315	pctDone = (recv.iCursor*100)/recv.nUsed;
1316	if( pctDone!=lastPctDone ){
1317	printf("\rprocessed: %d%% ", pctDone);
1318	lastPctDone = pctDone;
1319	fflush(stdout);
1320	}
1321	}
1322
1323	/* file UUID SIZE \n CONTENT
1324	** file UUID DELTASRC SIZE \n CONTENT
1325	**
1326	** Receive a file transmitted from the server.
1327	*/
1328	if( blob_eq(&xfer.aToken[0],"file") ){
1329	xfer_accept_file(&xfer, cloneFlag);
1330	}else
1331
1332	/* gimme UUID
1333	**
1334	** Server is requesting a file. If the file is a manifest, assume
	@@ -1313,11 +1384,11 @@
1384	}
1385	if( zPCode==0 ){
1386	zPCode = mprintf("%b", &xfer.aToken[2]);
1387	db_set("project-code", zPCode, 0);
1388	}
1389	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1390	nCardSent++;
1391	}else
1392
1393	/* config NAME SIZE \n CONTENT
1394	**
	@@ -1372,10 +1443,21 @@
1443	** same server.
1444	*/
1445	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1446	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1447	}else
1448
1449	/* clone_seqno N
1450	**
1451	** When doing a clone, the server tries to send all of its artifacts
1452	** in sequence. This card indicates the sequence number of the next
1453	** blob that needs to be sent. If N<=0 that indicates that all blobs
1454	** have been sent.
1455	*/
1456	if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
1457	blob_is_int(&xfer.aToken[1], &cloneSeqno);
1458	}else
1459
1460	/* message MESSAGE
1461	**
1462	** Print a message. Similar to "error" but does not stop processing.
1463	**
	@@ -1450,10 +1532,12 @@
1532	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1533	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1534	go = 1;
1535	mxPhantomReq = nFileRecv*2;
1536	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1537	}else if( cloneFlag && nFileRecv>0 ){
1538	go = 1;
1539	}
1540	nCardRcvd = 0;
1541	xfer.nFileRcvd = 0;
1542	xfer.nDeltaRcvd = 0;
1543	xfer.nDanglingFile = 0;
	@@ -1465,10 +1549,13 @@
1549	go = 1;
1550	}
1551
1552	/* If this is a clone, the go at least two rounds */
1553	if( cloneFlag && nCycle==1 ) go = 1;
1554
1555	/* Stop the cycle if the server sends a "clone_seqno 0" card */
1556	if( cloneSeqno<=0 ) go = 0;
1557	};
1558	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1559	if( zPushHookPattern && zPushHookPattern[0] ){
1560	blob_appendf(&send, "#%s%s\n",
1561	((nFileSend > 0)?"P":"F"), zPushHookPattern);
	@@ -1484,7 +1571,8 @@
1571	nSent, nRcvd);
1572	transport_close();
1573	transport_global_shutdown();
1574	db_multi_exec("DROP TABLE onremote");
1575	manifest_crosslink_end();
1576	content_enable_dephantomize(1);
1577	db_end_transaction(0);
1578	}
1579

M src/xfer.c

+129 -41

		--- src/xfer.c
		+++ src/xfer.c
		@@ -192,11 +192,17 @@
192	192	/*
193	193	** Remember that the other side of the connection already has a copy
194	194	** of the file rid.
195	195	*/
196	196	static void remote_has(int rid){
197		- if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
	197	+ if( rid ){
	198	+ static Stmt q;
	199	+ db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
	200	+ db_bind_int(&q, ":r", rid);
	201	+ db_step(&q);
	202	+ db_reset(&q);
	203	+ }
198	204	}
199	205
200	206	/*
201	207	** The aToken[0..nToken-1] blob array is a parse of a "file" line
202	208	** message. This routine finishes parsing that message and does
		@@ -215,11 +221,11 @@
215	221	** be initialized to an empty string.
216	222	**
217	223	** Any artifact successfully received by this routine is considered to
218	224	** be public and is therefore removed from the "private" table.
219	225	*/
220		-static void xfer_accept_file(Xfer *pXfer){
	226	+static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
221	227	int n;
222	228	int rid;
223	229	int srcid = 0;
224	230	Blob content, hash;
225	231
		@@ -234,13 +240,26 @@
234	240	return;
235	241	}
236	242	blob_zero(&content);
237	243	blob_zero(&hash);
238	244	blob_extract(pXfer->pIn, n, &content);
239		- if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
	245	+ if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
240	246	/* Ignore files that have been shunned */
241	247	return;
	248	+ }
	249	+ if( cloneFlag ){
	250	+ if( pXfer->nToken==4 ){
	251	+ srcid = rid_from_uuid(&pXfer->aToken[2], 1);
	252	+ pXfer->nDeltaRcvd++;
	253	+ }else{
	254	+ srcid = 0;
	255	+ pXfer->nFileRcvd++;
	256	+ }
	257	+ rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
	258	+ remote_has(rid);
	259	+ blob_reset(&content);
	260	+ return;
242	261	}
243	262	if( pXfer->nToken==4 ){
244	263	Blob src, next;
245	264	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
246	265	if( content_get(srcid, &src)==0 ){
		@@ -587,14 +606,15 @@
587	606	** Check to see if the number of unclustered entries is greater than
588	607	** 100 and if it is, form a new cluster. Unclustered phantoms do not
589	608	** count toward the 100 total. And phantoms are never added to a new
590	609	** cluster.
591	610	*/
592		-static void create_cluster(void){
	611	+void create_cluster(void){
593	612	Blob cluster, cksum;
594	613	Stmt q;
595	614	int nUncl;
	615	+ int nRow = 0;
596	616
597	617	/* We should not ever get any private artifacts in the unclustered table.
598	618	** But if we do (because of a bug) now is a good time to delete them. */
599	619	db_multi_exec(
600	620	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
		@@ -601,30 +621,41 @@
601	621	);
602	622
603	623	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
604	624	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
605	625	" WHERE rid=unclustered.rid)");
606		- if( nUncl<100 ){
607		- return;
608		- }
609		- blob_zero(&cluster);
610		- db_prepare(&q, "SELECT uuid FROM unclustered, blob"
611		- " WHERE NOT EXISTS(SELECT 1 FROM phantom"
612		- " WHERE rid=unclustered.rid)"
613		- " AND unclustered.rid=blob.rid"
614		- " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
615		- " ORDER BY 1");
616		- while( db_step(&q)==SQLITE_ROW ){
617		- blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
618		- }
619		- db_finalize(&q);
620		- md5sum_blob(&cluster, &cksum);
621		- blob_appendf(&cluster, "Z %b\n", &cksum);
622		- blob_reset(&cksum);
623		- db_multi_exec("DELETE FROM unclustered");
624		- content_put(&cluster, 0, 0);
625		- blob_reset(&cluster);
	626	+ if( nUncl>=100 ){
	627	+ blob_zero(&cluster);
	628	+ db_prepare(&q, "SELECT uuid FROM unclustered, blob"
	629	+ " WHERE NOT EXISTS(SELECT 1 FROM phantom"
	630	+ " WHERE rid=unclustered.rid)"
	631	+ " AND unclustered.rid=blob.rid"
	632	+ " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
	633	+ " ORDER BY 1");
	634	+ while( db_step(&q)==SQLITE_ROW ){
	635	+ blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
	636	+ nRow++;
	637	+ if( nRow>=800 && nUncl>nRow+100 ){
	638	+ md5sum_blob(&cluster, &cksum);
	639	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	640	+ blob_reset(&cksum);
	641	+ content_put(&cluster, 0, 0);
	642	+ blob_reset(&cluster);
	643	+ nUncl -= nRow;
	644	+ nRow = 0;
	645	+ }
	646	+ }
	647	+ db_finalize(&q);
	648	+ db_multi_exec("DELETE FROM unclustered");
	649	+ if( nRow>0 ){
	650	+ md5sum_blob(&cluster, &cksum);
	651	+ blob_appendf(&cluster, "Z %b\n", &cksum);
	652	+ blob_reset(&cksum);
	653	+ content_put(&cluster, 0, 0);
	654	+ blob_reset(&cluster);
	655	+ }
	656	+ }
626	657	}
627	658
628	659	/*
629	660	** Send an igot message for every entry in unclustered table.
630	661	** Return the number of cards sent.
		@@ -729,19 +760,17 @@
729	760	blobarray_zero(xfer.aToken, count(xfer.aToken));
730	761	cgi_set_content_type(g.zContentType);
731	762	blob_zero(&xfer.err);
732	763	xfer.pIn = &g.cgiIn;
733	764	xfer.pOut = cgi_output_blob();
734		- xfer.mxSend = db_get_int("max-download", 5000000);
	765	+ xfer.mxSend = db_get_int("max-download", 20000000);
735	766	g.xferPanic = 1;
736	767
737	768	db_begin_transaction();
738	769	db_multi_exec(
739	770	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
740	771	);
741		- zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
742		- @ # timestamp %s(zNow)
743	772	manifest_crosslink_begin();
744	773	while( blob_line(xfer.pIn, &xfer.line) ){
745	774	if( blob_buffer(&xfer.line)[0]=='#' ){
746	775	if( lenPushHookPattern
747	776	&& blob_buffer(&xfer.line)[1]
		@@ -765,11 +794,11 @@
765	794	cgi_reset_content();
766	795	@ error not\sauthorized\sto\swrite
767	796	nErr++;
768	797	break;
769	798	}
770		- xfer_accept_file(&xfer);
	799	+ xfer_accept_file(&xfer, 0);
771	800	if( blob_size(&xfer.err) ){
772	801	cgi_reset_content();
773	802	@ error %T(blob_str(&xfer.err))
774	803	nErr++;
775	804	break;
		@@ -851,26 +880,42 @@
851	880	isPush = 1;
852	881	}
853	882	}
854	883	}else
855	884
856		- /* clone
	885	+ /* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
857	886	**
858	887	** The client knows nothing. Tell all.
859	888	*/
860	889	if( blob_eq(&xfer.aToken[0], "clone") ){
	890	+ int iVers;
861	891	login_check_credentials();
862	892	if( !g.okClone ){
863	893	cgi_reset_content();
864	894	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
865	895	@ error not\sauthorized\sto\sclone
866	896	nErr++;
867	897	break;
868	898	}
869		- isClone = 1;
870		- isPull = 1;
871		- deltaFlag = 1;
	899	+ if( xfer.nToken==3
	900	+ && blob_is_int(&xfer.aToken[1], &iVers)
	901	+ && iVers>=2
	902	+ ){
	903	+ int seqno, max;
	904	+ blob_is_int(&xfer.aToken[2], &seqno);
	905	+ max = db_int(0, "SELECT max(rid) FROM blob");
	906	+ while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
	907	+ send_file(&xfer, seqno, 0, 1);
	908	+ seqno++;
	909	+ }
	910	+ if( seqno>=max ) seqno = 0;
	911	+ @ clone_seqno %d(seqno)
	912	+ }else{
	913	+ isClone = 1;
	914	+ isPull = 1;
	915	+ deltaFlag = 1;
	916	+ }
872	917	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
873	918	}else
874	919
875	920	/* login USER NONCE SIGNATURE
876	921	**
		@@ -1007,10 +1052,18 @@
1007	1052	}
1008	1053	if( recvConfig ){
1009	1054	configure_finalize_receive();
1010	1055	}
1011	1056	manifest_crosslink_end();
	1057	+
	1058	+ /* Send the server timestamp last, in case prior processing happened
	1059	+ ** to use up a significant fraction of our time window.
	1060	+ */
	1061	+ zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
	1062	+ @ # timestamp %s(zNow)
	1063	+ free(zNow);
	1064	+
1012	1065	db_end_transaction(0);
1013	1066	}
1014	1067
1015	1068	/*
1016	1069	** COMMAND: test-xfer
		@@ -1076,13 +1129,17 @@
1076	1129	int origConfigRcvMask; /* Original value of configRcvMask */
1077	1130	int nFileRecv; /* Number of files received */
1078	1131	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1079	1132	const char zCookie; / Server cookie */
1080	1133	int nSent, nRcvd; /* Bytes sent and received (after compression) */
	1134	+ int cloneSeqno = 1; /* Sequence number for clones */
1081	1135	Blob send; /* Text we are sending to the server */
1082	1136	Blob recv; /* Reply we got back from the server */
1083	1137	Xfer xfer; /* Transfer data */
	1138	+ int pctDone; /* Percentage done with a message */
	1139	+ int lastPctDone = -1; /* Last displayed pctDone */
	1140	+ double rArrivalTime; /* Time at which a message arrived */
1084	1141	const char *zSCode = db_get("server-code", "x");
1085	1142	const char *zPCode = db_get("project-code", 0);
1086	1143	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1087	1144	int allowForced = db_get_boolean("push-hook-force", 0);
1088	1145
		@@ -1113,15 +1170,16 @@
1113	1170
1114	1171	/*
1115	1172	** Always begin with a clone, pull, or push message
1116	1173	*/
1117	1174	if( cloneFlag ){
1118		- blob_appendf(&send, "clone\n");
	1175	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1119	1176	pushFlag = 0;
1120	1177	pullFlag = 0;
1121	1178	nCardSent++;
1122	1179	/* TBD: Request all transferable configuration values */
	1180	+ content_enable_dephantomize(0);
1123	1181	}else if( pullFlag ){
1124	1182	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1125	1183	nCardSent++;
1126	1184	}
1127	1185	if( pushFlag ){
		@@ -1145,11 +1203,11 @@
1145	1203	}
1146	1204
1147	1205	/* Generate gimme cards for phantoms and leaf cards
1148	1206	** for all leaves.
1149	1207	*/
1150		- if( pullFlag \|\| cloneFlag ){
	1208	+ if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1151	1209	request_phantoms(&xfer, mxPhantomReq);
1152	1210	}
1153	1211	if( pushFlag ){
1154	1212	send_unsent(&xfer);
1155	1213	nCardSent += send_unclustered(&xfer);
		@@ -1194,22 +1252,27 @@
1194	1252	blob_appendf(&send, "# %s\n", zRandomness);
1195	1253	free(zRandomness);
1196	1254
1197	1255	/* Exchange messages with the server */
1198	1256	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1199		- fossil_print(zValueFormat, "Send:",
	1257	+ fossil_print(zValueFormat, "Sent:",
1200	1258	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1201	1259	xfer.nFileSent, xfer.nDeltaSent);
1202	1260	nCardSent = 0;
1203	1261	nCardRcvd = 0;
1204	1262	xfer.nFileSent = 0;
1205	1263	xfer.nDeltaSent = 0;
1206	1264	xfer.nGimmeSent = 0;
1207	1265	xfer.nIGotSent = 0;
	1266	+ if( !g.cgiOutput && !g.fQuiet ){
	1267	+ printf("waiting for server...");
	1268	+ }
1208	1269	fflush(stdout);
1209	1270	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
	1271	+ lastPctDone = -1;
1210	1272	blob_reset(&send);
	1273	+ rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1211	1274
1212	1275	/* Begin constructing the next message (which might never be
1213	1276	** sent) by beginning with the pull or push cards
1214	1277	*/
1215	1278	if( pullFlag ){
		@@ -1222,18 +1285,22 @@
1222	1285	}
1223	1286	go = 0;
1224	1287
1225	1288	/* Process the reply that came back from the server */
1226	1289	while( blob_line(&recv, &xfer.line) ){
	1290	+ if( g.fHttpTrace ){
	1291	+ printf("\rGOT: %.*s", (int)blob_size(&xfer.line),
	1292	+ blob_buffer(&xfer.line));
	1293	+ }
1227	1294	if( blob_buffer(&xfer.line)[0]=='#' ){
1228	1295	const char *zLine = blob_buffer(&xfer.line);
1229	1296	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1230	1297	char zTime[20];
1231	1298	double rDiff;
1232	1299	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1233		- rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1234		- zTime);
	1300	+ rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
	1301	+ zTime, rArrivalTime);
1235	1302	if( rDiff<0.0 ) rDiff = -rDiff;
1236	1303	if( rDiff>9e98 ) rDiff = 0.0;
1237	1304	if( (rDiff24.03600.0)>=60.0 ){
1238	1305	fossil_warning("* time skew * server time differs by %s",
1239	1306	db_timespan_name(rDiff));
		@@ -1243,21 +1310,25 @@
1243	1310	continue;
1244	1311	}
1245	1312	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1246	1313	nCardRcvd++;
1247	1314	if( !g.cgiOutput && !g.fQuiet ){
1248		- printf("\r%d", nCardRcvd);
1249		- fflush(stdout);
	1315	+ pctDone = (recv.iCursor*100)/recv.nUsed;
	1316	+ if( pctDone!=lastPctDone ){
	1317	+ printf("\rprocessed: %d%% ", pctDone);
	1318	+ lastPctDone = pctDone;
	1319	+ fflush(stdout);
	1320	+ }
1250	1321	}
1251	1322
1252	1323	/* file UUID SIZE \n CONTENT
1253	1324	** file UUID DELTASRC SIZE \n CONTENT
1254	1325	**
1255	1326	** Receive a file transmitted from the server.
1256	1327	*/
1257	1328	if( blob_eq(&xfer.aToken[0],"file") ){
1258		- xfer_accept_file(&xfer);
	1329	+ xfer_accept_file(&xfer, cloneFlag);
1259	1330	}else
1260	1331
1261	1332	/* gimme UUID
1262	1333	**
1263	1334	** Server is requesting a file. If the file is a manifest, assume
		@@ -1313,11 +1384,11 @@
1313	1384	}
1314	1385	if( zPCode==0 ){
1315	1386	zPCode = mprintf("%b", &xfer.aToken[2]);
1316	1387	db_set("project-code", zPCode, 0);
1317	1388	}
1318		- blob_appendf(&send, "clone\n");
	1389	+ blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1319	1390	nCardSent++;
1320	1391	}else
1321	1392
1322	1393	/* config NAME SIZE \n CONTENT
1323	1394	**
		@@ -1372,10 +1443,21 @@
1372	1443	** same server.
1373	1444	*/
1374	1445	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1375	1446	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1376	1447	}else
	1448	+
	1449	+ /* clone_seqno N
	1450	+ **
	1451	+ ** When doing a clone, the server tries to send all of its artifacts
	1452	+ ** in sequence. This card indicates the sequence number of the next
	1453	+ ** blob that needs to be sent. If N<=0 that indicates that all blobs
	1454	+ ** have been sent.
	1455	+ */
	1456	+ if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
	1457	+ blob_is_int(&xfer.aToken[1], &cloneSeqno);
	1458	+ }else
1377	1459
1378	1460	/* message MESSAGE
1379	1461	**
1380	1462	** Print a message. Similar to "error" but does not stop processing.
1381	1463	**
		@@ -1450,10 +1532,12 @@
1450	1532	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1451	1533	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1452	1534	go = 1;
1453	1535	mxPhantomReq = nFileRecv*2;
1454	1536	if( mxPhantomReq<200 ) mxPhantomReq = 200;
	1537	+ }else if( cloneFlag && nFileRecv>0 ){
	1538	+ go = 1;
1455	1539	}
1456	1540	nCardRcvd = 0;
1457	1541	xfer.nFileRcvd = 0;
1458	1542	xfer.nDeltaRcvd = 0;
1459	1543	xfer.nDanglingFile = 0;
		@@ -1465,10 +1549,13 @@
1465	1549	go = 1;
1466	1550	}
1467	1551
1468	1552	/* If this is a clone, the go at least two rounds */
1469	1553	if( cloneFlag && nCycle==1 ) go = 1;
	1554	+
	1555	+ /* Stop the cycle if the server sends a "clone_seqno 0" card */
	1556	+ if( cloneSeqno<=0 ) go = 0;
1470	1557	};
1471	1558	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1472	1559	if( zPushHookPattern && zPushHookPattern[0] ){
1473	1560	blob_appendf(&send, "#%s%s\n",
1474	1561	((nFileSend > 0)?"P":"F"), zPushHookPattern);
		@@ -1484,7 +1571,8 @@
1484	1571	nSent, nRcvd);
1485	1572	transport_close();
1486	1573	transport_global_shutdown();
1487	1574	db_multi_exec("DROP TABLE onremote");
1488	1575	manifest_crosslink_end();
	1576	+ content_enable_dephantomize(1);
1489	1577	db_end_transaction(0);
1490	1578	}
1491	1579

	--- src/xfer.c
	+++ src/xfer.c
	@@ -192,11 +192,17 @@
192	/*
193	** Remember that the other side of the connection already has a copy
194	** of the file rid.
195	*/
196	static void remote_has(int rid){
197	if( rid ) db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);






198	}
199
200	/*
201	** The aToken[0..nToken-1] blob array is a parse of a "file" line
202	** message. This routine finishes parsing that message and does
	@@ -215,11 +221,11 @@
215	** be initialized to an empty string.
216	**
217	** Any artifact successfully received by this routine is considered to
218	** be public and is therefore removed from the "private" table.
219	*/
220	static void xfer_accept_file(Xfer *pXfer){
221	int n;
222	int rid;
223	int srcid = 0;
224	Blob content, hash;
225
	@@ -234,13 +240,26 @@
234	return;
235	}
236	blob_zero(&content);
237	blob_zero(&hash);
238	blob_extract(pXfer->pIn, n, &content);
239	if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
240	/* Ignore files that have been shunned */
241	return;













242	}
243	if( pXfer->nToken==4 ){
244	Blob src, next;
245	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
246	if( content_get(srcid, &src)==0 ){
	@@ -587,14 +606,15 @@
587	** Check to see if the number of unclustered entries is greater than
588	** 100 and if it is, form a new cluster. Unclustered phantoms do not
589	** count toward the 100 total. And phantoms are never added to a new
590	** cluster.
591	*/
592	static void create_cluster(void){
593	Blob cluster, cksum;
594	Stmt q;
595	int nUncl;

596
597	/* We should not ever get any private artifacts in the unclustered table.
598	** But if we do (because of a bug) now is a good time to delete them. */
599	db_multi_exec(
600	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -601,30 +621,41 @@
601	);
602
603	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
604	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
605	" WHERE rid=unclustered.rid)");
606	if( nUncl<100 ){
607	return;
608	}
609	blob_zero(&cluster);
610	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
611	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
612	" WHERE rid=unclustered.rid)"
613	" AND unclustered.rid=blob.rid"
614	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
615	" ORDER BY 1");
616	while( db_step(&q)==SQLITE_ROW ){
617	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
618	}
619	db_finalize(&q);
620	md5sum_blob(&cluster, &cksum);
621	blob_appendf(&cluster, "Z %b\n", &cksum);
622	blob_reset(&cksum);
623	db_multi_exec("DELETE FROM unclustered");
624	content_put(&cluster, 0, 0);
625	blob_reset(&cluster);











626	}
627
628	/*
629	** Send an igot message for every entry in unclustered table.
630	** Return the number of cards sent.
	@@ -729,19 +760,17 @@
729	blobarray_zero(xfer.aToken, count(xfer.aToken));
730	cgi_set_content_type(g.zContentType);
731	blob_zero(&xfer.err);
732	xfer.pIn = &g.cgiIn;
733	xfer.pOut = cgi_output_blob();
734	xfer.mxSend = db_get_int("max-download", 5000000);
735	g.xferPanic = 1;
736
737	db_begin_transaction();
738	db_multi_exec(
739	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
740	);
741	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
742	@ # timestamp %s(zNow)
743	manifest_crosslink_begin();
744	while( blob_line(xfer.pIn, &xfer.line) ){
745	if( blob_buffer(&xfer.line)[0]=='#' ){
746	if( lenPushHookPattern
747	&& blob_buffer(&xfer.line)[1]
	@@ -765,11 +794,11 @@
765	cgi_reset_content();
766	@ error not\sauthorized\sto\swrite
767	nErr++;
768	break;
769	}
770	xfer_accept_file(&xfer);
771	if( blob_size(&xfer.err) ){
772	cgi_reset_content();
773	@ error %T(blob_str(&xfer.err))
774	nErr++;
775	break;
	@@ -851,26 +880,42 @@
851	isPush = 1;
852	}
853	}
854	}else
855
856	/* clone
857	**
858	** The client knows nothing. Tell all.
859	*/
860	if( blob_eq(&xfer.aToken[0], "clone") ){

861	login_check_credentials();
862	if( !g.okClone ){
863	cgi_reset_content();
864	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
865	@ error not\sauthorized\sto\sclone
866	nErr++;
867	break;
868	}
869	isClone = 1;
870	isPull = 1;
871	deltaFlag = 1;















872	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
873	}else
874
875	/* login USER NONCE SIGNATURE
876	**
	@@ -1007,10 +1052,18 @@
1007	}
1008	if( recvConfig ){
1009	configure_finalize_receive();
1010	}
1011	manifest_crosslink_end();








1012	db_end_transaction(0);
1013	}
1014
1015	/*
1016	** COMMAND: test-xfer
	@@ -1076,13 +1129,17 @@
1076	int origConfigRcvMask; /* Original value of configRcvMask */
1077	int nFileRecv; /* Number of files received */
1078	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1079	const char zCookie; / Server cookie */
1080	int nSent, nRcvd; /* Bytes sent and received (after compression) */

1081	Blob send; /* Text we are sending to the server */
1082	Blob recv; /* Reply we got back from the server */
1083	Xfer xfer; /* Transfer data */



1084	const char *zSCode = db_get("server-code", "x");
1085	const char *zPCode = db_get("project-code", 0);
1086	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1087	int allowForced = db_get_boolean("push-hook-force", 0);
1088
	@@ -1113,15 +1170,16 @@
1113
1114	/*
1115	** Always begin with a clone, pull, or push message
1116	*/
1117	if( cloneFlag ){
1118	blob_appendf(&send, "clone\n");
1119	pushFlag = 0;
1120	pullFlag = 0;
1121	nCardSent++;
1122	/* TBD: Request all transferable configuration values */

1123	}else if( pullFlag ){
1124	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1125	nCardSent++;
1126	}
1127	if( pushFlag ){
	@@ -1145,11 +1203,11 @@
1145	}
1146
1147	/* Generate gimme cards for phantoms and leaf cards
1148	** for all leaves.
1149	*/
1150	if( pullFlag \|\| cloneFlag ){
1151	request_phantoms(&xfer, mxPhantomReq);
1152	}
1153	if( pushFlag ){
1154	send_unsent(&xfer);
1155	nCardSent += send_unclustered(&xfer);
	@@ -1194,22 +1252,27 @@
1194	blob_appendf(&send, "# %s\n", zRandomness);
1195	free(zRandomness);
1196
1197	/* Exchange messages with the server */
1198	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1199	fossil_print(zValueFormat, "Send:",
1200	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1201	xfer.nFileSent, xfer.nDeltaSent);
1202	nCardSent = 0;
1203	nCardRcvd = 0;
1204	xfer.nFileSent = 0;
1205	xfer.nDeltaSent = 0;
1206	xfer.nGimmeSent = 0;
1207	xfer.nIGotSent = 0;



1208	fflush(stdout);
1209	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);

1210	blob_reset(&send);

1211
1212	/* Begin constructing the next message (which might never be
1213	** sent) by beginning with the pull or push cards
1214	*/
1215	if( pullFlag ){
	@@ -1222,18 +1285,22 @@
1222	}
1223	go = 0;
1224
1225	/* Process the reply that came back from the server */
1226	while( blob_line(&recv, &xfer.line) ){




1227	if( blob_buffer(&xfer.line)[0]=='#' ){
1228	const char *zLine = blob_buffer(&xfer.line);
1229	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1230	char zTime[20];
1231	double rDiff;
1232	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1233	rDiff = db_double(9e99, "SELECT julianday('%q') - julianday('now')",
1234	zTime);
1235	if( rDiff<0.0 ) rDiff = -rDiff;
1236	if( rDiff>9e98 ) rDiff = 0.0;
1237	if( (rDiff24.03600.0)>=60.0 ){
1238	fossil_warning("* time skew * server time differs by %s",
1239	db_timespan_name(rDiff));
	@@ -1243,21 +1310,25 @@
1243	continue;
1244	}
1245	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1246	nCardRcvd++;
1247	if( !g.cgiOutput && !g.fQuiet ){
1248	printf("\r%d", nCardRcvd);
1249	fflush(stdout);




1250	}
1251
1252	/* file UUID SIZE \n CONTENT
1253	** file UUID DELTASRC SIZE \n CONTENT
1254	**
1255	** Receive a file transmitted from the server.
1256	*/
1257	if( blob_eq(&xfer.aToken[0],"file") ){
1258	xfer_accept_file(&xfer);
1259	}else
1260
1261	/* gimme UUID
1262	**
1263	** Server is requesting a file. If the file is a manifest, assume
	@@ -1313,11 +1384,11 @@
1313	}
1314	if( zPCode==0 ){
1315	zPCode = mprintf("%b", &xfer.aToken[2]);
1316	db_set("project-code", zPCode, 0);
1317	}
1318	blob_appendf(&send, "clone\n");
1319	nCardSent++;
1320	}else
1321
1322	/* config NAME SIZE \n CONTENT
1323	**
	@@ -1372,10 +1443,21 @@
1372	** same server.
1373	*/
1374	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1375	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1376	}else











1377
1378	/* message MESSAGE
1379	**
1380	** Print a message. Similar to "error" but does not stop processing.
1381	**
	@@ -1450,10 +1532,12 @@
1450	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1451	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1452	go = 1;
1453	mxPhantomReq = nFileRecv*2;
1454	if( mxPhantomReq<200 ) mxPhantomReq = 200;


1455	}
1456	nCardRcvd = 0;
1457	xfer.nFileRcvd = 0;
1458	xfer.nDeltaRcvd = 0;
1459	xfer.nDanglingFile = 0;
	@@ -1465,10 +1549,13 @@
1465	go = 1;
1466	}
1467
1468	/* If this is a clone, the go at least two rounds */
1469	if( cloneFlag && nCycle==1 ) go = 1;



1470	};
1471	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1472	if( zPushHookPattern && zPushHookPattern[0] ){
1473	blob_appendf(&send, "#%s%s\n",
1474	((nFileSend > 0)?"P":"F"), zPushHookPattern);
	@@ -1484,7 +1571,8 @@
1484	nSent, nRcvd);
1485	transport_close();
1486	transport_global_shutdown();
1487	db_multi_exec("DROP TABLE onremote");
1488	manifest_crosslink_end();

1489	db_end_transaction(0);
1490	}
1491

	--- src/xfer.c
	+++ src/xfer.c
	@@ -192,11 +192,17 @@
192	/*
193	** Remember that the other side of the connection already has a copy
194	** of the file rid.
195	*/
196	static void remote_has(int rid){
197	if( rid ){
198	static Stmt q;
199	db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
200	db_bind_int(&q, ":r", rid);
201	db_step(&q);
202	db_reset(&q);
203	}
204	}
205
206	/*
207	** The aToken[0..nToken-1] blob array is a parse of a "file" line
208	** message. This routine finishes parsing that message and does
	@@ -215,11 +221,11 @@
221	** be initialized to an empty string.
222	**
223	** Any artifact successfully received by this routine is considered to
224	** be public and is therefore removed from the "private" table.
225	*/
226	static void xfer_accept_file(Xfer *pXfer, int cloneFlag){
227	int n;
228	int rid;
229	int srcid = 0;
230	Blob content, hash;
231
	@@ -234,13 +240,26 @@
240	return;
241	}
242	blob_zero(&content);
243	blob_zero(&hash);
244	blob_extract(pXfer->pIn, n, &content);
245	if( !cloneFlag && uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
246	/* Ignore files that have been shunned */
247	return;
248	}
249	if( cloneFlag ){
250	if( pXfer->nToken==4 ){
251	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
252	pXfer->nDeltaRcvd++;
253	}else{
254	srcid = 0;
255	pXfer->nFileRcvd++;
256	}
257	rid = content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
258	remote_has(rid);
259	blob_reset(&content);
260	return;
261	}
262	if( pXfer->nToken==4 ){
263	Blob src, next;
264	srcid = rid_from_uuid(&pXfer->aToken[2], 1);
265	if( content_get(srcid, &src)==0 ){
	@@ -587,14 +606,15 @@
606	** Check to see if the number of unclustered entries is greater than
607	** 100 and if it is, form a new cluster. Unclustered phantoms do not
608	** count toward the 100 total. And phantoms are never added to a new
609	** cluster.
610	*/
611	void create_cluster(void){
612	Blob cluster, cksum;
613	Stmt q;
614	int nUncl;
615	int nRow = 0;
616
617	/* We should not ever get any private artifacts in the unclustered table.
618	** But if we do (because of a bug) now is a good time to delete them. */
619	db_multi_exec(
620	"DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
	@@ -601,30 +621,41 @@
621	);
622
623	nUncl = db_int(0, "SELECT count() FROM unclustered /scan*/"
624	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
625	" WHERE rid=unclustered.rid)");
626	if( nUncl>=100 ){
627	blob_zero(&cluster);
628	db_prepare(&q, "SELECT uuid FROM unclustered, blob"
629	" WHERE NOT EXISTS(SELECT 1 FROM phantom"
630	" WHERE rid=unclustered.rid)"
631	" AND unclustered.rid=blob.rid"
632	" AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
633	" ORDER BY 1");
634	while( db_step(&q)==SQLITE_ROW ){
635	blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
636	nRow++;
637	if( nRow>=800 && nUncl>nRow+100 ){
638	md5sum_blob(&cluster, &cksum);
639	blob_appendf(&cluster, "Z %b\n", &cksum);
640	blob_reset(&cksum);
641	content_put(&cluster, 0, 0);
642	blob_reset(&cluster);
643	nUncl -= nRow;
644	nRow = 0;
645	}
646	}
647	db_finalize(&q);
648	db_multi_exec("DELETE FROM unclustered");
649	if( nRow>0 ){
650	md5sum_blob(&cluster, &cksum);
651	blob_appendf(&cluster, "Z %b\n", &cksum);
652	blob_reset(&cksum);
653	content_put(&cluster, 0, 0);
654	blob_reset(&cluster);
655	}
656	}
657	}
658
659	/*
660	** Send an igot message for every entry in unclustered table.
661	** Return the number of cards sent.
	@@ -729,19 +760,17 @@
760	blobarray_zero(xfer.aToken, count(xfer.aToken));
761	cgi_set_content_type(g.zContentType);
762	blob_zero(&xfer.err);
763	xfer.pIn = &g.cgiIn;
764	xfer.pOut = cgi_output_blob();
765	xfer.mxSend = db_get_int("max-download", 20000000);
766	g.xferPanic = 1;
767
768	db_begin_transaction();
769	db_multi_exec(
770	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
771	);


772	manifest_crosslink_begin();
773	while( blob_line(xfer.pIn, &xfer.line) ){
774	if( blob_buffer(&xfer.line)[0]=='#' ){
775	if( lenPushHookPattern
776	&& blob_buffer(&xfer.line)[1]
	@@ -765,11 +794,11 @@
794	cgi_reset_content();
795	@ error not\sauthorized\sto\swrite
796	nErr++;
797	break;
798	}
799	xfer_accept_file(&xfer, 0);
800	if( blob_size(&xfer.err) ){
801	cgi_reset_content();
802	@ error %T(blob_str(&xfer.err))
803	nErr++;
804	break;
	@@ -851,26 +880,42 @@
880	isPush = 1;
881	}
882	}
883	}else
884
885	/* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
886	**
887	** The client knows nothing. Tell all.
888	*/
889	if( blob_eq(&xfer.aToken[0], "clone") ){
890	int iVers;
891	login_check_credentials();
892	if( !g.okClone ){
893	cgi_reset_content();
894	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
895	@ error not\sauthorized\sto\sclone
896	nErr++;
897	break;
898	}
899	if( xfer.nToken==3
900	&& blob_is_int(&xfer.aToken[1], &iVers)
901	&& iVers>=2
902	){
903	int seqno, max;
904	blob_is_int(&xfer.aToken[2], &seqno);
905	max = db_int(0, "SELECT max(rid) FROM blob");
906	while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max ){
907	send_file(&xfer, seqno, 0, 1);
908	seqno++;
909	}
910	if( seqno>=max ) seqno = 0;
911	@ clone_seqno %d(seqno)
912	}else{
913	isClone = 1;
914	isPull = 1;
915	deltaFlag = 1;
916	}
917	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
918	}else
919
920	/* login USER NONCE SIGNATURE
921	**
	@@ -1007,10 +1052,18 @@
1052	}
1053	if( recvConfig ){
1054	configure_finalize_receive();
1055	}
1056	manifest_crosslink_end();
1057
1058	/* Send the server timestamp last, in case prior processing happened
1059	** to use up a significant fraction of our time window.
1060	*/
1061	zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
1062	@ # timestamp %s(zNow)
1063	free(zNow);
1064
1065	db_end_transaction(0);
1066	}
1067
1068	/*
1069	** COMMAND: test-xfer
	@@ -1076,13 +1129,17 @@
1129	int origConfigRcvMask; /* Original value of configRcvMask */
1130	int nFileRecv; /* Number of files received */
1131	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1132	const char zCookie; / Server cookie */
1133	int nSent, nRcvd; /* Bytes sent and received (after compression) */
1134	int cloneSeqno = 1; /* Sequence number for clones */
1135	Blob send; /* Text we are sending to the server */
1136	Blob recv; /* Reply we got back from the server */
1137	Xfer xfer; /* Transfer data */
1138	int pctDone; /* Percentage done with a message */
1139	int lastPctDone = -1; /* Last displayed pctDone */
1140	double rArrivalTime; /* Time at which a message arrived */
1141	const char *zSCode = db_get("server-code", "x");
1142	const char *zPCode = db_get("project-code", 0);
1143	const char *zPushHookPattern = db_get("push-hook-pattern-client", "");
1144	int allowForced = db_get_boolean("push-hook-force", 0);
1145
	@@ -1113,15 +1170,16 @@
1170
1171	/*
1172	** Always begin with a clone, pull, or push message
1173	*/
1174	if( cloneFlag ){
1175	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1176	pushFlag = 0;
1177	pullFlag = 0;
1178	nCardSent++;
1179	/* TBD: Request all transferable configuration values */
1180	content_enable_dephantomize(0);
1181	}else if( pullFlag ){
1182	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
1183	nCardSent++;
1184	}
1185	if( pushFlag ){
	@@ -1145,11 +1203,11 @@
1203	}
1204
1205	/* Generate gimme cards for phantoms and leaf cards
1206	** for all leaves.
1207	*/
1208	if( pullFlag \|\| (cloneFlag && cloneSeqno==1) ){
1209	request_phantoms(&xfer, mxPhantomReq);
1210	}
1211	if( pushFlag ){
1212	send_unsent(&xfer);
1213	nCardSent += send_unclustered(&xfer);
	@@ -1194,22 +1252,27 @@
1252	blob_appendf(&send, "# %s\n", zRandomness);
1253	free(zRandomness);
1254
1255	/* Exchange messages with the server */
1256	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
1257	fossil_print(zValueFormat, "Sent:",
1258	blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
1259	xfer.nFileSent, xfer.nDeltaSent);
1260	nCardSent = 0;
1261	nCardRcvd = 0;
1262	xfer.nFileSent = 0;
1263	xfer.nDeltaSent = 0;
1264	xfer.nGimmeSent = 0;
1265	xfer.nIGotSent = 0;
1266	if( !g.cgiOutput && !g.fQuiet ){
1267	printf("waiting for server...");
1268	}
1269	fflush(stdout);
1270	http_exchange(&send, &recv, cloneFlag==0 \|\| nCycle>0);
1271	lastPctDone = -1;
1272	blob_reset(&send);
1273	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
1274
1275	/* Begin constructing the next message (which might never be
1276	** sent) by beginning with the pull or push cards
1277	*/
1278	if( pullFlag ){
	@@ -1222,18 +1285,22 @@
1285	}
1286	go = 0;
1287
1288	/* Process the reply that came back from the server */
1289	while( blob_line(&recv, &xfer.line) ){
1290	if( g.fHttpTrace ){
1291	printf("\rGOT: %.*s", (int)blob_size(&xfer.line),
1292	blob_buffer(&xfer.line));
1293	}
1294	if( blob_buffer(&xfer.line)[0]=='#' ){
1295	const char *zLine = blob_buffer(&xfer.line);
1296	if( memcmp(zLine, "# timestamp ", 12)==0 ){
1297	char zTime[20];
1298	double rDiff;
1299	sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
1300	rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
1301	zTime, rArrivalTime);
1302	if( rDiff<0.0 ) rDiff = -rDiff;
1303	if( rDiff>9e98 ) rDiff = 0.0;
1304	if( (rDiff24.03600.0)>=60.0 ){
1305	fossil_warning("* time skew * server time differs by %s",
1306	db_timespan_name(rDiff));
	@@ -1243,21 +1310,25 @@
1310	continue;
1311	}
1312	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1313	nCardRcvd++;
1314	if( !g.cgiOutput && !g.fQuiet ){
1315	pctDone = (recv.iCursor*100)/recv.nUsed;
1316	if( pctDone!=lastPctDone ){
1317	printf("\rprocessed: %d%% ", pctDone);
1318	lastPctDone = pctDone;
1319	fflush(stdout);
1320	}
1321	}
1322
1323	/* file UUID SIZE \n CONTENT
1324	** file UUID DELTASRC SIZE \n CONTENT
1325	**
1326	** Receive a file transmitted from the server.
1327	*/
1328	if( blob_eq(&xfer.aToken[0],"file") ){
1329	xfer_accept_file(&xfer, cloneFlag);
1330	}else
1331
1332	/* gimme UUID
1333	**
1334	** Server is requesting a file. If the file is a manifest, assume
	@@ -1313,11 +1384,11 @@
1384	}
1385	if( zPCode==0 ){
1386	zPCode = mprintf("%b", &xfer.aToken[2]);
1387	db_set("project-code", zPCode, 0);
1388	}
1389	blob_appendf(&send, "clone 2 %d\n", cloneSeqno);
1390	nCardSent++;
1391	}else
1392
1393	/* config NAME SIZE \n CONTENT
1394	**
	@@ -1372,10 +1443,21 @@
1443	** same server.
1444	*/
1445	if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1446	db_set("cookie", blob_str(&xfer.aToken[1]), 0);
1447	}else
1448
1449	/* clone_seqno N
1450	**
1451	** When doing a clone, the server tries to send all of its artifacts
1452	** in sequence. This card indicates the sequence number of the next
1453	** blob that needs to be sent. If N<=0 that indicates that all blobs
1454	** have been sent.
1455	*/
1456	if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
1457	blob_is_int(&xfer.aToken[1], &cloneSeqno);
1458	}else
1459
1460	/* message MESSAGE
1461	**
1462	** Print a message. Similar to "error" but does not stop processing.
1463	**
	@@ -1450,10 +1532,12 @@
1532	nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
1533	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1534	go = 1;
1535	mxPhantomReq = nFileRecv*2;
1536	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1537	}else if( cloneFlag && nFileRecv>0 ){
1538	go = 1;
1539	}
1540	nCardRcvd = 0;
1541	xfer.nFileRcvd = 0;
1542	xfer.nDeltaRcvd = 0;
1543	xfer.nDanglingFile = 0;
	@@ -1465,10 +1549,13 @@
1549	go = 1;
1550	}
1551
1552	/* If this is a clone, the go at least two rounds */
1553	if( cloneFlag && nCycle==1 ) go = 1;
1554
1555	/* Stop the cycle if the server sends a "clone_seqno 0" card */
1556	if( cloneSeqno<=0 ) go = 0;
1557	};
1558	if( pushFlag && ( (nFileSend > 0) \|\| allowForced ) ){
1559	if( zPushHookPattern && zPushHookPattern[0] ){
1560	blob_appendf(&send, "#%s%s\n",
1561	((nFileSend > 0)?"P":"F"), zPushHookPattern);
	@@ -1484,7 +1571,8 @@
1571	nSent, nRcvd);
1572	transport_close();
1573	transport_global_shutdown();
1574	db_multi_exec("DROP TABLE onremote");
1575	manifest_crosslink_end();
1576	content_enable_dephantomize(1);
1577	db_end_transaction(0);
1578	}
1579

Fossil SCM

Keyboard Shortcuts