Fossil SCM

Enhancements to the sync protocol designed to reduce bandwidth in cases where there are a lot of private artifacts on either side of the sync.

drh 2020-04-13 12:51 trunk merge

Commit 1f7b409e876abe26d2b931298bdf0933223e0cfdc757d8facfa7d7d8feb37267

Parent 648bc30531a76ff…

5 files changed +18 -3 +6 +1 -13 +137 -55 +7

~ src/mkversion.c ~ src/schema.c ~ src/sync.c ~ src/xfer.c ~ www/sync.wiki

M src/mkversion.c

+18 -3

		--- src/mkversion.c
		+++ src/mkversion.c
		@@ -8,10 +8,11 @@
8	8	** Note that the manifest.uuid and manifest files are generated by Fossil.
9	9	*/
10	10	#include <stdio.h>
11	11	#include <string.h>
12	12	#include <stdlib.h>
	13	+#include <ctype.h>
13	14
14	15	static FILE open_for_reading(const char zFilename){
15	16	FILE *f = fopen(zFilename, "r");
16	17	if( f==0 ){
17	18	fprintf(stderr, "cannot open \"%s\" for reading\n", zFilename);
		@@ -46,14 +47,28 @@
46	47	*z = 0;
47	48	printf("#define MANIFEST_UUID \"%s\"\n",b);
48	49	printf("#define MANIFEST_VERSION \"[%10.10s]\"\n",b);
49	50	m = open_for_reading(argv[2]);
50	51	while(b == fgets(b, sizeof(b)-1,m)){
51		- if(0 == strncmp("D ",b,2)){
52		- printf("#define MANIFEST_DATE \"%.10s %.8s\"\n",b+2,b+13);
53		- printf("#define MANIFEST_YEAR \"%.4s\"\n",b+2);
	52	+ if(0 == strncmp("D ",b,2)){
	53	+ int k, n;
	54	+ char zDateNum[30];
	55	+ printf("#define MANIFEST_DATE \"%.10s %.8s\"\n",b+2,b+13);
	56	+ printf("#define MANIFEST_YEAR \"%.4s\"\n",b+2);
	57	+ n = 0;
	58	+ for(k=0; k<10; k++){
	59	+ if( isdigit(b[k+2]) ) zDateNum[n++] = b[k+2];
	60	+ }
	61	+ zDateNum[n] = 0;
	62	+ printf("#define MANIFEST_NUMERIC_DATE %s\n", zDateNum);
	63	+ n = 0;
	64	+ for(k=0; k<8; k++){
	65	+ if( isdigit(b[k+13]) ) zDateNum[n++] = b[k+13];
54	66	}
	67	+ zDateNum[n] = 0;
	68	+ printf("#define MANIFEST_NUMERIC_TIME %s\n", zDateNum);
	69	+ }
55	70	}
56	71	fclose(m);
57	72	v = open_for_reading(argv[3]);
58	73	if( fgets(b, sizeof(b)-1,v)==0 ){
59	74	fprintf(stderr, "malformed VERSION file: %s\n", argv[3]);
60	75

	--- src/mkversion.c
	+++ src/mkversion.c
	@@ -8,10 +8,11 @@
8	** Note that the manifest.uuid and manifest files are generated by Fossil.
9	*/
10	#include <stdio.h>
11	#include <string.h>
12	#include <stdlib.h>

13
14	static FILE open_for_reading(const char zFilename){
15	FILE *f = fopen(zFilename, "r");
16	if( f==0 ){
17	fprintf(stderr, "cannot open \"%s\" for reading\n", zFilename);
	@@ -46,14 +47,28 @@
46	*z = 0;
47	printf("#define MANIFEST_UUID \"%s\"\n",b);
48	printf("#define MANIFEST_VERSION \"[%10.10s]\"\n",b);
49	m = open_for_reading(argv[2]);
50	while(b == fgets(b, sizeof(b)-1,m)){
51	if(0 == strncmp("D ",b,2)){
52	printf("#define MANIFEST_DATE \"%.10s %.8s\"\n",b+2,b+13);
53	printf("#define MANIFEST_YEAR \"%.4s\"\n",b+2);











54	}



55	}
56	fclose(m);
57	v = open_for_reading(argv[3]);
58	if( fgets(b, sizeof(b)-1,v)==0 ){
59	fprintf(stderr, "malformed VERSION file: %s\n", argv[3]);
60

	--- src/mkversion.c
	+++ src/mkversion.c
	@@ -8,10 +8,11 @@
8	** Note that the manifest.uuid and manifest files are generated by Fossil.
9	*/
10	#include <stdio.h>
11	#include <string.h>
12	#include <stdlib.h>
13	#include <ctype.h>
14
15	static FILE open_for_reading(const char zFilename){
16	FILE *f = fopen(zFilename, "r");
17	if( f==0 ){
18	fprintf(stderr, "cannot open \"%s\" for reading\n", zFilename);
	@@ -46,14 +47,28 @@
47	*z = 0;
48	printf("#define MANIFEST_UUID \"%s\"\n",b);
49	printf("#define MANIFEST_VERSION \"[%10.10s]\"\n",b);
50	m = open_for_reading(argv[2]);
51	while(b == fgets(b, sizeof(b)-1,m)){
52	if(0 == strncmp("D ",b,2)){
53	int k, n;
54	char zDateNum[30];
55	printf("#define MANIFEST_DATE \"%.10s %.8s\"\n",b+2,b+13);
56	printf("#define MANIFEST_YEAR \"%.4s\"\n",b+2);
57	n = 0;
58	for(k=0; k<10; k++){
59	if( isdigit(b[k+2]) ) zDateNum[n++] = b[k+2];
60	}
61	zDateNum[n] = 0;
62	printf("#define MANIFEST_NUMERIC_DATE %s\n", zDateNum);
63	n = 0;
64	for(k=0; k<8; k++){
65	if( isdigit(b[k+13]) ) zDateNum[n++] = b[k+13];
66	}
67	zDateNum[n] = 0;
68	printf("#define MANIFEST_NUMERIC_TIME %s\n", zDateNum);
69	}
70	}
71	fclose(m);
72	v = open_for_reading(argv[3]);
73	if( fgets(b, sizeof(b)-1,v)==0 ){
74	fprintf(stderr, "malformed VERSION file: %s\n", argv[3]);
75

M src/schema.c

		--- src/schema.c
		+++ src/schema.c
		@@ -158,10 +158,16 @@
158	158	@ );
159	159	@
160	160	@ -- Artifacts that should not be pushed are stored in the "private"
161	161	@ -- table. Private artifacts are omitted from the "unclustered" and
162	162	@ -- "unsent" tables.
	163	+@ --
	164	+@ -- A phantom artifact (that is, an artifact with BLOB.SIZE<0 - an artifact
	165	+@ -- for which we do not know the content) might also be marked as private.
	166	+@ -- This comes about when an artifact is named in a manifest or tag but
	167	+@ -- the content of that artifact is held privately by some other peer
	168	+@ -- repository.
163	169	@ --
164	170	@ CREATE TABLE private(rid INTEGER PRIMARY KEY);
165	171	@
166	172	@ -- An entry in this table describes a database query that generates a
167	173	@ -- table of tickets.
168	174

	--- src/schema.c
	+++ src/schema.c
	@@ -158,10 +158,16 @@
158	@ );
159	@
160	@ -- Artifacts that should not be pushed are stored in the "private"
161	@ -- table. Private artifacts are omitted from the "unclustered" and
162	@ -- "unsent" tables.






163	@ --
164	@ CREATE TABLE private(rid INTEGER PRIMARY KEY);
165	@
166	@ -- An entry in this table describes a database query that generates a
167	@ -- table of tickets.
168

	--- src/schema.c
	+++ src/schema.c
	@@ -158,10 +158,16 @@
158	@ );
159	@
160	@ -- Artifacts that should not be pushed are stored in the "private"
161	@ -- table. Private artifacts are omitted from the "unclustered" and
162	@ -- "unsent" tables.
163	@ --
164	@ -- A phantom artifact (that is, an artifact with BLOB.SIZE<0 - an artifact
165	@ -- for which we do not know the content) might also be marked as private.
166	@ -- This comes about when an artifact is named in a manifest or tag but
167	@ -- the content of that artifact is held privately by some other peer
168	@ -- repository.
169	@ --
170	@ CREATE TABLE private(rid INTEGER PRIMARY KEY);
171	@
172	@ -- An entry in this table describes a database query that generates a
173	@ -- table of tickets.
174

M src/sync.c

+1 -13

		--- src/sync.c
		+++ src/sync.c
		@@ -61,22 +61,10 @@
61	61	g.url.flags \|= URL_PROMPT_PW;
62	62	url_prompt_for_password();
63	63	}
64	64	g.zHttpAuth = get_httpauth();
65	65	url_remember();
66		-#if 0 /* Disabled for now */
67		- if( (flags & AUTOSYNC_PULL)!=0 && db_get_boolean("auto-shun",1) ){
68		- /* When doing an automatic pull, also automatically pull shuns from
69		- ** the server if pull_shuns is enabled.
70		- **
71		- ** TODO: What happens if the shun list gets really big?
72		- ** Maybe the shunning list should only be pulled on every 10th
73		- ** autosync, or something?
74		- */
75		- configSync = CONFIGSET_SHUN;
76		- }
77		-#endif
78	66	if( find_option("verbose","v",0)!=0 ) flags \|= SYNC_VERBOSE;
79	67	fossil_print("Autosync: %s\n", g.url.canonical);
80	68	url_enable_proxy("via proxy: ");
81	69	rc = client_sync(flags, configSync, 0, 0);
82	70	return rc;
		@@ -163,11 +151,11 @@
163	151	url_proxy_options();
164	152	clone_ssh_find_options();
165	153	if( !uvOnly ) db_find_and_open_repository(0, 0);
166	154	db_open_config(0, 1);
167	155	if( g.argc==2 ){
168		- if( db_get_boolean("auto-shun",1) ) configSync = CONFIGSET_SHUN;
	156	+ if( db_get_boolean("auto-shun",0) ) configSync = CONFIGSET_SHUN;
169	157	}else if( g.argc==3 ){
170	158	zUrl = g.argv[2];
171	159	}
172	160	if( ((*pSyncFlags) & (SYNC_PUSH\|SYNC_PULL))==(SYNC_PUSH\|SYNC_PULL)
173	161	&& db_get_boolean("uv-sync",0)
174	162

	--- src/sync.c
	+++ src/sync.c
	@@ -61,22 +61,10 @@
61	g.url.flags \|= URL_PROMPT_PW;
62	url_prompt_for_password();
63	}
64	g.zHttpAuth = get_httpauth();
65	url_remember();
66	#if 0 /* Disabled for now */
67	if( (flags & AUTOSYNC_PULL)!=0 && db_get_boolean("auto-shun",1) ){
68	/* When doing an automatic pull, also automatically pull shuns from
69	** the server if pull_shuns is enabled.
70	**
71	** TODO: What happens if the shun list gets really big?
72	** Maybe the shunning list should only be pulled on every 10th
73	** autosync, or something?
74	*/
75	configSync = CONFIGSET_SHUN;
76	}
77	#endif
78	if( find_option("verbose","v",0)!=0 ) flags \|= SYNC_VERBOSE;
79	fossil_print("Autosync: %s\n", g.url.canonical);
80	url_enable_proxy("via proxy: ");
81	rc = client_sync(flags, configSync, 0, 0);
82	return rc;
	@@ -163,11 +151,11 @@
163	url_proxy_options();
164	clone_ssh_find_options();
165	if( !uvOnly ) db_find_and_open_repository(0, 0);
166	db_open_config(0, 1);
167	if( g.argc==2 ){
168	if( db_get_boolean("auto-shun",1) ) configSync = CONFIGSET_SHUN;
169	}else if( g.argc==3 ){
170	zUrl = g.argv[2];
171	}
172	if( ((*pSyncFlags) & (SYNC_PUSH\|SYNC_PULL))==(SYNC_PUSH\|SYNC_PULL)
173	&& db_get_boolean("uv-sync",0)
174

	--- src/sync.c
	+++ src/sync.c
	@@ -61,22 +61,10 @@
61	g.url.flags \|= URL_PROMPT_PW;
62	url_prompt_for_password();
63	}
64	g.zHttpAuth = get_httpauth();
65	url_remember();












66	if( find_option("verbose","v",0)!=0 ) flags \|= SYNC_VERBOSE;
67	fossil_print("Autosync: %s\n", g.url.canonical);
68	url_enable_proxy("via proxy: ");
69	rc = client_sync(flags, configSync, 0, 0);
70	return rc;
	@@ -163,11 +151,11 @@
151	url_proxy_options();
152	clone_ssh_find_options();
153	if( !uvOnly ) db_find_and_open_repository(0, 0);
154	db_open_config(0, 1);
155	if( g.argc==2 ){
156	if( db_get_boolean("auto-shun",0) ) configSync = CONFIGSET_SHUN;
157	}else if( g.argc==3 ){
158	zUrl = g.argv[2];
159	}
160	if( ((*pSyncFlags) & (SYNC_PUSH\|SYNC_PULL))==(SYNC_PUSH\|SYNC_PULL)
161	&& db_get_boolean("uv-sync",0)
162

M src/xfer.c

+137 -55

		--- src/xfer.c
		+++ src/xfer.c
		@@ -49,11 +49,13 @@
49	49	int nDanglingFile; /* Number of dangling deltas received */
50	50	int mxSend; /* Stop sending "file" when pOut reaches this size */
51	51	int resync; /* Send igot cards for all holdings */
52	52	u8 syncPrivate; /* True to enable syncing private content */
53	53	u8 nextIsPrivate; /* If true, next "file" received is a private */
54		- u32 clientVersion; /* Version of the client software */
	54	+ u32 remoteVersion; /* Version of fossil running on the other side */
	55	+ u32 remoteDate; /* Date for specific client software edition */
	56	+ u32 remoteTime; /* Time of date correspoding on remoteDate */
55	57	time_t maxTime; /* Time when this transfer should be finished */
56	58	};
57	59
58	60
59	61	/*
		@@ -524,20 +526,19 @@
524	526	static void send_file(Xfer pXfer, int rid, Blob pUuid, int nativeDelta){
525	527	Blob content, uuid;
526	528	int size = 0;
527	529	int isPriv = content_is_private(rid);
528	530
529		- if( pXfer->syncPrivate==0 && isPriv ) return;
530	531	if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
531	532	return;
532	533	}
533	534	blob_zero(&uuid);
534	535	db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
535	536	if( blob_size(&uuid)==0 ){
536	537	return;
537	538	}
538		- if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->clientVersion<20000 ){
	539	+ if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->remoteVersion<20000 ){
539	540	xfer_cannot_send_sha3_error(pXfer);
540	541	return;
541	542	}
542	543	if( pUuid ){
543	544	if( blob_compare(pUuid, &uuid)!=0 ){
		@@ -548,10 +549,21 @@
548	549	pUuid = &uuid;
549	550	}
550	551	if( uuid_is_shunned(blob_str(pUuid)) ){
551	552	blob_reset(&uuid);
552	553	return;
	554	+ }
	555	+ if( isPriv && pXfer->syncPrivate==0 ){
	556	+ if( pXfer->remoteDate>=20200413 ){
	557	+ /* If the artifact is private and we are not doing a private sync,
	558	+ ** at least tell the other side that the artifact exists and is
	559	+ ** known to be private. But only do this for newer clients since
	560	+ ** older ones will throw an error if they get a private igot card
	561	+ ** and private syncing is disallowed */
	562	+ blob_appendf(pXfer->pOut, "igot %b 1\n", pUuid);
	563	+ }
	564	+ return;
553	565	}
554	566	if( (pXfer->maxTime != -1 && time(NULL) >= pXfer->maxTime) \|\|
555	567	pXfer->mxSend<=blob_size(pXfer->pOut) ){
556	568	const char *zFormat = isPriv ? "igot %b 1\n" : "igot %b\n";
557	569	blob_appendf(pXfer->pOut, zFormat /works-like:"%b"/, pUuid);
		@@ -626,11 +638,11 @@
626	638	szC = db_column_bytes(&q1, 2);
627	639	zContent = db_column_raw(&q1, 2);
628	640	srcIsPrivate = db_column_int(&q1, 3);
629	641	zDelta = db_column_text(&q1, 4);
630	642	if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
631		- if( pXfer->clientVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
	643	+ if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
632	644	xfer_cannot_send_sha3_error(pXfer);
633	645	db_reset(&q1);
634	646	return;
635	647	}
636	648	blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
		@@ -690,11 +702,11 @@
690	702	);
691	703	}
692	704	if( db_step(&q1)==SQLITE_ROW ){
693	705	sqlite3_int64 mtime = db_column_int64(&q1, 0);
694	706	const char *zHash = db_column_text(&q1, 1);
695		- if( pXfer->clientVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
	707	+ if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
696	708	xfer_cannot_send_sha3_error(pXfer);
697	709	db_reset(&q1);
698	710	return;
699	711	}
700	712	if( blob_size(pXfer->pOut)>=pXfer->mxSend ){
		@@ -956,30 +968,48 @@
956	968	}
957	969
958	970	/*
959	971	** Send an igot message for every entry in unclustered table.
960	972	** Return the number of cards sent.
	973	+**
	974	+** Except:
	975	+** * Do not send igot cards for shunned artifacts
	976	+** * Do not send igot cards for phantoms
	977	+** * Do not send igot cards for private artifacts
	978	+** * Do not send igot cards for any artifact that is in the
	979	+** ONREMOTE table, if that table exists.
	980	+**
	981	+** If the pXfer->resync flag is set, that means we are doing a "--verily"
	982	+** sync and all artifacts that don't meet the restrictions above should
	983	+** be sent.
961	984	*/
962	985	static int send_unclustered(Xfer *pXfer){
963	986	Stmt q;
964	987	int cnt = 0;
	988	+ const char *zExtra;
	989	+ if( db_table_exists("temp","onremote") ){
	990	+ zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
	991	+ }else{
	992	+ zExtra = "";
	993	+ }
965	994	if( pXfer->resync ){
966	995	db_prepare(&q,
967	996	"SELECT uuid, rid FROM blob"
968	997	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
969	998	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
970		- " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"
	999	+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s"
971	1000	" AND blob.rid<=%d"
972	1001	" ORDER BY blob.rid DESC",
973		- pXfer->resync
	1002	+ zExtra /safe-for-%s/, pXfer->resync
974	1003	);
975	1004	}else{
976	1005	db_prepare(&q,
977	1006	"SELECT uuid FROM unclustered JOIN blob USING(rid) /scan/"
978	1007	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
979	1008	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
980		- " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"
	1009	+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
	1010	+ zExtra /safe-for-%s/
981	1011	);
982	1012	}
983	1013	while( db_step(&q)==SQLITE_ROW ){
984	1014	blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
985	1015	cnt++;
		@@ -1191,11 +1221,11 @@
1191	1221	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1192	1222
1193	1223	/* file HASH SIZE \n CONTENT
1194	1224	** file HASH DELTASRC SIZE \n CONTENT
1195	1225	**
1196		- ** Accept a file from the client.
	1226	+ ** Server accepts a file from the client.
1197	1227	*/
1198	1228	if( blob_eq(&xfer.aToken[0], "file") ){
1199	1229	if( !isPush ){
1200	1230	cgi_reset_content();
1201	1231	@ error not\sauthorized\sto\swrite
		@@ -1212,11 +1242,11 @@
1212	1242	}else
1213	1243
1214	1244	/* cfile HASH USIZE CSIZE \n CONTENT
1215	1245	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
1216	1246	**
1217		- ** Accept a file from the client.
	1247	+ ** Server accepts a compressed file from the client.
1218	1248	*/
1219	1249	if( blob_eq(&xfer.aToken[0], "cfile") ){
1220	1250	if( !isPush ){
1221	1251	cgi_reset_content();
1222	1252	@ error not\sauthorized\sto\swrite
		@@ -1232,11 +1262,11 @@
1232	1262	}
1233	1263	}else
1234	1264
1235	1265	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
1236	1266	**
1237		- ** Accept an unversioned file from the client.
	1267	+ ** Server accepts an unversioned file from the client.
1238	1268	*/
1239	1269	if( blob_eq(&xfer.aToken[0], "uvfile") ){
1240	1270	xfer_accept_unversioned_file(&xfer, g.perm.WrUnver);
1241	1271	if( blob_size(&xfer.err) ){
1242	1272	cgi_reset_content();
		@@ -1246,11 +1276,11 @@
1246	1276	}
1247	1277	}else
1248	1278
1249	1279	/* gimme HASH
1250	1280	**
1251		- ** Client is requesting a file. Send it.
	1281	+ ** Client is requesting a file from the server. Send it.
1252	1282	*/
1253	1283	if( blob_eq(&xfer.aToken[0], "gimme")
1254	1284	&& xfer.nToken==2
1255	1285	&& blob_is_hname(&xfer.aToken[1])
1256	1286	){
		@@ -1263,11 +1293,11 @@
1263	1293	}
1264	1294	}else
1265	1295
1266	1296	/* uvgimme NAME
1267	1297	**
1268		- ** Client is requesting an unversioned file. Send it.
	1298	+ ** Client is requesting an unversioned file from the server. Send it.
1269	1299	*/
1270	1300	if( blob_eq(&xfer.aToken[0], "uvgimme")
1271	1301	&& xfer.nToken==2
1272	1302	&& blob_is_filename(&xfer.aToken[1])
1273	1303	){
		@@ -1275,24 +1305,38 @@
1275	1305	}else
1276	1306
1277	1307	/* igot HASH ?ISPRIVATE?
1278	1308	**
1279	1309	** Client announces that it has a particular file. If the ISPRIVATE
1280		- ** argument exists and is non-zero, then the file is a private file.
	1310	+ ** argument exists and is "1", then the file is a private file.
1281	1311	*/
1282	1312	if( xfer.nToken>=2
1283	1313	&& blob_eq(&xfer.aToken[0], "igot")
1284	1314	&& blob_is_hname(&xfer.aToken[1])
1285	1315	){
1286	1316	if( isPush ){
	1317	+ int rid = 0;
1287	1318	if( xfer.nToken==2 \|\| blob_eq(&xfer.aToken[2],"1")==0 ){
1288		- rid_from_uuid(&xfer.aToken[1], 1, 0);
	1319	+ /* Client says the artifact is public */
	1320	+ rid = rid_from_uuid(&xfer.aToken[1], 1, 0);
1289	1321	}else if( g.perm.Private ){
1290		- rid_from_uuid(&xfer.aToken[1], 1, 1);
	1322	+ /* Client says the artifact is private and the client has
	1323	+ ** permission to push private content. Create a new phantom
	1324	+ ** artifact that is marked private. */
	1325	+ rid = rid_from_uuid(&xfer.aToken[1], 1, 1);
1291	1326	}else{
1292		- server_private_xfer_not_authorized();
	1327	+ /* Client says the artifact is private and the client is unable
	1328	+ ** or unwilling to send us the artifact. If we already hold the
	1329	+ ** artifact here on the server as a phantom, make sure that
	1330	+ ** phantom is marked as private so that we don't keep asking about
	1331	+ ** it in subsequent sync requests. */
	1332	+ rid = rid_from_uuid(&xfer.aToken[1], 0, 1);
	1333	+ if( rid>0 ){
	1334	+ db_multi_exec("INSERT OR IGNORE INTO private(rid) VALUES(%d)",rid);
	1335	+ }
1293	1336	}
	1337	+ if( rid ) remote_has(rid);
1294	1338	}
1295	1339	}else
1296	1340
1297	1341
1298	1342	/* pull SERVERCODE PROJECTCODE
		@@ -1388,11 +1432,12 @@
1388	1432	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1389	1433	}else
1390	1434
1391	1435	/* login USER NONCE SIGNATURE
1392	1436	**
1393		- ** Check for a valid login. This has to happen before anything else.
	1437	+ ** The client has sent login credentials to the server.
	1438	+ ** Validate the login. This has to happen before anything else.
1394	1439	** The client can send multiple logins. Permissions are cumulative.
1395	1440	*/
1396	1441	if( blob_eq(&xfer.aToken[0], "login")
1397	1442	&& xfer.nToken==4
1398	1443	){
		@@ -1410,11 +1455,11 @@
1410	1455	}
1411	1456	}else
1412	1457
1413	1458	/* reqconfig NAME
1414	1459	**
1415		- ** Request a configuration value
	1460	+ ** Client is requesting a configuration value from the server
1416	1461	*/
1417	1462	if( blob_eq(&xfer.aToken[0], "reqconfig")
1418	1463	&& xfer.nToken==2
1419	1464	){
1420	1465	if( g.perm.Read ){
		@@ -1429,12 +1474,12 @@
1429	1474	}
1430	1475	}else
1431	1476
1432	1477	/* config NAME SIZE \n CONTENT
1433	1478	**
1434		- ** Receive a configuration value from the client. This is only
1435		- ** permitted for high-privilege users.
	1479	+ ** Client has sent a configuration value to the server.
	1480	+ ** This is only permitted for high-privilege users.
1436	1481	*/
1437	1482	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
1438	1483	&& blob_is_int(&xfer.aToken[2], &size) ){
1439	1484	const char *zName = blob_str(&xfer.aToken[1]);
1440	1485	Blob content;
		@@ -1474,11 +1519,11 @@
1474	1519	}else
1475	1520
1476	1521
1477	1522	/* private
1478	1523	**
1479		- ** This card indicates that the next "file" or "cfile" will contain
	1524	+ ** The client card indicates that the next "file" or "cfile" will contain
1480	1525	** private content.
1481	1526	*/
1482	1527	if( blob_eq(&xfer.aToken[0], "private") ){
1483	1528	if( !g.perm.Private ){
1484	1529	server_private_xfer_not_authorized();
		@@ -1495,10 +1540,12 @@
1495	1540	** ignored.
1496	1541	*/
1497	1542	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
1498	1543
1499	1544	/* pragma send-private
	1545	+ **
	1546	+ ** The client is requesting private artifacts.
1500	1547	**
1501	1548	** If the user has the "x" privilege (which must be set explicitly -
1502	1549	** it is not automatic with "a" or "s") then this pragma causes
1503	1550	** private information to be pulled in addition to public records.
1504	1551	*/
		@@ -1511,22 +1558,32 @@
1511	1558	}
1512	1559	}
1513	1560
1514	1561	/* pragma send-catalog
1515	1562	**
1516		- ** Send igot cards for all known artifacts.
	1563	+ ** The client wants to see igot cards for all known artifacts.
	1564	+ ** This is used as part of "sync --verily" to help ensure that
	1565	+ ** no artifacts have been missed on prior syncs.
1517	1566	*/
1518	1567	if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1519	1568	xfer.resync = 0x7fffffff;
1520	1569	}
1521	1570
1522		- /* pragma client-version VERSION
	1571	+ /* pragma client-version VERSION ?DATE? ?TIME?
1523	1572	**
1524		- ** Let the server know what version of Fossil is running on the client.
	1573	+ ** The client announces to the server what version of Fossil it
	1574	+ ** is running. The DATE and TIME are a pure numeric ISO8601 time
	1575	+ ** for the specific check-in of the client.
1525	1576	*/
1526	1577	if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
1527		- xfer.clientVersion = atoi(blob_str(&xfer.aToken[2]));
	1578	+ xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
	1579	+ if( xfer.nToken>=5 ){
	1580	+ xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
	1581	+ xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
	1582	+ @ pragma server-version %d(RELEASE_VERSION_NUMBER) \
	1583	+ @ %d(MANIFEST_NUMERIC_DATE) %d(MANIFEST_NUMERIC_TIME)
	1584	+ }
1528	1585	}
1529	1586
1530	1587	/* pragma uv-hash HASH
1531	1588	**
1532	1589	** The client wants to make sure that unversioned files are all synced.
		@@ -1550,11 +1607,11 @@
1550	1607
1551	1608	/* pragma ci-lock CHECKIN-HASH CLIENT-ID
1552	1609	**
1553	1610	** The client wants to make non-branch commit against the check-in
1554	1611	** identified by CHECKIN-HASH. The server will remember this and
1555		- ** subsequent ci-lock request from different clients will generate
	1612	+ ** subsequent ci-lock requests from different clients will generate
1556	1613	** a ci-lock-fail pragma in the reply.
1557	1614	*/
1558	1615	if( blob_eq(&xfer.aToken[1], "ci-lock")
1559	1616	&& xfer.nToken==4
1560	1617	&& blob_is_hname(&xfer.aToken[2])
		@@ -1808,11 +1865,11 @@
1808	1865	memset(&xfer, 0, sizeof(xfer));
1809	1866	xfer.pIn = &recv;
1810	1867	xfer.pOut = &send;
1811	1868	xfer.mxSend = db_get_int("max-upload", 250000);
1812	1869	xfer.maxTime = -1;
1813		- xfer.clientVersion = RELEASE_VERSION_NUMBER;
	1870	+ xfer.remoteVersion = RELEASE_VERSION_NUMBER;
1814	1871	if( syncFlags & SYNC_PRIVATE ){
1815	1872	g.perm.Private = 1;
1816	1873	xfer.syncPrivate = 1;
1817	1874	}
1818	1875
		@@ -1856,13 +1913,16 @@
1856	1913	" SELECT name, 0 FROM unversioned WHERE hash IS NOT NULL;"
1857	1914	);
1858	1915	}
1859	1916
1860	1917	/*
1861		- ** Always begin with a clone, pull, or push message
	1918	+ ** The request from the client always begin with a clone, pull,
	1919	+ ** or push message.
1862	1920	*/
1863		- blob_appendf(&send, "pragma client-version %d\n", RELEASE_VERSION_NUMBER);
	1921	+ blob_appendf(&send, "pragma client-version %d %d %d\n",
	1922	+ RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
	1923	+ MANIFEST_NUMERIC_TIME);
1864	1924	if( syncFlags & SYNC_CLONE ){
1865	1925	blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
1866	1926	syncFlags &= ~(SYNC_PUSH\|SYNC_PULL);
1867	1927	nCardSent++;
1868	1928	/* TBD: Request all transferable configuration values */
		@@ -1898,20 +1958,19 @@
1898	1958	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
1899	1959	);
1900	1960	manifest_crosslink_begin();
1901	1961
1902	1962
1903		- /* Send back the most recently received cookie. Let the server
1904		- ** figure out if this is a cookie that it cares about.
	1963	+ /* Client sends the most recently received cookie back to the server.
	1964	+ ** Let the server figure out if this is a cookie that it cares about.
1905	1965	*/
1906	1966	zCookie = db_get("cookie", 0);
1907	1967	if( zCookie ){
1908	1968	blob_appendf(&send, "cookie %s\n", zCookie);
1909	1969	}
1910	1970
1911		- /* Generate gimme cards for phantoms and leaf cards
1912		- ** for all leaves.
	1971	+ /* Client sends gimme cards for phantoms
1913	1972	*/
1914	1973	if( (syncFlags & SYNC_PULL)!=0
1915	1974	\|\| ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
1916	1975	){
1917	1976	request_phantoms(&xfer, mxPhantomReq);
		@@ -1920,11 +1979,11 @@
1920	1979	send_unsent(&xfer);
1921	1980	nCardSent += send_unclustered(&xfer);
1922	1981	if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
1923	1982	}
1924	1983
1925		- /* Send configuration parameter requests. On a clone, delay sending
	1984	+ /* Client sends configuration parameter requests. On a clone, delay sending
1926	1985	** this until the second cycle since the login card might fail on
1927	1986	** the first cycle.
1928	1987	*/
1929	1988	if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0) ){
1930	1989	const char *zName;
		@@ -1937,13 +1996,13 @@
1937	1996	}
1938	1997	origConfigRcvMask = configRcvMask;
1939	1998	configRcvMask = 0;
1940	1999	}
1941	2000
1942		- /* Send a request to sync unversioned files. On a clone, delay sending
1943		- ** this until the second cycle since the login card might fail on
1944		- ** the first cycle.
	2001	+ /* Client sends a request to sync unversioned files.
	2002	+ ** On a clone, delay sending this until the second cycle since
	2003	+ ** the login card might fail on the first cycle.
1945	2004	*/
1946	2005	if( (syncFlags & SYNC_UNVERSIONED)!=0
1947	2006	&& ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0)
1948	2007	&& !uvHashSent
1949	2008	){
		@@ -1950,11 +2009,12 @@
1950	2009	blob_appendf(&send, "pragma uv-hash %s\n", unversioned_content_hash(0));
1951	2010	nCardSent++;
1952	2011	uvHashSent = 1;
1953	2012	}
1954	2013
1955		- /* Send configuration parameters being pushed */
	2014	+ /* On a "fossil config push", the client send configuration parameters
	2015	+ ** being pushed up to the server */
1956	2016	if( configSendMask ){
1957	2017	if( zOpType==0 ) zOpType = "Push";
1958	2018	nCardSent += configure_send_group(xfer.pOut, configSendMask, 0);
1959	2019	configSendMask = 0;
1960	2020	}
		@@ -2010,11 +2070,11 @@
2010	2070	zCkinLock = 0;
2011	2071	}else if( zClientId ){
2012	2072	blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
2013	2073	}
2014	2074
2015		- /* Append randomness to the end of the message. This makes all
	2075	+ /* Append randomness to the end of the uplink message. This makes all
2016	2076	** messages unique so that that the login-card nonce will always
2017	2077	** be unique.
2018	2078	*/
2019	2079	zRandomness = db_text(0, "SELECT hex(randomblob(20))");
2020	2080	blob_appendf(&send, "# %s\n", zRandomness);
		@@ -2055,11 +2115,13 @@
2055	2115	xfer.nGimmeSent = 0;
2056	2116	xfer.nIGotSent = 0;
2057	2117
2058	2118	lastPctDone = -1;
2059	2119	blob_reset(&send);
2060		- blob_appendf(&send, "pragma client-version %d\n", RELEASE_VERSION_NUMBER);
	2120	+ blob_appendf(&send, "pragma client-version %d %d %d\n",
	2121	+ RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
	2122	+ MANIFEST_NUMERIC_TIME);
2061	2123	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
2062	2124
2063	2125	/* Send the send-private pragma if we are trying to sync private data */
2064	2126	if( syncFlags & SYNC_PRIVATE ){
2065	2127	blob_append(&send, "pragma send-private\n", -1);
		@@ -2112,30 +2174,30 @@
2112	2174	}
2113	2175
2114	2176	/* file HASH SIZE \n CONTENT
2115	2177	** file HASH DELTASRC SIZE \n CONTENT
2116	2178	**
2117		- ** Receive a file transmitted from the server.
	2179	+ ** Client receives a file transmitted from the server.
2118	2180	*/
2119	2181	if( blob_eq(&xfer.aToken[0],"file") ){
2120	2182	xfer_accept_file(&xfer, (syncFlags & SYNC_CLONE)!=0, 0, 0);
2121	2183	nArtifactRcvd++;
2122	2184	}else
2123	2185
2124	2186	/* cfile HASH USIZE CSIZE \n CONTENT
2125	2187	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
2126	2188	**
2127		- ** Receive a compressed file transmitted from the server.
	2189	+ ** Client receives a compressed file transmitted from the server.
2128	2190	*/
2129	2191	if( blob_eq(&xfer.aToken[0],"cfile") ){
2130	2192	xfer_accept_compressed_file(&xfer, 0, 0);
2131	2193	nArtifactRcvd++;
2132	2194	}else
2133	2195
2134	2196	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
2135	2197	**
2136		- ** Accept an unversioned file from the server.
	2198	+ ** Client accepts an unversioned file from the server.
2137	2199	*/
2138	2200	if( blob_eq(&xfer.aToken[0], "uvfile") ){
2139	2201	xfer_accept_unversioned_file(&xfer, 1);
2140	2202	nArtifactRcvd++;
2141	2203	nUvFileRcvd++;
		@@ -2145,13 +2207,14 @@
2145	2207	}
2146	2208	}else
2147	2209
2148	2210	/* gimme HASH
2149	2211	**
2150		- ** Server is requesting a file. If the file is a manifest, assume
2151		- ** that the server will also want to know all of the content files
2152		- ** associated with the manifest and send those too.
	2212	+ ** Client receives an artifact request from the server.
	2213	+ ** If the file is a manifest, assume that the server will also want
	2214	+ ** to know all of the content artifacts associated with the manifest
	2215	+ ** and send those too.
2153	2216	*/
2154	2217	if( blob_eq(&xfer.aToken[0], "gimme")
2155	2218	&& xfer.nToken==2
2156	2219	&& blob_is_hname(&xfer.aToken[1])
2157	2220	){
		@@ -2182,10 +2245,11 @@
2182	2245	rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2183	2246	if( rid>0 ){
2184	2247	if( !isPriv ) content_make_public(rid);
2185	2248	}else if( isPriv && !g.perm.Private ){
2186	2249	/* ignore private files */
	2250	+ db_multi_exec("INSERT OR IGNORE INTO private VALUES(%d)", rid);
2187	2251	}else if( (syncFlags & (SYNC_PULL\|SYNC_CLONE))!=0 ){
2188	2252	rid = content_new(blob_str(&xfer.aToken[1]), isPriv);
2189	2253	if( rid ) newPhantom = 1;
2190	2254	}
2191	2255	remote_has(rid);
		@@ -2264,11 +2328,11 @@
2264	2328	}else
2265	2329
2266	2330	/* push SERVERCODE PRODUCTCODE
2267	2331	**
2268	2332	** Should only happen in response to a clone. This message tells
2269		- ** the client what product to use for the new database.
	2333	+ ** the client what product code to use for the new database.
2270	2334	*/
2271	2335	if( blob_eq(&xfer.aToken[0],"push")
2272	2336	&& xfer.nToken==3
2273	2337	&& (syncFlags & SYNC_CLONE)!=0
2274	2338	&& blob_is_hname(&xfer.aToken[2])
		@@ -2281,11 +2345,11 @@
2281	2345	nCardSent++;
2282	2346	}else
2283	2347
2284	2348	/* config NAME SIZE \n CONTENT
2285	2349	**
2286		- ** Receive a configuration value from the server.
	2350	+ ** Client receive a configuration value from the server.
2287	2351	**
2288	2352	** The received configuration setting is silently ignored if it was
2289	2353	** not requested by a prior "reqconfig" sent from client to server.
2290	2354	*/
2291	2355	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
		@@ -2303,11 +2367,11 @@
2303	2367	}else
2304	2368
2305	2369
2306	2370	/* cookie TEXT
2307	2371	**
2308		- ** The server might include a cookie in its reply. The client
	2372	+ ** The client reserves a cookie from the server. The client
2309	2373	** should remember this cookie and send it back to the server
2310	2374	** in its next query.
2311	2375	**
2312	2376	** Each cookie received overwrites the prior cookie from the
2313	2377	** same server.
		@@ -2317,12 +2381,12 @@
2317	2381	}else
2318	2382
2319	2383
2320	2384	/* private
2321	2385	**
2322		- ** This card indicates that the next "file" or "cfile" will contain
2323		- ** private content.
	2386	+ ** The server tells the client that the next "file" or "cfile" will
	2387	+ ** contain private content.
2324	2388	*/
2325	2389	if( blob_eq(&xfer.aToken[0], "private") ){
2326	2390	xfer.nextIsPrivate = 1;
2327	2391	}else
2328	2392
		@@ -2338,11 +2402,12 @@
2338	2402	blob_is_int(&xfer.aToken[1], &cloneSeqno);
2339	2403	}else
2340	2404
2341	2405	/* message MESSAGE
2342	2406	**
2343		- ** Print a message. Similar to "error" but does not stop processing.
	2407	+ ** A message is received from the server. Print it.
	2408	+ ** Similar to "error" but does not stop processing.
2344	2409	**
2345	2410	** If the "login failed" message is seen, clear the sync password prior
2346	2411	** to the next cycle.
2347	2412	*/
2348	2413	if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
		@@ -2364,11 +2429,27 @@
2364	2429	** The server can send pragmas to try to convey meta-information to
2365	2430	** the client. These are informational only. Unknown pragmas are
2366	2431	** silently ignored.
2367	2432	*/
2368	2433	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
2369		- /* If the server is unwill to accept new unversioned content (because
	2434	+ /* pragma server-version VERSION ?DATE? ?TIME?
	2435	+ **
	2436	+ ** The servger announces to the server what version of Fossil it
	2437	+ ** is running. The DATE and TIME are a pure numeric ISO8601 time
	2438	+ ** for the specific check-in of the client.
	2439	+ */
	2440	+ if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
	2441	+ xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
	2442	+ if( xfer.nToken>=5 ){
	2443	+ xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
	2444	+ xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
	2445	+ }
	2446	+ }
	2447	+
	2448	+ /* pragma uv-pull-only
	2449	+ **
	2450	+ ** If the server is unwill to accept new unversioned content (because
2370	2451	** this client lacks the necessary permissions) then it sends a
2371	2452	** "uv-pull-only" pragma so that the client will know not to waste
2372	2453	** bandwidth trying to upload unversioned content. If the server
2373	2454	** does accept new unversioned content, it sends "uv-push-ok".
2374	2455	*/
		@@ -2407,11 +2488,12 @@
2407	2488	}
2408	2489	}else
2409	2490
2410	2491	/* error MESSAGE
2411	2492	**
2412		- ** Report an error and abandon the sync session.
	2493	+ ** The server is reporting an error. The client will abandon
	2494	+ ** the sync session.
2413	2495	**
2414	2496	** Except, when cloning we will sometimes get an error on the
2415	2497	** first message exchange because the project-code is unknown
2416	2498	** and so the login card on the request was invalid. The project-code
2417	2499	** is returned in the reply before the error card, so second and
2418	2500

	--- src/xfer.c
	+++ src/xfer.c
	@@ -49,11 +49,13 @@
49	int nDanglingFile; /* Number of dangling deltas received */
50	int mxSend; /* Stop sending "file" when pOut reaches this size */
51	int resync; /* Send igot cards for all holdings */
52	u8 syncPrivate; /* True to enable syncing private content */
53	u8 nextIsPrivate; /* If true, next "file" received is a private */
54	u32 clientVersion; /* Version of the client software */


55	time_t maxTime; /* Time when this transfer should be finished */
56	};
57
58
59	/*
	@@ -524,20 +526,19 @@
524	static void send_file(Xfer pXfer, int rid, Blob pUuid, int nativeDelta){
525	Blob content, uuid;
526	int size = 0;
527	int isPriv = content_is_private(rid);
528
529	if( pXfer->syncPrivate==0 && isPriv ) return;
530	if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
531	return;
532	}
533	blob_zero(&uuid);
534	db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
535	if( blob_size(&uuid)==0 ){
536	return;
537	}
538	if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->clientVersion<20000 ){
539	xfer_cannot_send_sha3_error(pXfer);
540	return;
541	}
542	if( pUuid ){
543	if( blob_compare(pUuid, &uuid)!=0 ){
	@@ -548,10 +549,21 @@
548	pUuid = &uuid;
549	}
550	if( uuid_is_shunned(blob_str(pUuid)) ){
551	blob_reset(&uuid);
552	return;











553	}
554	if( (pXfer->maxTime != -1 && time(NULL) >= pXfer->maxTime) \|\|
555	pXfer->mxSend<=blob_size(pXfer->pOut) ){
556	const char *zFormat = isPriv ? "igot %b 1\n" : "igot %b\n";
557	blob_appendf(pXfer->pOut, zFormat /works-like:"%b"/, pUuid);
	@@ -626,11 +638,11 @@
626	szC = db_column_bytes(&q1, 2);
627	zContent = db_column_raw(&q1, 2);
628	srcIsPrivate = db_column_int(&q1, 3);
629	zDelta = db_column_text(&q1, 4);
630	if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
631	if( pXfer->clientVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
632	xfer_cannot_send_sha3_error(pXfer);
633	db_reset(&q1);
634	return;
635	}
636	blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
	@@ -690,11 +702,11 @@
690	);
691	}
692	if( db_step(&q1)==SQLITE_ROW ){
693	sqlite3_int64 mtime = db_column_int64(&q1, 0);
694	const char *zHash = db_column_text(&q1, 1);
695	if( pXfer->clientVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
696	xfer_cannot_send_sha3_error(pXfer);
697	db_reset(&q1);
698	return;
699	}
700	if( blob_size(pXfer->pOut)>=pXfer->mxSend ){
	@@ -956,30 +968,48 @@
956	}
957
958	/*
959	** Send an igot message for every entry in unclustered table.
960	** Return the number of cards sent.











961	*/
962	static int send_unclustered(Xfer *pXfer){
963	Stmt q;
964	int cnt = 0;






965	if( pXfer->resync ){
966	db_prepare(&q,
967	"SELECT uuid, rid FROM blob"
968	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
969	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
970	" AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"
971	" AND blob.rid<=%d"
972	" ORDER BY blob.rid DESC",
973	pXfer->resync
974	);
975	}else{
976	db_prepare(&q,
977	"SELECT uuid FROM unclustered JOIN blob USING(rid) /scan/"
978	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
979	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
980	" AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"

981	);
982	}
983	while( db_step(&q)==SQLITE_ROW ){
984	blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
985	cnt++;
	@@ -1191,11 +1221,11 @@
1191	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1192
1193	/* file HASH SIZE \n CONTENT
1194	** file HASH DELTASRC SIZE \n CONTENT
1195	**
1196	** Accept a file from the client.
1197	*/
1198	if( blob_eq(&xfer.aToken[0], "file") ){
1199	if( !isPush ){
1200	cgi_reset_content();
1201	@ error not\sauthorized\sto\swrite
	@@ -1212,11 +1242,11 @@
1212	}else
1213
1214	/* cfile HASH USIZE CSIZE \n CONTENT
1215	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
1216	**
1217	** Accept a file from the client.
1218	*/
1219	if( blob_eq(&xfer.aToken[0], "cfile") ){
1220	if( !isPush ){
1221	cgi_reset_content();
1222	@ error not\sauthorized\sto\swrite
	@@ -1232,11 +1262,11 @@
1232	}
1233	}else
1234
1235	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
1236	**
1237	** Accept an unversioned file from the client.
1238	*/
1239	if( blob_eq(&xfer.aToken[0], "uvfile") ){
1240	xfer_accept_unversioned_file(&xfer, g.perm.WrUnver);
1241	if( blob_size(&xfer.err) ){
1242	cgi_reset_content();
	@@ -1246,11 +1276,11 @@
1246	}
1247	}else
1248
1249	/* gimme HASH
1250	**
1251	** Client is requesting a file. Send it.
1252	*/
1253	if( blob_eq(&xfer.aToken[0], "gimme")
1254	&& xfer.nToken==2
1255	&& blob_is_hname(&xfer.aToken[1])
1256	){
	@@ -1263,11 +1293,11 @@
1263	}
1264	}else
1265
1266	/* uvgimme NAME
1267	**
1268	** Client is requesting an unversioned file. Send it.
1269	*/
1270	if( blob_eq(&xfer.aToken[0], "uvgimme")
1271	&& xfer.nToken==2
1272	&& blob_is_filename(&xfer.aToken[1])
1273	){
	@@ -1275,24 +1305,38 @@
1275	}else
1276
1277	/* igot HASH ?ISPRIVATE?
1278	**
1279	** Client announces that it has a particular file. If the ISPRIVATE
1280	** argument exists and is non-zero, then the file is a private file.
1281	*/
1282	if( xfer.nToken>=2
1283	&& blob_eq(&xfer.aToken[0], "igot")
1284	&& blob_is_hname(&xfer.aToken[1])
1285	){
1286	if( isPush ){

1287	if( xfer.nToken==2 \|\| blob_eq(&xfer.aToken[2],"1")==0 ){
1288	rid_from_uuid(&xfer.aToken[1], 1, 0);

1289	}else if( g.perm.Private ){
1290	rid_from_uuid(&xfer.aToken[1], 1, 1);



1291	}else{
1292	server_private_xfer_not_authorized();








1293	}

1294	}
1295	}else
1296
1297
1298	/* pull SERVERCODE PROJECTCODE
	@@ -1388,11 +1432,12 @@
1388	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1389	}else
1390
1391	/* login USER NONCE SIGNATURE
1392	**
1393	** Check for a valid login. This has to happen before anything else.

1394	** The client can send multiple logins. Permissions are cumulative.
1395	*/
1396	if( blob_eq(&xfer.aToken[0], "login")
1397	&& xfer.nToken==4
1398	){
	@@ -1410,11 +1455,11 @@
1410	}
1411	}else
1412
1413	/* reqconfig NAME
1414	**
1415	** Request a configuration value
1416	*/
1417	if( blob_eq(&xfer.aToken[0], "reqconfig")
1418	&& xfer.nToken==2
1419	){
1420	if( g.perm.Read ){
	@@ -1429,12 +1474,12 @@
1429	}
1430	}else
1431
1432	/* config NAME SIZE \n CONTENT
1433	**
1434	** Receive a configuration value from the client. This is only
1435	** permitted for high-privilege users.
1436	*/
1437	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
1438	&& blob_is_int(&xfer.aToken[2], &size) ){
1439	const char *zName = blob_str(&xfer.aToken[1]);
1440	Blob content;
	@@ -1474,11 +1519,11 @@
1474	}else
1475
1476
1477	/* private
1478	**
1479	** This card indicates that the next "file" or "cfile" will contain
1480	** private content.
1481	*/
1482	if( blob_eq(&xfer.aToken[0], "private") ){
1483	if( !g.perm.Private ){
1484	server_private_xfer_not_authorized();
	@@ -1495,10 +1540,12 @@
1495	** ignored.
1496	*/
1497	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
1498
1499	/* pragma send-private


1500	**
1501	** If the user has the "x" privilege (which must be set explicitly -
1502	** it is not automatic with "a" or "s") then this pragma causes
1503	** private information to be pulled in addition to public records.
1504	*/
	@@ -1511,22 +1558,32 @@
1511	}
1512	}
1513
1514	/* pragma send-catalog
1515	**
1516	** Send igot cards for all known artifacts.


1517	*/
1518	if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1519	xfer.resync = 0x7fffffff;
1520	}
1521
1522	/* pragma client-version VERSION
1523	**
1524	** Let the server know what version of Fossil is running on the client.


1525	*/
1526	if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
1527	xfer.clientVersion = atoi(blob_str(&xfer.aToken[2]));






1528	}
1529
1530	/* pragma uv-hash HASH
1531	**
1532	** The client wants to make sure that unversioned files are all synced.
	@@ -1550,11 +1607,11 @@
1550
1551	/* pragma ci-lock CHECKIN-HASH CLIENT-ID
1552	**
1553	** The client wants to make non-branch commit against the check-in
1554	** identified by CHECKIN-HASH. The server will remember this and
1555	** subsequent ci-lock request from different clients will generate
1556	** a ci-lock-fail pragma in the reply.
1557	*/
1558	if( blob_eq(&xfer.aToken[1], "ci-lock")
1559	&& xfer.nToken==4
1560	&& blob_is_hname(&xfer.aToken[2])
	@@ -1808,11 +1865,11 @@
1808	memset(&xfer, 0, sizeof(xfer));
1809	xfer.pIn = &recv;
1810	xfer.pOut = &send;
1811	xfer.mxSend = db_get_int("max-upload", 250000);
1812	xfer.maxTime = -1;
1813	xfer.clientVersion = RELEASE_VERSION_NUMBER;
1814	if( syncFlags & SYNC_PRIVATE ){
1815	g.perm.Private = 1;
1816	xfer.syncPrivate = 1;
1817	}
1818
	@@ -1856,13 +1913,16 @@
1856	" SELECT name, 0 FROM unversioned WHERE hash IS NOT NULL;"
1857	);
1858	}
1859
1860	/*
1861	** Always begin with a clone, pull, or push message

1862	*/
1863	blob_appendf(&send, "pragma client-version %d\n", RELEASE_VERSION_NUMBER);


1864	if( syncFlags & SYNC_CLONE ){
1865	blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
1866	syncFlags &= ~(SYNC_PUSH\|SYNC_PULL);
1867	nCardSent++;
1868	/* TBD: Request all transferable configuration values */
	@@ -1898,20 +1958,19 @@
1898	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
1899	);
1900	manifest_crosslink_begin();
1901
1902
1903	/* Send back the most recently received cookie. Let the server
1904	** figure out if this is a cookie that it cares about.
1905	*/
1906	zCookie = db_get("cookie", 0);
1907	if( zCookie ){
1908	blob_appendf(&send, "cookie %s\n", zCookie);
1909	}
1910
1911	/* Generate gimme cards for phantoms and leaf cards
1912	** for all leaves.
1913	*/
1914	if( (syncFlags & SYNC_PULL)!=0
1915	\|\| ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
1916	){
1917	request_phantoms(&xfer, mxPhantomReq);
	@@ -1920,11 +1979,11 @@
1920	send_unsent(&xfer);
1921	nCardSent += send_unclustered(&xfer);
1922	if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
1923	}
1924
1925	/* Send configuration parameter requests. On a clone, delay sending
1926	** this until the second cycle since the login card might fail on
1927	** the first cycle.
1928	*/
1929	if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0) ){
1930	const char *zName;
	@@ -1937,13 +1996,13 @@
1937	}
1938	origConfigRcvMask = configRcvMask;
1939	configRcvMask = 0;
1940	}
1941
1942	/* Send a request to sync unversioned files. On a clone, delay sending
1943	** this until the second cycle since the login card might fail on
1944	** the first cycle.
1945	*/
1946	if( (syncFlags & SYNC_UNVERSIONED)!=0
1947	&& ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0)
1948	&& !uvHashSent
1949	){
	@@ -1950,11 +2009,12 @@
1950	blob_appendf(&send, "pragma uv-hash %s\n", unversioned_content_hash(0));
1951	nCardSent++;
1952	uvHashSent = 1;
1953	}
1954
1955	/* Send configuration parameters being pushed */

1956	if( configSendMask ){
1957	if( zOpType==0 ) zOpType = "Push";
1958	nCardSent += configure_send_group(xfer.pOut, configSendMask, 0);
1959	configSendMask = 0;
1960	}
	@@ -2010,11 +2070,11 @@
2010	zCkinLock = 0;
2011	}else if( zClientId ){
2012	blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
2013	}
2014
2015	/* Append randomness to the end of the message. This makes all
2016	** messages unique so that that the login-card nonce will always
2017	** be unique.
2018	*/
2019	zRandomness = db_text(0, "SELECT hex(randomblob(20))");
2020	blob_appendf(&send, "# %s\n", zRandomness);
	@@ -2055,11 +2115,13 @@
2055	xfer.nGimmeSent = 0;
2056	xfer.nIGotSent = 0;
2057
2058	lastPctDone = -1;
2059	blob_reset(&send);
2060	blob_appendf(&send, "pragma client-version %d\n", RELEASE_VERSION_NUMBER);


2061	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
2062
2063	/* Send the send-private pragma if we are trying to sync private data */
2064	if( syncFlags & SYNC_PRIVATE ){
2065	blob_append(&send, "pragma send-private\n", -1);
	@@ -2112,30 +2174,30 @@
2112	}
2113
2114	/* file HASH SIZE \n CONTENT
2115	** file HASH DELTASRC SIZE \n CONTENT
2116	**
2117	** Receive a file transmitted from the server.
2118	*/
2119	if( blob_eq(&xfer.aToken[0],"file") ){
2120	xfer_accept_file(&xfer, (syncFlags & SYNC_CLONE)!=0, 0, 0);
2121	nArtifactRcvd++;
2122	}else
2123
2124	/* cfile HASH USIZE CSIZE \n CONTENT
2125	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
2126	**
2127	** Receive a compressed file transmitted from the server.
2128	*/
2129	if( blob_eq(&xfer.aToken[0],"cfile") ){
2130	xfer_accept_compressed_file(&xfer, 0, 0);
2131	nArtifactRcvd++;
2132	}else
2133
2134	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
2135	**
2136	** Accept an unversioned file from the server.
2137	*/
2138	if( blob_eq(&xfer.aToken[0], "uvfile") ){
2139	xfer_accept_unversioned_file(&xfer, 1);
2140	nArtifactRcvd++;
2141	nUvFileRcvd++;
	@@ -2145,13 +2207,14 @@
2145	}
2146	}else
2147
2148	/* gimme HASH
2149	**
2150	** Server is requesting a file. If the file is a manifest, assume
2151	** that the server will also want to know all of the content files
2152	** associated with the manifest and send those too.

2153	*/
2154	if( blob_eq(&xfer.aToken[0], "gimme")
2155	&& xfer.nToken==2
2156	&& blob_is_hname(&xfer.aToken[1])
2157	){
	@@ -2182,10 +2245,11 @@
2182	rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2183	if( rid>0 ){
2184	if( !isPriv ) content_make_public(rid);
2185	}else if( isPriv && !g.perm.Private ){
2186	/* ignore private files */

2187	}else if( (syncFlags & (SYNC_PULL\|SYNC_CLONE))!=0 ){
2188	rid = content_new(blob_str(&xfer.aToken[1]), isPriv);
2189	if( rid ) newPhantom = 1;
2190	}
2191	remote_has(rid);
	@@ -2264,11 +2328,11 @@
2264	}else
2265
2266	/* push SERVERCODE PRODUCTCODE
2267	**
2268	** Should only happen in response to a clone. This message tells
2269	** the client what product to use for the new database.
2270	*/
2271	if( blob_eq(&xfer.aToken[0],"push")
2272	&& xfer.nToken==3
2273	&& (syncFlags & SYNC_CLONE)!=0
2274	&& blob_is_hname(&xfer.aToken[2])
	@@ -2281,11 +2345,11 @@
2281	nCardSent++;
2282	}else
2283
2284	/* config NAME SIZE \n CONTENT
2285	**
2286	** Receive a configuration value from the server.
2287	**
2288	** The received configuration setting is silently ignored if it was
2289	** not requested by a prior "reqconfig" sent from client to server.
2290	*/
2291	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
	@@ -2303,11 +2367,11 @@
2303	}else
2304
2305
2306	/* cookie TEXT
2307	**
2308	** The server might include a cookie in its reply. The client
2309	** should remember this cookie and send it back to the server
2310	** in its next query.
2311	**
2312	** Each cookie received overwrites the prior cookie from the
2313	** same server.
	@@ -2317,12 +2381,12 @@
2317	}else
2318
2319
2320	/* private
2321	**
2322	** This card indicates that the next "file" or "cfile" will contain
2323	** private content.
2324	*/
2325	if( blob_eq(&xfer.aToken[0], "private") ){
2326	xfer.nextIsPrivate = 1;
2327	}else
2328
	@@ -2338,11 +2402,12 @@
2338	blob_is_int(&xfer.aToken[1], &cloneSeqno);
2339	}else
2340
2341	/* message MESSAGE
2342	**
2343	** Print a message. Similar to "error" but does not stop processing.

2344	**
2345	** If the "login failed" message is seen, clear the sync password prior
2346	** to the next cycle.
2347	*/
2348	if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
	@@ -2364,11 +2429,27 @@
2364	** The server can send pragmas to try to convey meta-information to
2365	** the client. These are informational only. Unknown pragmas are
2366	** silently ignored.
2367	*/
2368	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
2369	/* If the server is unwill to accept new unversioned content (because
















2370	** this client lacks the necessary permissions) then it sends a
2371	** "uv-pull-only" pragma so that the client will know not to waste
2372	** bandwidth trying to upload unversioned content. If the server
2373	** does accept new unversioned content, it sends "uv-push-ok".
2374	*/
	@@ -2407,11 +2488,12 @@
2407	}
2408	}else
2409
2410	/* error MESSAGE
2411	**
2412	** Report an error and abandon the sync session.

2413	**
2414	** Except, when cloning we will sometimes get an error on the
2415	** first message exchange because the project-code is unknown
2416	** and so the login card on the request was invalid. The project-code
2417	** is returned in the reply before the error card, so second and
2418

	--- src/xfer.c
	+++ src/xfer.c
	@@ -49,11 +49,13 @@
49	int nDanglingFile; /* Number of dangling deltas received */
50	int mxSend; /* Stop sending "file" when pOut reaches this size */
51	int resync; /* Send igot cards for all holdings */
52	u8 syncPrivate; /* True to enable syncing private content */
53	u8 nextIsPrivate; /* If true, next "file" received is a private */
54	u32 remoteVersion; /* Version of fossil running on the other side */
55	u32 remoteDate; /* Date for specific client software edition */
56	u32 remoteTime; /* Time of date correspoding on remoteDate */
57	time_t maxTime; /* Time when this transfer should be finished */
58	};
59
60
61	/*
	@@ -524,20 +526,19 @@
526	static void send_file(Xfer pXfer, int rid, Blob pUuid, int nativeDelta){
527	Blob content, uuid;
528	int size = 0;
529	int isPriv = content_is_private(rid);
530

531	if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
532	return;
533	}
534	blob_zero(&uuid);
535	db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
536	if( blob_size(&uuid)==0 ){
537	return;
538	}
539	if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->remoteVersion<20000 ){
540	xfer_cannot_send_sha3_error(pXfer);
541	return;
542	}
543	if( pUuid ){
544	if( blob_compare(pUuid, &uuid)!=0 ){
	@@ -548,10 +549,21 @@
549	pUuid = &uuid;
550	}
551	if( uuid_is_shunned(blob_str(pUuid)) ){
552	blob_reset(&uuid);
553	return;
554	}
555	if( isPriv && pXfer->syncPrivate==0 ){
556	if( pXfer->remoteDate>=20200413 ){
557	/* If the artifact is private and we are not doing a private sync,
558	** at least tell the other side that the artifact exists and is
559	** known to be private. But only do this for newer clients since
560	** older ones will throw an error if they get a private igot card
561	** and private syncing is disallowed */
562	blob_appendf(pXfer->pOut, "igot %b 1\n", pUuid);
563	}
564	return;
565	}
566	if( (pXfer->maxTime != -1 && time(NULL) >= pXfer->maxTime) \|\|
567	pXfer->mxSend<=blob_size(pXfer->pOut) ){
568	const char *zFormat = isPriv ? "igot %b 1\n" : "igot %b\n";
569	blob_appendf(pXfer->pOut, zFormat /works-like:"%b"/, pUuid);
	@@ -626,11 +638,11 @@
638	szC = db_column_bytes(&q1, 2);
639	zContent = db_column_raw(&q1, 2);
640	srcIsPrivate = db_column_int(&q1, 3);
641	zDelta = db_column_text(&q1, 4);
642	if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
643	if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
644	xfer_cannot_send_sha3_error(pXfer);
645	db_reset(&q1);
646	return;
647	}
648	blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
	@@ -690,11 +702,11 @@
702	);
703	}
704	if( db_step(&q1)==SQLITE_ROW ){
705	sqlite3_int64 mtime = db_column_int64(&q1, 0);
706	const char *zHash = db_column_text(&q1, 1);
707	if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
708	xfer_cannot_send_sha3_error(pXfer);
709	db_reset(&q1);
710	return;
711	}
712	if( blob_size(pXfer->pOut)>=pXfer->mxSend ){
	@@ -956,30 +968,48 @@
968	}
969
970	/*
971	** Send an igot message for every entry in unclustered table.
972	** Return the number of cards sent.
973	**
974	** Except:
975	** * Do not send igot cards for shunned artifacts
976	** * Do not send igot cards for phantoms
977	** * Do not send igot cards for private artifacts
978	** * Do not send igot cards for any artifact that is in the
979	** ONREMOTE table, if that table exists.
980	**
981	** If the pXfer->resync flag is set, that means we are doing a "--verily"
982	** sync and all artifacts that don't meet the restrictions above should
983	** be sent.
984	*/
985	static int send_unclustered(Xfer *pXfer){
986	Stmt q;
987	int cnt = 0;
988	const char *zExtra;
989	if( db_table_exists("temp","onremote") ){
990	zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
991	}else{
992	zExtra = "";
993	}
994	if( pXfer->resync ){
995	db_prepare(&q,
996	"SELECT uuid, rid FROM blob"
997	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
998	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
999	" AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s"
1000	" AND blob.rid<=%d"
1001	" ORDER BY blob.rid DESC",
1002	zExtra /safe-for-%s/, pXfer->resync
1003	);
1004	}else{
1005	db_prepare(&q,
1006	"SELECT uuid FROM unclustered JOIN blob USING(rid) /scan/"
1007	" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1008	" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1009	" AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
1010	zExtra /safe-for-%s/
1011	);
1012	}
1013	while( db_step(&q)==SQLITE_ROW ){
1014	blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
1015	cnt++;
	@@ -1191,11 +1221,11 @@
1221	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1222
1223	/* file HASH SIZE \n CONTENT
1224	** file HASH DELTASRC SIZE \n CONTENT
1225	**
1226	** Server accepts a file from the client.
1227	*/
1228	if( blob_eq(&xfer.aToken[0], "file") ){
1229	if( !isPush ){
1230	cgi_reset_content();
1231	@ error not\sauthorized\sto\swrite
	@@ -1212,11 +1242,11 @@
1242	}else
1243
1244	/* cfile HASH USIZE CSIZE \n CONTENT
1245	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
1246	**
1247	** Server accepts a compressed file from the client.
1248	*/
1249	if( blob_eq(&xfer.aToken[0], "cfile") ){
1250	if( !isPush ){
1251	cgi_reset_content();
1252	@ error not\sauthorized\sto\swrite
	@@ -1232,11 +1262,11 @@
1262	}
1263	}else
1264
1265	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
1266	**
1267	** Server accepts an unversioned file from the client.
1268	*/
1269	if( blob_eq(&xfer.aToken[0], "uvfile") ){
1270	xfer_accept_unversioned_file(&xfer, g.perm.WrUnver);
1271	if( blob_size(&xfer.err) ){
1272	cgi_reset_content();
	@@ -1246,11 +1276,11 @@
1276	}
1277	}else
1278
1279	/* gimme HASH
1280	**
1281	** Client is requesting a file from the server. Send it.
1282	*/
1283	if( blob_eq(&xfer.aToken[0], "gimme")
1284	&& xfer.nToken==2
1285	&& blob_is_hname(&xfer.aToken[1])
1286	){
	@@ -1263,11 +1293,11 @@
1293	}
1294	}else
1295
1296	/* uvgimme NAME
1297	**
1298	** Client is requesting an unversioned file from the server. Send it.
1299	*/
1300	if( blob_eq(&xfer.aToken[0], "uvgimme")
1301	&& xfer.nToken==2
1302	&& blob_is_filename(&xfer.aToken[1])
1303	){
	@@ -1275,24 +1305,38 @@
1305	}else
1306
1307	/* igot HASH ?ISPRIVATE?
1308	**
1309	** Client announces that it has a particular file. If the ISPRIVATE
1310	** argument exists and is "1", then the file is a private file.
1311	*/
1312	if( xfer.nToken>=2
1313	&& blob_eq(&xfer.aToken[0], "igot")
1314	&& blob_is_hname(&xfer.aToken[1])
1315	){
1316	if( isPush ){
1317	int rid = 0;
1318	if( xfer.nToken==2 \|\| blob_eq(&xfer.aToken[2],"1")==0 ){
1319	/* Client says the artifact is public */
1320	rid = rid_from_uuid(&xfer.aToken[1], 1, 0);
1321	}else if( g.perm.Private ){
1322	/* Client says the artifact is private and the client has
1323	** permission to push private content. Create a new phantom
1324	** artifact that is marked private. */
1325	rid = rid_from_uuid(&xfer.aToken[1], 1, 1);
1326	}else{
1327	/* Client says the artifact is private and the client is unable
1328	** or unwilling to send us the artifact. If we already hold the
1329	** artifact here on the server as a phantom, make sure that
1330	** phantom is marked as private so that we don't keep asking about
1331	** it in subsequent sync requests. */
1332	rid = rid_from_uuid(&xfer.aToken[1], 0, 1);
1333	if( rid>0 ){
1334	db_multi_exec("INSERT OR IGNORE INTO private(rid) VALUES(%d)",rid);
1335	}
1336	}
1337	if( rid ) remote_has(rid);
1338	}
1339	}else
1340
1341
1342	/* pull SERVERCODE PROJECTCODE
	@@ -1388,11 +1432,12 @@
1432	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1433	}else
1434
1435	/* login USER NONCE SIGNATURE
1436	**
1437	** The client has sent login credentials to the server.
1438	** Validate the login. This has to happen before anything else.
1439	** The client can send multiple logins. Permissions are cumulative.
1440	*/
1441	if( blob_eq(&xfer.aToken[0], "login")
1442	&& xfer.nToken==4
1443	){
	@@ -1410,11 +1455,11 @@
1455	}
1456	}else
1457
1458	/* reqconfig NAME
1459	**
1460	** Client is requesting a configuration value from the server
1461	*/
1462	if( blob_eq(&xfer.aToken[0], "reqconfig")
1463	&& xfer.nToken==2
1464	){
1465	if( g.perm.Read ){
	@@ -1429,12 +1474,12 @@
1474	}
1475	}else
1476
1477	/* config NAME SIZE \n CONTENT
1478	**
1479	** Client has sent a configuration value to the server.
1480	** This is only permitted for high-privilege users.
1481	*/
1482	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
1483	&& blob_is_int(&xfer.aToken[2], &size) ){
1484	const char *zName = blob_str(&xfer.aToken[1]);
1485	Blob content;
	@@ -1474,11 +1519,11 @@
1519	}else
1520
1521
1522	/* private
1523	**
1524	** The client card indicates that the next "file" or "cfile" will contain
1525	** private content.
1526	*/
1527	if( blob_eq(&xfer.aToken[0], "private") ){
1528	if( !g.perm.Private ){
1529	server_private_xfer_not_authorized();
	@@ -1495,10 +1540,12 @@
1540	** ignored.
1541	*/
1542	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
1543
1544	/* pragma send-private
1545	**
1546	** The client is requesting private artifacts.
1547	**
1548	** If the user has the "x" privilege (which must be set explicitly -
1549	** it is not automatic with "a" or "s") then this pragma causes
1550	** private information to be pulled in addition to public records.
1551	*/
	@@ -1511,22 +1558,32 @@
1558	}
1559	}
1560
1561	/* pragma send-catalog
1562	**
1563	** The client wants to see igot cards for all known artifacts.
1564	** This is used as part of "sync --verily" to help ensure that
1565	** no artifacts have been missed on prior syncs.
1566	*/
1567	if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1568	xfer.resync = 0x7fffffff;
1569	}
1570
1571	/* pragma client-version VERSION ?DATE? ?TIME?
1572	**
1573	** The client announces to the server what version of Fossil it
1574	** is running. The DATE and TIME are a pure numeric ISO8601 time
1575	** for the specific check-in of the client.
1576	*/
1577	if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
1578	xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
1579	if( xfer.nToken>=5 ){
1580	xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
1581	xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
1582	@ pragma server-version %d(RELEASE_VERSION_NUMBER) \
1583	@ %d(MANIFEST_NUMERIC_DATE) %d(MANIFEST_NUMERIC_TIME)
1584	}
1585	}
1586
1587	/* pragma uv-hash HASH
1588	**
1589	** The client wants to make sure that unversioned files are all synced.
	@@ -1550,11 +1607,11 @@
1607
1608	/* pragma ci-lock CHECKIN-HASH CLIENT-ID
1609	**
1610	** The client wants to make non-branch commit against the check-in
1611	** identified by CHECKIN-HASH. The server will remember this and
1612	** subsequent ci-lock requests from different clients will generate
1613	** a ci-lock-fail pragma in the reply.
1614	*/
1615	if( blob_eq(&xfer.aToken[1], "ci-lock")
1616	&& xfer.nToken==4
1617	&& blob_is_hname(&xfer.aToken[2])
	@@ -1808,11 +1865,11 @@
1865	memset(&xfer, 0, sizeof(xfer));
1866	xfer.pIn = &recv;
1867	xfer.pOut = &send;
1868	xfer.mxSend = db_get_int("max-upload", 250000);
1869	xfer.maxTime = -1;
1870	xfer.remoteVersion = RELEASE_VERSION_NUMBER;
1871	if( syncFlags & SYNC_PRIVATE ){
1872	g.perm.Private = 1;
1873	xfer.syncPrivate = 1;
1874	}
1875
	@@ -1856,13 +1913,16 @@
1913	" SELECT name, 0 FROM unversioned WHERE hash IS NOT NULL;"
1914	);
1915	}
1916
1917	/*
1918	** The request from the client always begin with a clone, pull,
1919	** or push message.
1920	*/
1921	blob_appendf(&send, "pragma client-version %d %d %d\n",
1922	RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
1923	MANIFEST_NUMERIC_TIME);
1924	if( syncFlags & SYNC_CLONE ){
1925	blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
1926	syncFlags &= ~(SYNC_PUSH\|SYNC_PULL);
1927	nCardSent++;
1928	/* TBD: Request all transferable configuration values */
	@@ -1898,20 +1958,19 @@
1958	"CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
1959	);
1960	manifest_crosslink_begin();
1961
1962
1963	/* Client sends the most recently received cookie back to the server.
1964	** Let the server figure out if this is a cookie that it cares about.
1965	*/
1966	zCookie = db_get("cookie", 0);
1967	if( zCookie ){
1968	blob_appendf(&send, "cookie %s\n", zCookie);
1969	}
1970
1971	/* Client sends gimme cards for phantoms

1972	*/
1973	if( (syncFlags & SYNC_PULL)!=0
1974	\|\| ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
1975	){
1976	request_phantoms(&xfer, mxPhantomReq);
	@@ -1920,11 +1979,11 @@
1979	send_unsent(&xfer);
1980	nCardSent += send_unclustered(&xfer);
1981	if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
1982	}
1983
1984	/* Client sends configuration parameter requests. On a clone, delay sending
1985	** this until the second cycle since the login card might fail on
1986	** the first cycle.
1987	*/
1988	if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0) ){
1989	const char *zName;
	@@ -1937,13 +1996,13 @@
1996	}
1997	origConfigRcvMask = configRcvMask;
1998	configRcvMask = 0;
1999	}
2000
2001	/* Client sends a request to sync unversioned files.
2002	** On a clone, delay sending this until the second cycle since
2003	** the login card might fail on the first cycle.
2004	*/
2005	if( (syncFlags & SYNC_UNVERSIONED)!=0
2006	&& ((syncFlags & SYNC_CLONE)==0 \|\| nCycle>0)
2007	&& !uvHashSent
2008	){
	@@ -1950,11 +2009,12 @@
2009	blob_appendf(&send, "pragma uv-hash %s\n", unversioned_content_hash(0));
2010	nCardSent++;
2011	uvHashSent = 1;
2012	}
2013
2014	/* On a "fossil config push", the client send configuration parameters
2015	** being pushed up to the server */
2016	if( configSendMask ){
2017	if( zOpType==0 ) zOpType = "Push";
2018	nCardSent += configure_send_group(xfer.pOut, configSendMask, 0);
2019	configSendMask = 0;
2020	}
	@@ -2010,11 +2070,11 @@
2070	zCkinLock = 0;
2071	}else if( zClientId ){
2072	blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
2073	}
2074
2075	/* Append randomness to the end of the uplink message. This makes all
2076	** messages unique so that that the login-card nonce will always
2077	** be unique.
2078	*/
2079	zRandomness = db_text(0, "SELECT hex(randomblob(20))");
2080	blob_appendf(&send, "# %s\n", zRandomness);
	@@ -2055,11 +2115,13 @@
2115	xfer.nGimmeSent = 0;
2116	xfer.nIGotSent = 0;
2117
2118	lastPctDone = -1;
2119	blob_reset(&send);
2120	blob_appendf(&send, "pragma client-version %d %d %d\n",
2121	RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
2122	MANIFEST_NUMERIC_TIME);
2123	rArrivalTime = db_double(0.0, "SELECT julianday('now')");
2124
2125	/* Send the send-private pragma if we are trying to sync private data */
2126	if( syncFlags & SYNC_PRIVATE ){
2127	blob_append(&send, "pragma send-private\n", -1);
	@@ -2112,30 +2174,30 @@
2174	}
2175
2176	/* file HASH SIZE \n CONTENT
2177	** file HASH DELTASRC SIZE \n CONTENT
2178	**
2179	** Client receives a file transmitted from the server.
2180	*/
2181	if( blob_eq(&xfer.aToken[0],"file") ){
2182	xfer_accept_file(&xfer, (syncFlags & SYNC_CLONE)!=0, 0, 0);
2183	nArtifactRcvd++;
2184	}else
2185
2186	/* cfile HASH USIZE CSIZE \n CONTENT
2187	** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
2188	**
2189	** Client receives a compressed file transmitted from the server.
2190	*/
2191	if( blob_eq(&xfer.aToken[0],"cfile") ){
2192	xfer_accept_compressed_file(&xfer, 0, 0);
2193	nArtifactRcvd++;
2194	}else
2195
2196	/* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
2197	**
2198	** Client accepts an unversioned file from the server.
2199	*/
2200	if( blob_eq(&xfer.aToken[0], "uvfile") ){
2201	xfer_accept_unversioned_file(&xfer, 1);
2202	nArtifactRcvd++;
2203	nUvFileRcvd++;
	@@ -2145,13 +2207,14 @@
2207	}
2208	}else
2209
2210	/* gimme HASH
2211	**
2212	** Client receives an artifact request from the server.
2213	** If the file is a manifest, assume that the server will also want
2214	** to know all of the content artifacts associated with the manifest
2215	** and send those too.
2216	*/
2217	if( blob_eq(&xfer.aToken[0], "gimme")
2218	&& xfer.nToken==2
2219	&& blob_is_hname(&xfer.aToken[1])
2220	){
	@@ -2182,10 +2245,11 @@
2245	rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2246	if( rid>0 ){
2247	if( !isPriv ) content_make_public(rid);
2248	}else if( isPriv && !g.perm.Private ){
2249	/* ignore private files */
2250	db_multi_exec("INSERT OR IGNORE INTO private VALUES(%d)", rid);
2251	}else if( (syncFlags & (SYNC_PULL\|SYNC_CLONE))!=0 ){
2252	rid = content_new(blob_str(&xfer.aToken[1]), isPriv);
2253	if( rid ) newPhantom = 1;
2254	}
2255	remote_has(rid);
	@@ -2264,11 +2328,11 @@
2328	}else
2329
2330	/* push SERVERCODE PRODUCTCODE
2331	**
2332	** Should only happen in response to a clone. This message tells
2333	** the client what product code to use for the new database.
2334	*/
2335	if( blob_eq(&xfer.aToken[0],"push")
2336	&& xfer.nToken==3
2337	&& (syncFlags & SYNC_CLONE)!=0
2338	&& blob_is_hname(&xfer.aToken[2])
	@@ -2281,11 +2345,11 @@
2345	nCardSent++;
2346	}else
2347
2348	/* config NAME SIZE \n CONTENT
2349	**
2350	** Client receive a configuration value from the server.
2351	**
2352	** The received configuration setting is silently ignored if it was
2353	** not requested by a prior "reqconfig" sent from client to server.
2354	*/
2355	if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
	@@ -2303,11 +2367,11 @@
2367	}else
2368
2369
2370	/* cookie TEXT
2371	**
2372	** The client reserves a cookie from the server. The client
2373	** should remember this cookie and send it back to the server
2374	** in its next query.
2375	**
2376	** Each cookie received overwrites the prior cookie from the
2377	** same server.
	@@ -2317,12 +2381,12 @@
2381	}else
2382
2383
2384	/* private
2385	**
2386	** The server tells the client that the next "file" or "cfile" will
2387	** contain private content.
2388	*/
2389	if( blob_eq(&xfer.aToken[0], "private") ){
2390	xfer.nextIsPrivate = 1;
2391	}else
2392
	@@ -2338,11 +2402,12 @@
2402	blob_is_int(&xfer.aToken[1], &cloneSeqno);
2403	}else
2404
2405	/* message MESSAGE
2406	**
2407	** A message is received from the server. Print it.
2408	** Similar to "error" but does not stop processing.
2409	**
2410	** If the "login failed" message is seen, clear the sync password prior
2411	** to the next cycle.
2412	*/
2413	if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
	@@ -2364,11 +2429,27 @@
2429	** The server can send pragmas to try to convey meta-information to
2430	** the client. These are informational only. Unknown pragmas are
2431	** silently ignored.
2432	*/
2433	if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
2434	/* pragma server-version VERSION ?DATE? ?TIME?
2435	**
2436	** The servger announces to the server what version of Fossil it
2437	** is running. The DATE and TIME are a pure numeric ISO8601 time
2438	** for the specific check-in of the client.
2439	*/
2440	if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
2441	xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
2442	if( xfer.nToken>=5 ){
2443	xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
2444	xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
2445	}
2446	}
2447
2448	/* pragma uv-pull-only
2449	**
2450	** If the server is unwill to accept new unversioned content (because
2451	** this client lacks the necessary permissions) then it sends a
2452	** "uv-pull-only" pragma so that the client will know not to waste
2453	** bandwidth trying to upload unversioned content. If the server
2454	** does accept new unversioned content, it sends "uv-push-ok".
2455	*/
	@@ -2407,11 +2488,12 @@
2488	}
2489	}else
2490
2491	/* error MESSAGE
2492	**
2493	** The server is reporting an error. The client will abandon
2494	** the sync session.
2495	**
2496	** Except, when cloning we will sometimes get an error on the
2497	** first message exchange because the project-code is unknown
2498	** and so the login card on the request was invalid. The project-code
2499	** is returned in the reply before the error card, so second and
2500

M www/sync.wiki

		--- www/sync.wiki
		+++ www/sync.wiki
		@@ -421,10 +421,13 @@
421	421	using a gimme card in either the reply or in the next message.</p>
422	422
423	423	<p>If the second argument exists and is "1", then the artifact
424	424	identified by the first argument is private on the sender and should
425	425	be ignored unless a "--private" [/help?cmd=sync\|sync] is occurring.
	426	+
	427	+<p>The name "igot" comes from the English slang expression "I got" meaning
	428	+"I have".
426	429
427	430	<h4>3.6.1 Unversioned Igot Cards</h4>
428	431
429	432	<p>Zero or more "uvigot" cards are sent from server to client when
430	433	synchronizing unversioned content. The format of a uvigot card is
		@@ -469,10 +472,14 @@
469	472	<p>The argument to the gimme card is the ID of the artifact that
470	473	the sender wants. The receiver will typically respond to a
471	474	gimme card by sending a file card in its reply or in the next
472	475	message.</p>
473	476
	477	+<p>The "gimme" name means "give me". The imperative "give me" is
	478	+pronounced as if it were a single word "gimme" in some dialects of
	479	+English (including the dialect spoken by the original author of Fossil).
	480	+
474	481	<h4>3.7.1 Unversioned Gimme Cards</h4>
475	482
476	483	<p>Sync synchronizing unversioned content, the client may send "uvgimme"
477	484	cards to the server. A uvgimme card requests that the server send
478	485	unversioned content to the client. The format of a uvgimme card is
479	486

	--- www/sync.wiki
	+++ www/sync.wiki
	@@ -421,10 +421,13 @@
421	using a gimme card in either the reply or in the next message.</p>
422
423	<p>If the second argument exists and is "1", then the artifact
424	identified by the first argument is private on the sender and should
425	be ignored unless a "--private" [/help?cmd=sync\|sync] is occurring.



426
427	<h4>3.6.1 Unversioned Igot Cards</h4>
428
429	<p>Zero or more "uvigot" cards are sent from server to client when
430	synchronizing unversioned content. The format of a uvigot card is
	@@ -469,10 +472,14 @@
469	<p>The argument to the gimme card is the ID of the artifact that
470	the sender wants. The receiver will typically respond to a
471	gimme card by sending a file card in its reply or in the next
472	message.</p>
473




474	<h4>3.7.1 Unversioned Gimme Cards</h4>
475
476	<p>Sync synchronizing unversioned content, the client may send "uvgimme"
477	cards to the server. A uvgimme card requests that the server send
478	unversioned content to the client. The format of a uvgimme card is
479

	--- www/sync.wiki
	+++ www/sync.wiki
	@@ -421,10 +421,13 @@
421	using a gimme card in either the reply or in the next message.</p>
422
423	<p>If the second argument exists and is "1", then the artifact
424	identified by the first argument is private on the sender and should
425	be ignored unless a "--private" [/help?cmd=sync\|sync] is occurring.
426
427	<p>The name "igot" comes from the English slang expression "I got" meaning
428	"I have".
429
430	<h4>3.6.1 Unversioned Igot Cards</h4>
431
432	<p>Zero or more "uvigot" cards are sent from server to client when
433	synchronizing unversioned content. The format of a uvigot card is
	@@ -469,10 +472,14 @@
472	<p>The argument to the gimme card is the ID of the artifact that
473	the sender wants. The receiver will typically respond to a
474	gimme card by sending a file card in its reply or in the next
475	message.</p>
476
477	<p>The "gimme" name means "give me". The imperative "give me" is
478	pronounced as if it were a single word "gimme" in some dialects of
479	English (including the dialect spoken by the original author of Fossil).
480
481	<h4>3.7.1 Unversioned Gimme Cards</h4>
482
483	<p>Sync synchronizing unversioned content, the client may send "uvgimme"
484	cards to the server. A uvgimme card requests that the server send
485	unversioned content to the client. The format of a uvgimme card is
486

Fossil SCM

Keyboard Shortcuts