Fossil SCM

Improvements to cloning performance.

drh 2008-05-18 15:51 trunk

Commit 243e02bfbdf5629131d8c6b0709e538ca8a33897

Parent a241c8111cd62ae…

6 files changed +8 +6 -4 +1 +23 -6 +3 -2 +66 -25

~ src/clone.c ~ src/content.c ~ src/main.c ~ src/manifest.c ~ src/schema.c ~ src/xfer.c

M src/clone.c

		--- src/clone.c
		+++ src/clone.c
		@@ -74,10 +74,18 @@
74	74	zTab, zTab);
75	75	}
76	76	db_finalize(&q);
77	77	}else{
78	78	url_enable_proxy(0);
	79	+ g.xlinkClusterOnly = 1;
79	80	client_sync(0,0,1);
	81	+ g.xlinkClusterOnly = 0;
80	82	}
81	83	verify_cancel();
82	84	db_end_transaction(0);
	85	+ db_close();
	86	+ db_open_repository(g.argv[3]);
	87	+ db_begin_transaction();
	88	+ printf("Rebuilding repository meta-data...\n");
	89	+ rebuild_db(0, 1);
	90	+ db_end_transaction(0);
83	91	}
84	92

	--- src/clone.c
	+++ src/clone.c
	@@ -74,10 +74,18 @@
74	zTab, zTab);
75	}
76	db_finalize(&q);
77	}else{
78	url_enable_proxy(0);

79	client_sync(0,0,1);

80	}
81	verify_cancel();
82	db_end_transaction(0);






83	}
84

	--- src/clone.c
	+++ src/clone.c
	@@ -74,10 +74,18 @@
74	zTab, zTab);
75	}
76	db_finalize(&q);
77	}else{
78	url_enable_proxy(0);
79	g.xlinkClusterOnly = 1;
80	client_sync(0,0,1);
81	g.xlinkClusterOnly = 0;
82	}
83	verify_cancel();
84	db_end_transaction(0);
85	db_close();
86	db_open_repository(g.argv[3]);
87	db_begin_transaction();
88	printf("Rebuilding repository meta-data...\n");
89	rebuild_db(0, 1);
90	db_end_transaction(0);
91	}
92

M src/content.c

+6 -4

		--- src/content.c
		+++ src/content.c
		@@ -365,13 +365,13 @@
365	365	** content is already in the database, just return the record ID.
366	366	**
367	367	** If srcId is specified, then pBlob is delta content from
368	368	** the srcId record. srcId might be a phantom.
369	369	**
370		-** If srcId is specified then the UUID is set to zUuid. Otherwise zUuid is
371		-** ignored. In the future this might change such that the content
372		-** hash is checked against zUuid to make sure it is correct.
	370	+** zUuid is the UUID of the artifact, if it is specified. When srcId is
	371	+** specified then zUuid must always be specified. If srcId is zero,
	372	+** and zUuid is zero then the correct zUuid is computed from pBlob.
373	373	**
374	374	** If the record already exists but is a phantom, the pBlob content
375	375	** is inserted and the phatom becomes a real record.
376	376	*/
377	377	int content_put(Blob pBlob, const char zUuid, int srcId){
		@@ -383,11 +383,13 @@
383	383	int markAsUnclustered = 0;
384	384	int isDephantomize = 0;
385	385
386	386	assert( g.repositoryOpen );
387	387	assert( pBlob!=0 );
388		- if( srcId==0 ){
	388	+ assert( srcId==0 \|\| zUuid!=0 );
	389	+ if( zUuid==0 ){
	390	+ assert( pBlob!=0 );
389	391	sha1sum_blob(pBlob, &hash);
390	392	}else{
391	393	blob_init(&hash, zUuid, -1);
392	394	}
393	395	size = blob_size(pBlob);
394	396

	--- src/content.c
	+++ src/content.c
	@@ -365,13 +365,13 @@
365	** content is already in the database, just return the record ID.
366	**
367	** If srcId is specified, then pBlob is delta content from
368	** the srcId record. srcId might be a phantom.
369	**
370	** If srcId is specified then the UUID is set to zUuid. Otherwise zUuid is
371	** ignored. In the future this might change such that the content
372	** hash is checked against zUuid to make sure it is correct.
373	**
374	** If the record already exists but is a phantom, the pBlob content
375	** is inserted and the phatom becomes a real record.
376	*/
377	int content_put(Blob pBlob, const char zUuid, int srcId){
	@@ -383,11 +383,13 @@
383	int markAsUnclustered = 0;
384	int isDephantomize = 0;
385
386	assert( g.repositoryOpen );
387	assert( pBlob!=0 );
388	if( srcId==0 ){


389	sha1sum_blob(pBlob, &hash);
390	}else{
391	blob_init(&hash, zUuid, -1);
392	}
393	size = blob_size(pBlob);
394

	--- src/content.c
	+++ src/content.c
	@@ -365,13 +365,13 @@
365	** content is already in the database, just return the record ID.
366	**
367	** If srcId is specified, then pBlob is delta content from
368	** the srcId record. srcId might be a phantom.
369	**
370	** zUuid is the UUID of the artifact, if it is specified. When srcId is
371	** specified then zUuid must always be specified. If srcId is zero,
372	** and zUuid is zero then the correct zUuid is computed from pBlob.
373	**
374	** If the record already exists but is a phantom, the pBlob content
375	** is inserted and the phatom becomes a real record.
376	*/
377	int content_put(Blob pBlob, const char zUuid, int srcId){
	@@ -383,11 +383,13 @@
383	int markAsUnclustered = 0;
384	int isDephantomize = 0;
385
386	assert( g.repositoryOpen );
387	assert( pBlob!=0 );
388	assert( srcId==0 \|\| zUuid!=0 );
389	if( zUuid==0 ){
390	assert( pBlob!=0 );
391	sha1sum_blob(pBlob, &hash);
392	}else{
393	blob_init(&hash, zUuid, -1);
394	}
395	size = blob_size(pBlob);
396

M src/main.c

		--- src/main.c
		+++ src/main.c
		@@ -74,10 +74,11 @@
74	74	Blob cgiIn; /* Input to an xfer www method */
75	75	int cgiPanic; /* Write error messages to CGI */
76	76	Th_Interp interp; / The TH1 interpreter */
77	77	FILE httpIn; / Accept HTTP input from here */
78	78	FILE httpOut; / Send HTTP output here */
	79	+ int xlinkClusterOnly; /* Set when cloning. Only process clusters */
79	80
80	81	int aCommitFile; / Array of files to be committed */
81	82
82	83	int urlIsFile; /* True if a "file:" url */
83	84	char urlName; / Hostname for http: or filename for file: */
84	85

	--- src/main.c
	+++ src/main.c
	@@ -74,10 +74,11 @@
74	Blob cgiIn; /* Input to an xfer www method */
75	int cgiPanic; /* Write error messages to CGI */
76	Th_Interp interp; / The TH1 interpreter */
77	FILE httpIn; / Accept HTTP input from here */
78	FILE httpOut; / Send HTTP output here */

79
80	int aCommitFile; / Array of files to be committed */
81
82	int urlIsFile; /* True if a "file:" url */
83	char urlName; / Hostname for http: or filename for file: */
84

	--- src/main.c
	+++ src/main.c
	@@ -74,10 +74,11 @@
74	Blob cgiIn; /* Input to an xfer www method */
75	int cgiPanic; /* Write error messages to CGI */
76	Th_Interp interp; / The TH1 interpreter */
77	FILE httpIn; / Accept HTTP input from here */
78	FILE httpOut; / Send HTTP output here */
79	int xlinkClusterOnly; /* Set when cloning. Only process clusters */
80
81	int aCommitFile; / Array of files to be committed */
82
83	int urlIsFile; /* True if a "file:" url */
84	char urlName; / Hostname for http: or filename for file: */
85

M src/manifest.c

+23 -6

		--- src/manifest.c
		+++ src/manifest.c
		@@ -754,26 +754,43 @@
754	754	}
755	755	manifest_clear(&other);
756	756	}
757	757
758	758	/*
759		-** Scan record rid/pContent to see if it is a manifest. If
760		-** it is a manifest, then populate the mlink, plink,
761		-** filename, and event tables with cross-reference information.
	759	+** Scan artifact rid/pContent to see if it is a control artifact of
	760	+** any key:
	761	+**
	762	+** * Manifest
	763	+** * Control
	764	+** * Wiki Page
	765	+** * Ticket Change
	766	+** * Cluster
	767	+**
	768	+** If the input is a control artifact, then make appropriate entries
	769	+** in the auxiliary tables of the database in order to crosslink the
	770	+** artifact.
	771	+**
	772	+** If global variable g.xlinkClusterOnly is true, then ignore all
	773	+** control artifacts other than clusters.
762	774	**
763		-** (Later:) Also check to see if pContent is a cluster. If it
764		-** is a cluster then remove all referenced elements from the
765		-** unclustered table and create phantoms for any unknown elements.
	775	+** Historical note: This routine original processed manifests only.
	776	+** Processing for other control artifacts was added later. The name
	777	+** of the routine, "manifest_crosslink", and the name of this source
	778	+** file, is a legacy of its original use.
766	779	*/
767	780	int manifest_crosslink(int rid, Blob *pContent){
768	781	int i;
769	782	Manifest m;
770	783	Stmt q;
771	784	int parentid = 0;
772	785
773	786	if( manifest_parse(&m, pContent)==0 ){
774	787	return 0;
	788	+ }
	789	+ if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
	790	+ manifest_clear(&m);
	791	+ return 0;
775	792	}
776	793	db_begin_transaction();
777	794	if( m.type==CFTYPE_MANIFEST ){
778	795	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
779	796	for(i=0; i<m.nParent; i++){
780	797

	--- src/manifest.c
	+++ src/manifest.c
	@@ -754,26 +754,43 @@
754	}
755	manifest_clear(&other);
756	}
757
758	/*
759	** Scan record rid/pContent to see if it is a manifest. If
760	** it is a manifest, then populate the mlink, plink,
761	** filename, and event tables with cross-reference information.












762	**
763	** (Later:) Also check to see if pContent is a cluster. If it
764	** is a cluster then remove all referenced elements from the
765	** unclustered table and create phantoms for any unknown elements.

766	*/
767	int manifest_crosslink(int rid, Blob *pContent){
768	int i;
769	Manifest m;
770	Stmt q;
771	int parentid = 0;
772
773	if( manifest_parse(&m, pContent)==0 ){
774	return 0;




775	}
776	db_begin_transaction();
777	if( m.type==CFTYPE_MANIFEST ){
778	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
779	for(i=0; i<m.nParent; i++){
780

	--- src/manifest.c
	+++ src/manifest.c
	@@ -754,26 +754,43 @@
754	}
755	manifest_clear(&other);
756	}
757
758	/*
759	** Scan artifact rid/pContent to see if it is a control artifact of
760	** any key:
761	**
762	** * Manifest
763	** * Control
764	** * Wiki Page
765	** * Ticket Change
766	** * Cluster
767	**
768	** If the input is a control artifact, then make appropriate entries
769	** in the auxiliary tables of the database in order to crosslink the
770	** artifact.
771	**
772	** If global variable g.xlinkClusterOnly is true, then ignore all
773	** control artifacts other than clusters.
774	**
775	** Historical note: This routine original processed manifests only.
776	** Processing for other control artifacts was added later. The name
777	** of the routine, "manifest_crosslink", and the name of this source
778	** file, is a legacy of its original use.
779	*/
780	int manifest_crosslink(int rid, Blob *pContent){
781	int i;
782	Manifest m;
783	Stmt q;
784	int parentid = 0;
785
786	if( manifest_parse(&m, pContent)==0 ){
787	return 0;
788	}
789	if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
790	manifest_clear(&m);
791	return 0;
792	}
793	db_begin_transaction();
794	if( m.type==CFTYPE_MANIFEST ){
795	if( !db_exists("SELECT 1 FROM mlink WHERE mid=%d", rid) ){
796	for(i=0; i<m.nParent; i++){
797

M src/schema.c

+3 -2

		--- src/schema.c
		+++ src/schema.c
		@@ -73,12 +73,13 @@
73	73	@ --
74	74	@ CREATE TABLE blob(
75	75	@ rid INTEGER PRIMARY KEY, -- Record ID
76	76	@ rcvid INTEGER, -- Origin of this record
77	77	@ size INTEGER, -- Size of content. -1 for a phantom.
78		-@ uuid TEXT UNIQUE, -- SHA1 hash of the content
79		-@ content BLOB -- Compressed content of this record
	78	+@ uuid TEXT UNIQUE NOT NULL, -- SHA1 hash of the content
	79	+@ content BLOB, -- Compressed content of this record
	80	+@ CHECK( length(uuid)==40 AND rid>0 )
80	81	@ );
81	82	@ CREATE TABLE delta(
82	83	@ rid INTEGER PRIMARY KEY, -- Record ID
83	84	@ srcid INTEGER NOT NULL REFERENCES blob -- Record holding source document
84	85	@ );
85	86

	--- src/schema.c
	+++ src/schema.c
	@@ -73,12 +73,13 @@
73	@ --
74	@ CREATE TABLE blob(
75	@ rid INTEGER PRIMARY KEY, -- Record ID
76	@ rcvid INTEGER, -- Origin of this record
77	@ size INTEGER, -- Size of content. -1 for a phantom.
78	@ uuid TEXT UNIQUE, -- SHA1 hash of the content
79	@ content BLOB -- Compressed content of this record

80	@ );
81	@ CREATE TABLE delta(
82	@ rid INTEGER PRIMARY KEY, -- Record ID
83	@ srcid INTEGER NOT NULL REFERENCES blob -- Record holding source document
84	@ );
85

	--- src/schema.c
	+++ src/schema.c
	@@ -73,12 +73,13 @@
73	@ --
74	@ CREATE TABLE blob(
75	@ rid INTEGER PRIMARY KEY, -- Record ID
76	@ rcvid INTEGER, -- Origin of this record
77	@ size INTEGER, -- Size of content. -1 for a phantom.
78	@ uuid TEXT UNIQUE NOT NULL, -- SHA1 hash of the content
79	@ content BLOB, -- Compressed content of this record
80	@ CHECK( length(uuid)==40 AND rid>0 )
81	@ );
82	@ CREATE TABLE delta(
83	@ rid INTEGER PRIMARY KEY, -- Record ID
84	@ srcid INTEGER NOT NULL REFERENCES blob -- Record holding source document
85	@ );
86

M src/xfer.c

+66 -25

		--- src/xfer.c
		+++ src/xfer.c
		@@ -65,12 +65,12 @@
65	65	Blob pOut; / Compose our reply here */
66	66	Blob line; /* The current line of input */
67	67	Blob aToken[5]; /* Tokenized version of line */
68	68	Blob err; /* Error message text */
69	69	int nToken; /* Number of tokens in line */
70		- int nIGotSent; /* Number of "igot" messages sent */
71		- int nGimmeSent; /* Number of gimme messages sent */
	70	+ int nIGotSent; /* Number of "igot" cards sent */
	71	+ int nGimmeSent; /* Number of gimme cards sent */
72	72	int nFileSent; /* Number of files sent */
73	73	int nDeltaSent; /* Number of deltas sent */
74	74	int nFileRcvd; /* Number of files received */
75	75	int nDeltaRcvd; /* Number of deltas received */
76	76	int nDanglingFile; /* Number of dangling deltas received */
		@@ -170,15 +170,16 @@
170	170	}
171	171	sha1sum_blob(&content, &hash);
172	172	if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){
173	173	blob_appendf(&pXfer->err, "content does not match sha1 hash");
174	174	}
	175	+ rid = content_put(&content, blob_str(&hash), 0);
175	176	blob_reset(&hash);
176		- rid = content_put(&content, 0, 0);
177	177	if( rid==0 ){
178	178	blob_appendf(&pXfer->err, "%s", g.zErrMsg);
179	179	}else{
	180	+ /* db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid); */
180	181	manifest_crosslink(rid, &content);
181	182	}
182	183	remote_has(rid);
183	184	}
184	185
		@@ -460,11 +461,11 @@
460	461	blob_reset(&cluster);
461	462	}
462	463
463	464	/*
464	465	** Send an igot message for every entry in unclustered table.
465		-** Return the number of messages sent.
	466	+** Return the number of cards sent.
466	467	*/
467	468	static int send_unclustered(Xfer *pXfer){
468	469	Stmt q;
469	470	int cnt = 0;
470	471	db_prepare(&q,
		@@ -476,10 +477,22 @@
476	477	cnt++;
477	478	}
478	479	db_finalize(&q);
479	480	return cnt;
480	481	}
	482	+
	483	+/*
	484	+** Send an igot message for every artifact.
	485	+*/
	486	+static void send_all(Xfer *pXfer){
	487	+ Stmt q;
	488	+ db_prepare(&q, "SELECT uuid FROM blob");
	489	+ while( db_step(&q)==SQLITE_ROW ){
	490	+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
	491	+ }
	492	+ db_finalize(&q);
	493	+}
481	494
482	495	/*
483	496	** If this variable is set, disable login checks. Used for debugging
484	497	** only.
485	498	*/
		@@ -496,10 +509,12 @@
496	509	int isPull = 0;
497	510	int isPush = 0;
498	511	int nErr = 0;
499	512	Xfer xfer;
500	513	int deltaFlag = 0;
	514	+ int isClone = 0;
	515	+ int nGimme = 0;
501	516
502	517	memset(&xfer, 0, sizeof(xfer));
503	518	blobarray_zero(xfer.aToken, count(xfer.aToken));
504	519	cgi_set_content_type(g.zContentType);
505	520	blob_zero(&xfer.err);
		@@ -541,10 +556,11 @@
541	556	*/
542	557	if( blob_eq(&xfer.aToken[0], "gimme")
543	558	&& xfer.nToken==2
544	559	&& blob_is_uuid(&xfer.aToken[1])
545	560	){
	561	+ nGimme++;
546	562	if( isPull ){
547	563	int rid = rid_from_uuid(&xfer.aToken[1], 0);
548	564	if( rid ){
549	565	send_file(&xfer, rid, &xfer.aToken[1], deltaFlag);
550	566	}
		@@ -634,10 +650,11 @@
634	650	cgi_reset_content();
635	651	@ error not\sauthorized\sto\sclone
636	652	nErr++;
637	653	break;
638	654	}
	655	+ isClone = 1;
639	656	isPull = 1;
640	657	deltaFlag = 1;
641	658	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
642	659	}else
643	660
		@@ -698,11 +715,20 @@
698	715	blobarray_reset(xfer.aToken, xfer.nToken);
699	716	}
700	717	if( isPush ){
701	718	request_phantoms(&xfer, 500);
702	719	}
703		- if( isPull ){
	720	+ if( isClone && nGimme==0 ){
	721	+ /* The initial "clone" message from client to server contains no
	722	+ ** "gimme" cards. On that initial message, send the client an "igot"
	723	+ ** card for every artifact currently in the respository. This will
	724	+ ** cause the client to create phantoms for all artifacts, which will
	725	+ ** in turn make sure that the entire repository is sent efficiently
	726	+ ** and expeditiously.
	727	+ */
	728	+ send_all(&xfer);
	729	+ }else if( isPull ){
704	730	create_cluster();
705	731	send_unclustered(&xfer);
706	732	}
707	733	db_end_transaction(0);
708	734	}
		@@ -738,10 +764,16 @@
738	764	disableLogin = 1;
739	765	page_xfer();
740	766	printf("%s\n", cgi_extract_content(&notUsed));
741	767	}
742	768
	769	+/*
	770	+** Format strings for progress reporting.
	771	+*/
	772	+static const char zLabel[] = "%-10s %10s %10s %10s %10s %10s\n";
	773	+static const char zValue[] = "\r%-10s %10d %10d %10d %10d %10d\n";
	774	+
743	775
744	776	/*
745	777	** Sync to the host identified in g.urlName and g.urlPath. This
746	778	** routine is called by the client.
747	779	**
		@@ -751,11 +783,11 @@
751	783	*/
752	784	void client_sync(int pushFlag, int pullFlag, int cloneFlag){
753	785	int go = 1; /* Loop until zero */
754	786	const char *zSCode = db_get("server-code", "x");
755	787	const char *zPCode = db_get("project-code", 0);
756		- int nMsg = 0; /* Number of messages sent or received */
	788	+ int nCard = 0; /* Number of cards sent or received */
757	789	int nCycle = 0; /* Number of round trips to the server */
758	790	int nFileSend = 0;
759	791	int nFileRecv; /* Number of files received */
760	792	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
761	793	const char zCookie; / Server cookie */
		@@ -786,21 +818,21 @@
786	818	*/
787	819	if( cloneFlag ){
788	820	blob_appendf(&send, "clone\n");
789	821	pushFlag = 0;
790	822	pullFlag = 0;
791		- nMsg++;
	823	+ nCard++;
792	824	/* TBD: Request all transferable configuration values */
793	825	}else if( pullFlag ){
794	826	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
795		- nMsg++;
	827	+ nCard++;
796	828	}
797	829	if( pushFlag ){
798	830	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
799		- nMsg++;
	831	+ nCard++;
800	832	}
801		-
	833	+ printf(zLabel, "", "Bytes", "Cards", "Artifacts", "Deltas", "Dangling");
802	834
803	835	while( go ){
804	836	int newPhantom = 0;
805	837
806	838	/* Send make the most recently received cookie. Let the server
		@@ -809,54 +841,59 @@
809	841	zCookie = db_get("cookie", 0);
810	842	if( zCookie ){
811	843	blob_appendf(&send, "cookie %s\n", zCookie);
812	844	}
813	845
814		- /* Generate gimme messages for phantoms and leaf messages
	846	+ /* Generate gimme cards for phantoms and leaf cards
815	847	** for all leaves.
816	848	*/
817	849	if( pullFlag \|\| cloneFlag ){
818	850	request_phantoms(&xfer, mxPhantomReq);
819	851	}
820	852	if( pushFlag ){
821	853	send_unsent(&xfer);
822		- nMsg += send_unclustered(&xfer);
	854	+ nCard += send_unclustered(&xfer);
823	855	}
824	856
825	857	/* Exchange messages with the server */
826	858	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
827		- printf("Sent: %10d bytes, %5d messages, %5d files (%d+%d)\n",
828		- blob_size(&send), nMsg+xfer.nGimmeSent+xfer.nIGotSent,
	859	+ printf(zValue, "Send:",
	860	+ blob_size(&send), nCard+xfer.nGimmeSent+xfer.nIGotSent,
	861	+ xfer.nFileSent, xfer.nDeltaSent, 0);
	862	+#if 0
	863	+ printf("Sent: %10d bytes, %5d cards, %5d files (%d+%d)\n",
	864	+ blob_size(&send), nCard+xfer.nGimmeSent+xfer.nIGotSent,
829	865	nFileSend, xfer.nFileSent, xfer.nDeltaSent);
830		- nMsg = 0;
	866	+#endif
	867	+ nCard = 0;
831	868	xfer.nFileSent = 0;
832	869	xfer.nDeltaSent = 0;
833	870	xfer.nGimmeSent = 0;
834	871	fflush(stdout);
835	872	http_exchange(&send, &recv);
836	873	blob_reset(&send);
837	874
838	875	/* Begin constructing the next message (which might never be
839		- ** sent) by beginning with the pull or push messages
	876	+ ** sent) by beginning with the pull or push cards
840	877	*/
841	878	if( pullFlag ){
842	879	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
843		- nMsg++;
	880	+ nCard++;
844	881	}
845	882	if( pushFlag ){
846	883	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
847		- nMsg++;
	884	+ nCard++;
848	885	}
849	886
850	887	/* Process the reply that came back from the server */
851	888	while( blob_line(&recv, &xfer.line) ){
852	889	if( blob_buffer(&xfer.line)[0]=='#' ){
853	890	continue;
854	891	}
855	892	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
856		- nMsg++;
857		- printf("\r%d", nMsg);
	893	+ nCard++;
	894	+ printf("\r%d", nCard);
858	895	fflush(stdout);
859	896
860	897	/* file UUID SIZE \n CONTENT
861	898	** file UUID DELTASRC SIZE \n CONTENT
862	899	**
		@@ -920,11 +957,11 @@
920	957	if( zPCode==0 ){
921	958	zPCode = mprintf("%b", &xfer.aToken[2]);
922	959	db_set("project-code", zPCode, 0);
923	960	}
924	961	blob_appendf(&send, "clone\n");
925		- nMsg++;
	962	+ nCard++;
926	963	}else
927	964
928	965	/* config NAME SIZE \n CONTENT
929	966	**
930	967	** Receive a configuration value from the server.
		@@ -978,15 +1015,19 @@
978	1015	fossil_fatal("%b", &xfer.err);
979	1016	}
980	1017	blobarray_reset(xfer.aToken, xfer.nToken);
981	1018	blob_reset(&xfer.line);
982	1019	}
983		- printf("\rReceived: %10d bytes, %5d messages, %5d files (%d+%d+%d)\n",
984		- blob_size(&recv), nMsg,
	1020	+ printf(zValue, "Received:",
	1021	+ blob_size(&recv), nCard,
	1022	+ xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
	1023	+#if 0
	1024	+ printf("\rReceived: %10d bytes, %5d cards, %5d files (%d+%d+%d)\n",
	1025	+ blob_size(&recv), nCard,
985	1026	xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile,
986	1027	xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
987		-
	1028	+#endif
988	1029	blob_reset(&recv);
989	1030	nCycle++;
990	1031	go = 0;
991	1032
992	1033	/* If we received one or more files on the previous exchange but
		@@ -996,11 +1037,11 @@
996	1037	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
997	1038	go = 1;
998	1039	mxPhantomReq = nFileRecv*2;
999	1040	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1000	1041	}
1001		- nMsg = 0;
	1042	+ nCard = 0;
1002	1043	xfer.nFileRcvd = 0;
1003	1044	xfer.nDeltaRcvd = 0;
1004	1045	xfer.nDanglingFile = 0;
1005	1046
1006	1047	/* If we have one or more files queued to send, then go
1007	1048

	--- src/xfer.c
	+++ src/xfer.c
	@@ -65,12 +65,12 @@
65	Blob pOut; / Compose our reply here */
66	Blob line; /* The current line of input */
67	Blob aToken[5]; /* Tokenized version of line */
68	Blob err; /* Error message text */
69	int nToken; /* Number of tokens in line */
70	int nIGotSent; /* Number of "igot" messages sent */
71	int nGimmeSent; /* Number of gimme messages sent */
72	int nFileSent; /* Number of files sent */
73	int nDeltaSent; /* Number of deltas sent */
74	int nFileRcvd; /* Number of files received */
75	int nDeltaRcvd; /* Number of deltas received */
76	int nDanglingFile; /* Number of dangling deltas received */
	@@ -170,15 +170,16 @@
170	}
171	sha1sum_blob(&content, &hash);
172	if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){
173	blob_appendf(&pXfer->err, "content does not match sha1 hash");
174	}

175	blob_reset(&hash);
176	rid = content_put(&content, 0, 0);
177	if( rid==0 ){
178	blob_appendf(&pXfer->err, "%s", g.zErrMsg);
179	}else{

180	manifest_crosslink(rid, &content);
181	}
182	remote_has(rid);
183	}
184
	@@ -460,11 +461,11 @@
460	blob_reset(&cluster);
461	}
462
463	/*
464	** Send an igot message for every entry in unclustered table.
465	** Return the number of messages sent.
466	*/
467	static int send_unclustered(Xfer *pXfer){
468	Stmt q;
469	int cnt = 0;
470	db_prepare(&q,
	@@ -476,10 +477,22 @@
476	cnt++;
477	}
478	db_finalize(&q);
479	return cnt;
480	}












481
482	/*
483	** If this variable is set, disable login checks. Used for debugging
484	** only.
485	*/
	@@ -496,10 +509,12 @@
496	int isPull = 0;
497	int isPush = 0;
498	int nErr = 0;
499	Xfer xfer;
500	int deltaFlag = 0;


501
502	memset(&xfer, 0, sizeof(xfer));
503	blobarray_zero(xfer.aToken, count(xfer.aToken));
504	cgi_set_content_type(g.zContentType);
505	blob_zero(&xfer.err);
	@@ -541,10 +556,11 @@
541	*/
542	if( blob_eq(&xfer.aToken[0], "gimme")
543	&& xfer.nToken==2
544	&& blob_is_uuid(&xfer.aToken[1])
545	){

546	if( isPull ){
547	int rid = rid_from_uuid(&xfer.aToken[1], 0);
548	if( rid ){
549	send_file(&xfer, rid, &xfer.aToken[1], deltaFlag);
550	}
	@@ -634,10 +650,11 @@
634	cgi_reset_content();
635	@ error not\sauthorized\sto\sclone
636	nErr++;
637	break;
638	}

639	isPull = 1;
640	deltaFlag = 1;
641	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
642	}else
643
	@@ -698,11 +715,20 @@
698	blobarray_reset(xfer.aToken, xfer.nToken);
699	}
700	if( isPush ){
701	request_phantoms(&xfer, 500);
702	}
703	if( isPull ){









704	create_cluster();
705	send_unclustered(&xfer);
706	}
707	db_end_transaction(0);
708	}
	@@ -738,10 +764,16 @@
738	disableLogin = 1;
739	page_xfer();
740	printf("%s\n", cgi_extract_content(&notUsed));
741	}
742






743
744	/*
745	** Sync to the host identified in g.urlName and g.urlPath. This
746	** routine is called by the client.
747	**
	@@ -751,11 +783,11 @@
751	*/
752	void client_sync(int pushFlag, int pullFlag, int cloneFlag){
753	int go = 1; /* Loop until zero */
754	const char *zSCode = db_get("server-code", "x");
755	const char *zPCode = db_get("project-code", 0);
756	int nMsg = 0; /* Number of messages sent or received */
757	int nCycle = 0; /* Number of round trips to the server */
758	int nFileSend = 0;
759	int nFileRecv; /* Number of files received */
760	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
761	const char zCookie; / Server cookie */
	@@ -786,21 +818,21 @@
786	*/
787	if( cloneFlag ){
788	blob_appendf(&send, "clone\n");
789	pushFlag = 0;
790	pullFlag = 0;
791	nMsg++;
792	/* TBD: Request all transferable configuration values */
793	}else if( pullFlag ){
794	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
795	nMsg++;
796	}
797	if( pushFlag ){
798	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
799	nMsg++;
800	}
801
802
803	while( go ){
804	int newPhantom = 0;
805
806	/* Send make the most recently received cookie. Let the server
	@@ -809,54 +841,59 @@
809	zCookie = db_get("cookie", 0);
810	if( zCookie ){
811	blob_appendf(&send, "cookie %s\n", zCookie);
812	}
813
814	/* Generate gimme messages for phantoms and leaf messages
815	** for all leaves.
816	*/
817	if( pullFlag \|\| cloneFlag ){
818	request_phantoms(&xfer, mxPhantomReq);
819	}
820	if( pushFlag ){
821	send_unsent(&xfer);
822	nMsg += send_unclustered(&xfer);
823	}
824
825	/* Exchange messages with the server */
826	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
827	printf("Sent: %10d bytes, %5d messages, %5d files (%d+%d)\n",
828	blob_size(&send), nMsg+xfer.nGimmeSent+xfer.nIGotSent,




829	nFileSend, xfer.nFileSent, xfer.nDeltaSent);
830	nMsg = 0;

831	xfer.nFileSent = 0;
832	xfer.nDeltaSent = 0;
833	xfer.nGimmeSent = 0;
834	fflush(stdout);
835	http_exchange(&send, &recv);
836	blob_reset(&send);
837
838	/* Begin constructing the next message (which might never be
839	** sent) by beginning with the pull or push messages
840	*/
841	if( pullFlag ){
842	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
843	nMsg++;
844	}
845	if( pushFlag ){
846	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
847	nMsg++;
848	}
849
850	/* Process the reply that came back from the server */
851	while( blob_line(&recv, &xfer.line) ){
852	if( blob_buffer(&xfer.line)[0]=='#' ){
853	continue;
854	}
855	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
856	nMsg++;
857	printf("\r%d", nMsg);
858	fflush(stdout);
859
860	/* file UUID SIZE \n CONTENT
861	** file UUID DELTASRC SIZE \n CONTENT
862	**
	@@ -920,11 +957,11 @@
920	if( zPCode==0 ){
921	zPCode = mprintf("%b", &xfer.aToken[2]);
922	db_set("project-code", zPCode, 0);
923	}
924	blob_appendf(&send, "clone\n");
925	nMsg++;
926	}else
927
928	/* config NAME SIZE \n CONTENT
929	**
930	** Receive a configuration value from the server.
	@@ -978,15 +1015,19 @@
978	fossil_fatal("%b", &xfer.err);
979	}
980	blobarray_reset(xfer.aToken, xfer.nToken);
981	blob_reset(&xfer.line);
982	}
983	printf("\rReceived: %10d bytes, %5d messages, %5d files (%d+%d+%d)\n",
984	blob_size(&recv), nMsg,




985	xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile,
986	xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
987
988	blob_reset(&recv);
989	nCycle++;
990	go = 0;
991
992	/* If we received one or more files on the previous exchange but
	@@ -996,11 +1037,11 @@
996	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
997	go = 1;
998	mxPhantomReq = nFileRecv*2;
999	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1000	}
1001	nMsg = 0;
1002	xfer.nFileRcvd = 0;
1003	xfer.nDeltaRcvd = 0;
1004	xfer.nDanglingFile = 0;
1005
1006	/* If we have one or more files queued to send, then go
1007

	--- src/xfer.c
	+++ src/xfer.c
	@@ -65,12 +65,12 @@
65	Blob pOut; / Compose our reply here */
66	Blob line; /* The current line of input */
67	Blob aToken[5]; /* Tokenized version of line */
68	Blob err; /* Error message text */
69	int nToken; /* Number of tokens in line */
70	int nIGotSent; /* Number of "igot" cards sent */
71	int nGimmeSent; /* Number of gimme cards sent */
72	int nFileSent; /* Number of files sent */
73	int nDeltaSent; /* Number of deltas sent */
74	int nFileRcvd; /* Number of files received */
75	int nDeltaRcvd; /* Number of deltas received */
76	int nDanglingFile; /* Number of dangling deltas received */
	@@ -170,15 +170,16 @@
170	}
171	sha1sum_blob(&content, &hash);
172	if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){
173	blob_appendf(&pXfer->err, "content does not match sha1 hash");
174	}
175	rid = content_put(&content, blob_str(&hash), 0);
176	blob_reset(&hash);

177	if( rid==0 ){
178	blob_appendf(&pXfer->err, "%s", g.zErrMsg);
179	}else{
180	/* db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid); */
181	manifest_crosslink(rid, &content);
182	}
183	remote_has(rid);
184	}
185
	@@ -460,11 +461,11 @@
461	blob_reset(&cluster);
462	}
463
464	/*
465	** Send an igot message for every entry in unclustered table.
466	** Return the number of cards sent.
467	*/
468	static int send_unclustered(Xfer *pXfer){
469	Stmt q;
470	int cnt = 0;
471	db_prepare(&q,
	@@ -476,10 +477,22 @@
477	cnt++;
478	}
479	db_finalize(&q);
480	return cnt;
481	}
482
483	/*
484	** Send an igot message for every artifact.
485	*/
486	static void send_all(Xfer *pXfer){
487	Stmt q;
488	db_prepare(&q, "SELECT uuid FROM blob");
489	while( db_step(&q)==SQLITE_ROW ){
490	blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
491	}
492	db_finalize(&q);
493	}
494
495	/*
496	** If this variable is set, disable login checks. Used for debugging
497	** only.
498	*/
	@@ -496,10 +509,12 @@
509	int isPull = 0;
510	int isPush = 0;
511	int nErr = 0;
512	Xfer xfer;
513	int deltaFlag = 0;
514	int isClone = 0;
515	int nGimme = 0;
516
517	memset(&xfer, 0, sizeof(xfer));
518	blobarray_zero(xfer.aToken, count(xfer.aToken));
519	cgi_set_content_type(g.zContentType);
520	blob_zero(&xfer.err);
	@@ -541,10 +556,11 @@
556	*/
557	if( blob_eq(&xfer.aToken[0], "gimme")
558	&& xfer.nToken==2
559	&& blob_is_uuid(&xfer.aToken[1])
560	){
561	nGimme++;
562	if( isPull ){
563	int rid = rid_from_uuid(&xfer.aToken[1], 0);
564	if( rid ){
565	send_file(&xfer, rid, &xfer.aToken[1], deltaFlag);
566	}
	@@ -634,10 +650,11 @@
650	cgi_reset_content();
651	@ error not\sauthorized\sto\sclone
652	nErr++;
653	break;
654	}
655	isClone = 1;
656	isPull = 1;
657	deltaFlag = 1;
658	@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
659	}else
660
	@@ -698,11 +715,20 @@
715	blobarray_reset(xfer.aToken, xfer.nToken);
716	}
717	if( isPush ){
718	request_phantoms(&xfer, 500);
719	}
720	if( isClone && nGimme==0 ){
721	/* The initial "clone" message from client to server contains no
722	** "gimme" cards. On that initial message, send the client an "igot"
723	** card for every artifact currently in the respository. This will
724	** cause the client to create phantoms for all artifacts, which will
725	** in turn make sure that the entire repository is sent efficiently
726	** and expeditiously.
727	*/
728	send_all(&xfer);
729	}else if( isPull ){
730	create_cluster();
731	send_unclustered(&xfer);
732	}
733	db_end_transaction(0);
734	}
	@@ -738,10 +764,16 @@
764	disableLogin = 1;
765	page_xfer();
766	printf("%s\n", cgi_extract_content(&notUsed));
767	}
768
769	/*
770	** Format strings for progress reporting.
771	*/
772	static const char zLabel[] = "%-10s %10s %10s %10s %10s %10s\n";
773	static const char zValue[] = "\r%-10s %10d %10d %10d %10d %10d\n";
774
775
776	/*
777	** Sync to the host identified in g.urlName and g.urlPath. This
778	** routine is called by the client.
779	**
	@@ -751,11 +783,11 @@
783	*/
784	void client_sync(int pushFlag, int pullFlag, int cloneFlag){
785	int go = 1; /* Loop until zero */
786	const char *zSCode = db_get("server-code", "x");
787	const char *zPCode = db_get("project-code", 0);
788	int nCard = 0; /* Number of cards sent or received */
789	int nCycle = 0; /* Number of round trips to the server */
790	int nFileSend = 0;
791	int nFileRecv; /* Number of files received */
792	int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
793	const char zCookie; / Server cookie */
	@@ -786,21 +818,21 @@
818	*/
819	if( cloneFlag ){
820	blob_appendf(&send, "clone\n");
821	pushFlag = 0;
822	pullFlag = 0;
823	nCard++;
824	/* TBD: Request all transferable configuration values */
825	}else if( pullFlag ){
826	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
827	nCard++;
828	}
829	if( pushFlag ){
830	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
831	nCard++;
832	}
833	printf(zLabel, "", "Bytes", "Cards", "Artifacts", "Deltas", "Dangling");
834
835	while( go ){
836	int newPhantom = 0;
837
838	/* Send make the most recently received cookie. Let the server
	@@ -809,54 +841,59 @@
841	zCookie = db_get("cookie", 0);
842	if( zCookie ){
843	blob_appendf(&send, "cookie %s\n", zCookie);
844	}
845
846	/* Generate gimme cards for phantoms and leaf cards
847	** for all leaves.
848	*/
849	if( pullFlag \|\| cloneFlag ){
850	request_phantoms(&xfer, mxPhantomReq);
851	}
852	if( pushFlag ){
853	send_unsent(&xfer);
854	nCard += send_unclustered(&xfer);
855	}
856
857	/* Exchange messages with the server */
858	nFileSend = xfer.nFileSent + xfer.nDeltaSent;
859	printf(zValue, "Send:",
860	blob_size(&send), nCard+xfer.nGimmeSent+xfer.nIGotSent,
861	xfer.nFileSent, xfer.nDeltaSent, 0);
862	#if 0
863	printf("Sent: %10d bytes, %5d cards, %5d files (%d+%d)\n",
864	blob_size(&send), nCard+xfer.nGimmeSent+xfer.nIGotSent,
865	nFileSend, xfer.nFileSent, xfer.nDeltaSent);
866	#endif
867	nCard = 0;
868	xfer.nFileSent = 0;
869	xfer.nDeltaSent = 0;
870	xfer.nGimmeSent = 0;
871	fflush(stdout);
872	http_exchange(&send, &recv);
873	blob_reset(&send);
874
875	/* Begin constructing the next message (which might never be
876	** sent) by beginning with the pull or push cards
877	*/
878	if( pullFlag ){
879	blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
880	nCard++;
881	}
882	if( pushFlag ){
883	blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
884	nCard++;
885	}
886
887	/* Process the reply that came back from the server */
888	while( blob_line(&recv, &xfer.line) ){
889	if( blob_buffer(&xfer.line)[0]=='#' ){
890	continue;
891	}
892	xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
893	nCard++;
894	printf("\r%d", nCard);
895	fflush(stdout);
896
897	/* file UUID SIZE \n CONTENT
898	** file UUID DELTASRC SIZE \n CONTENT
899	**
	@@ -920,11 +957,11 @@
957	if( zPCode==0 ){
958	zPCode = mprintf("%b", &xfer.aToken[2]);
959	db_set("project-code", zPCode, 0);
960	}
961	blob_appendf(&send, "clone\n");
962	nCard++;
963	}else
964
965	/* config NAME SIZE \n CONTENT
966	**
967	** Receive a configuration value from the server.
	@@ -978,15 +1015,19 @@
1015	fossil_fatal("%b", &xfer.err);
1016	}
1017	blobarray_reset(xfer.aToken, xfer.nToken);
1018	blob_reset(&xfer.line);
1019	}
1020	printf(zValue, "Received:",
1021	blob_size(&recv), nCard,
1022	xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
1023	#if 0
1024	printf("\rReceived: %10d bytes, %5d cards, %5d files (%d+%d+%d)\n",
1025	blob_size(&recv), nCard,
1026	xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile,
1027	xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
1028	#endif
1029	blob_reset(&recv);
1030	nCycle++;
1031	go = 0;
1032
1033	/* If we received one or more files on the previous exchange but
	@@ -996,11 +1037,11 @@
1037	if( (nFileRecv>0 \|\| newPhantom) && db_exists("SELECT 1 FROM phantom") ){
1038	go = 1;
1039	mxPhantomReq = nFileRecv*2;
1040	if( mxPhantomReq<200 ) mxPhantomReq = 200;
1041	}
1042	nCard = 0;
1043	xfer.nFileRcvd = 0;
1044	xfer.nDeltaRcvd = 0;
1045	xfer.nDanglingFile = 0;
1046
1047	/* If we have one or more files queued to send, then go
1048

Fossil SCM

Keyboard Shortcuts