Fossil SCM

Additional speed improvements for clone and rebuild.

drh 2008-03-08 19:42 trunk

Commit 043d63d4aa79d9558729452b3eba280e756f00fe

Parent 8010bb41e1c777a…

3 files changed +36 -19 +10 -2 +16 -12

~ src/content.c ~ src/vfile.c ~ src/xfer.c

M src/content.c

+36 -19

		--- src/content.c
		+++ src/content.c
		@@ -364,12 +364,11 @@
364	364	** content is already in the database, just return the record ID.
365	365	**
366	366	** If srcId is specified, then pBlob is delta content from
367	367	** the srcId record. srcId might be a phantom.
368	368	**
369		-** A phantom is written if pBlob==0. If pBlob==0 or if srcId is
370		-** specified then the UUID is set to zUuid. Otherwise zUuid is
	369	+** If srcId is specified then the UUID is set to zUuid. Otherwise zUuid is
371	370	** ignored. In the future this might change such that the content
372	371	** hash is checked against zUuid to make sure it is correct.
373	372	**
374	373	** If the record already exists but is a phantom, the pBlob content
375	374	** is inserted and the phatom becomes a real record.
		@@ -382,20 +381,17 @@
382	381	Blob hash;
383	382	int markAsUnclustered = 0;
384	383	int isDephantomize = 0;
385	384
386	385	assert( g.repositoryOpen );
387		- if( pBlob && srcId==0 ){
	386	+ assert( pBlob!=0 );
	387	+ if( srcId==0 ){
388	388	sha1sum_blob(pBlob, &hash);
389	389	}else{
390	390	blob_init(&hash, zUuid, -1);
391	391	}
392		- if( pBlob==0 ){
393		- size = -1;
394		- }else{
395		- size = blob_size(pBlob);
396		- }
	392	+ size = blob_size(pBlob);
397	393	db_begin_transaction();
398	394
399	395	/* Check to see if the entry already exists and if it does whether
400	396	** or not the entry is a phantom
401	397	*/
		@@ -415,27 +411,26 @@
415	411	markAsUnclustered = 1;
416	412	}
417	413	db_finalize(&s1);
418	414
419	415	/* Construct a received-from ID if we do not already have one */
420		- if( g.rcvid==0 && pBlob!=0 ){
	416	+ if( g.rcvid==0 ){
421	417	db_multi_exec(
422	418	"INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
423	419	"VALUES(%d, julianday('now'), %Q, %Q)",
424	420	g.userUid, g.zNonce, g.zIpAddr
425	421	);
426	422	g.rcvid = db_last_insert_rowid();
427	423	}
428	424
	425	+ blob_compress(pBlob, &cmpr);
429	426	if( rid>0 ){
430	427	/* We are just adding data to a phantom */
431		- assert( pBlob!=0 );
432	428	db_prepare(&s1,
433	429	"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
434	430	g.rcvid, size, rid
435	431	);
436		- blob_compress(pBlob, &cmpr);
437	432	db_bind_blob(&s1, ":data", &cmpr);
438	433	db_exec(&s1);
439	434	db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
440	435	if( srcId==0 \|\| content_is_available(srcId) ){
441	436	isDephantomize = 1;
		@@ -446,20 +441,18 @@
446	441	db_prepare(&s1,
447	442	"INSERT INTO blob(rcvid,size,uuid,content)"
448	443	"VALUES(%d,%d,'%b',:data)",
449	444	g.rcvid, size, &hash
450	445	);
451		- if( pBlob ){
452		- blob_compress(pBlob, &cmpr);
453		- db_bind_blob(&s1, ":data", &cmpr);
454		- }
	446	+ blob_compress(pBlob, &cmpr);
455	447	db_exec(&s1);
456	448	rid = db_last_insert_rowid();
457	449	if( !pBlob ){
458	450	db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
459	451	}
460	452	}
	453	+ blob_reset(&cmpr);
461	454
462	455	/* If the srcId is specified, then the data we just added is
463	456	** really a delta. Record this fact in the delta table.
464	457	*/
465	458	if( srcId ){
		@@ -485,16 +478,40 @@
485	478	db_end_transaction(0);
486	479	blob_reset(&hash);
487	480
488	481	/* Make arrangements to verify that the data can be recovered
489	482	** before we commit */
490		- if( pBlob ){
491		- blob_reset(&cmpr);
492		- verify_before_commit(rid);
493		- }
	483	+ verify_before_commit(rid);
	484	+ return rid;
	485	+}
	486	+
	487	+/*
	488	+** Create a new phantom with the given UUID and return its artifact ID.
	489	+*/
	490	+int content_new(const char *zUuid){
	491	+ int rid;
	492	+ static Stmt s1, s2;
	493	+
	494	+ assert( g.repositoryOpen );
	495	+ db_begin_transaction();
	496	+ db_static_prepare(&s1,
	497	+ "INSERT INTO blob(rcvid,size,uuid,content)"
	498	+ "VALUES(0,-1,:uuid,NULL)"
	499	+ );
	500	+ db_bind_text(&s1, ":uuid", zUuid);
	501	+ db_exec(&s1);
	502	+ rid = db_last_insert_rowid();
	503	+ db_static_prepare(&s2,
	504	+ "INSERT INTO phantom VALUES(:rid)"
	505	+ );
	506	+ db_bind_int(&s2, ":rid", rid);
	507	+ db_exec(&s2);
	508	+ bag_insert(&contentCache.missing, rid);
	509	+ db_end_transaction(0);
494	510	return rid;
495	511	}
	512	+
496	513
497	514	/*
498	515	** COMMAND: test-content-put
499	516	**
500	517	** Extract a blob from the database and write it into a file.
501	518

	--- src/content.c
	+++ src/content.c
	@@ -364,12 +364,11 @@
364	** content is already in the database, just return the record ID.
365	**
366	** If srcId is specified, then pBlob is delta content from
367	** the srcId record. srcId might be a phantom.
368	**
369	** A phantom is written if pBlob==0. If pBlob==0 or if srcId is
370	** specified then the UUID is set to zUuid. Otherwise zUuid is
371	** ignored. In the future this might change such that the content
372	** hash is checked against zUuid to make sure it is correct.
373	**
374	** If the record already exists but is a phantom, the pBlob content
375	** is inserted and the phatom becomes a real record.
	@@ -382,20 +381,17 @@
382	Blob hash;
383	int markAsUnclustered = 0;
384	int isDephantomize = 0;
385
386	assert( g.repositoryOpen );
387	if( pBlob && srcId==0 ){

388	sha1sum_blob(pBlob, &hash);
389	}else{
390	blob_init(&hash, zUuid, -1);
391	}
392	if( pBlob==0 ){
393	size = -1;
394	}else{
395	size = blob_size(pBlob);
396	}
397	db_begin_transaction();
398
399	/* Check to see if the entry already exists and if it does whether
400	** or not the entry is a phantom
401	*/
	@@ -415,27 +411,26 @@
415	markAsUnclustered = 1;
416	}
417	db_finalize(&s1);
418
419	/* Construct a received-from ID if we do not already have one */
420	if( g.rcvid==0 && pBlob!=0 ){
421	db_multi_exec(
422	"INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
423	"VALUES(%d, julianday('now'), %Q, %Q)",
424	g.userUid, g.zNonce, g.zIpAddr
425	);
426	g.rcvid = db_last_insert_rowid();
427	}
428

429	if( rid>0 ){
430	/* We are just adding data to a phantom */
431	assert( pBlob!=0 );
432	db_prepare(&s1,
433	"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
434	g.rcvid, size, rid
435	);
436	blob_compress(pBlob, &cmpr);
437	db_bind_blob(&s1, ":data", &cmpr);
438	db_exec(&s1);
439	db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
440	if( srcId==0 \|\| content_is_available(srcId) ){
441	isDephantomize = 1;
	@@ -446,20 +441,18 @@
446	db_prepare(&s1,
447	"INSERT INTO blob(rcvid,size,uuid,content)"
448	"VALUES(%d,%d,'%b',:data)",
449	g.rcvid, size, &hash
450	);
451	if( pBlob ){
452	blob_compress(pBlob, &cmpr);
453	db_bind_blob(&s1, ":data", &cmpr);
454	}
455	db_exec(&s1);
456	rid = db_last_insert_rowid();
457	if( !pBlob ){
458	db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
459	}
460	}

461
462	/* If the srcId is specified, then the data we just added is
463	** really a delta. Record this fact in the delta table.
464	*/
465	if( srcId ){
	@@ -485,16 +478,40 @@
485	db_end_transaction(0);
486	blob_reset(&hash);
487
488	/* Make arrangements to verify that the data can be recovered
489	** before we commit */
490	if( pBlob ){
491	blob_reset(&cmpr);
492	verify_before_commit(rid);
493	}























494	return rid;
495	}

496
497	/*
498	** COMMAND: test-content-put
499	**
500	** Extract a blob from the database and write it into a file.
501

	--- src/content.c
	+++ src/content.c
	@@ -364,12 +364,11 @@
364	** content is already in the database, just return the record ID.
365	**
366	** If srcId is specified, then pBlob is delta content from
367	** the srcId record. srcId might be a phantom.
368	**
369	** If srcId is specified then the UUID is set to zUuid. Otherwise zUuid is

370	** ignored. In the future this might change such that the content
371	** hash is checked against zUuid to make sure it is correct.
372	**
373	** If the record already exists but is a phantom, the pBlob content
374	** is inserted and the phatom becomes a real record.
	@@ -382,20 +381,17 @@
381	Blob hash;
382	int markAsUnclustered = 0;
383	int isDephantomize = 0;
384
385	assert( g.repositoryOpen );
386	assert( pBlob!=0 );
387	if( srcId==0 ){
388	sha1sum_blob(pBlob, &hash);
389	}else{
390	blob_init(&hash, zUuid, -1);
391	}
392	size = blob_size(pBlob);




393	db_begin_transaction();
394
395	/* Check to see if the entry already exists and if it does whether
396	** or not the entry is a phantom
397	*/
	@@ -415,27 +411,26 @@
411	markAsUnclustered = 1;
412	}
413	db_finalize(&s1);
414
415	/* Construct a received-from ID if we do not already have one */
416	if( g.rcvid==0 ){
417	db_multi_exec(
418	"INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)"
419	"VALUES(%d, julianday('now'), %Q, %Q)",
420	g.userUid, g.zNonce, g.zIpAddr
421	);
422	g.rcvid = db_last_insert_rowid();
423	}
424
425	blob_compress(pBlob, &cmpr);
426	if( rid>0 ){
427	/* We are just adding data to a phantom */

428	db_prepare(&s1,
429	"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
430	g.rcvid, size, rid
431	);

432	db_bind_blob(&s1, ":data", &cmpr);
433	db_exec(&s1);
434	db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
435	if( srcId==0 \|\| content_is_available(srcId) ){
436	isDephantomize = 1;
	@@ -446,20 +441,18 @@
441	db_prepare(&s1,
442	"INSERT INTO blob(rcvid,size,uuid,content)"
443	"VALUES(%d,%d,'%b',:data)",
444	g.rcvid, size, &hash
445	);
446	blob_compress(pBlob, &cmpr);



447	db_exec(&s1);
448	rid = db_last_insert_rowid();
449	if( !pBlob ){
450	db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
451	}
452	}
453	blob_reset(&cmpr);
454
455	/* If the srcId is specified, then the data we just added is
456	** really a delta. Record this fact in the delta table.
457	*/
458	if( srcId ){
	@@ -485,16 +478,40 @@
478	db_end_transaction(0);
479	blob_reset(&hash);
480
481	/* Make arrangements to verify that the data can be recovered
482	** before we commit */
483	verify_before_commit(rid);
484	return rid;
485	}
486
487	/*
488	** Create a new phantom with the given UUID and return its artifact ID.
489	*/
490	int content_new(const char *zUuid){
491	int rid;
492	static Stmt s1, s2;
493
494	assert( g.repositoryOpen );
495	db_begin_transaction();
496	db_static_prepare(&s1,
497	"INSERT INTO blob(rcvid,size,uuid,content)"
498	"VALUES(0,-1,:uuid,NULL)"
499	);
500	db_bind_text(&s1, ":uuid", zUuid);
501	db_exec(&s1);
502	rid = db_last_insert_rowid();
503	db_static_prepare(&s2,
504	"INSERT INTO phantom VALUES(:rid)"
505	);
506	db_bind_int(&s2, ":rid", rid);
507	db_exec(&s2);
508	bag_insert(&contentCache.missing, rid);
509	db_end_transaction(0);
510	return rid;
511	}
512
513
514	/*
515	** COMMAND: test-content-put
516	**
517	** Extract a blob from the database and write it into a file.
518

M src/vfile.c

+10 -2

		--- src/vfile.c
		+++ src/vfile.c
		@@ -39,21 +39,29 @@
39	39	** If the UUID is not found and phantomize is 1, then attempt to
40	40	** create a phantom record.
41	41	*/
42	42	int uuid_to_rid(const char *zUuid, int phantomize){
43	43	int rid, sz;
	44	+ static Stmt q;
44	45	char z[UUID_SIZE+1];
45	46
46	47	sz = strlen(zUuid);
47	48	if( sz!=UUID_SIZE \|\| !validate16(zUuid, sz) ){
48	49	return 0;
49	50	}
50	51	strcpy(z, zUuid);
51	52	canonical16(z, sz);
52		- rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", z);
	53	+ db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
	54	+ db_bind_text(&q, ":uuid", z);
	55	+ if( db_step(&q)==SQLITE_ROW ){
	56	+ rid = db_column_int(&q, 0);
	57	+ }else{
	58	+ rid = 0;
	59	+ }
	60	+ db_reset(&q);
53	61	if( rid==0 && phantomize ){
54		- rid = content_put(0, zUuid, 0);
	62	+ rid = content_new(zUuid);
55	63	}
56	64	return rid;
57	65	}
58	66
59	67	/*
60	68

	--- src/vfile.c
	+++ src/vfile.c
	@@ -39,21 +39,29 @@
39	** If the UUID is not found and phantomize is 1, then attempt to
40	** create a phantom record.
41	*/
42	int uuid_to_rid(const char *zUuid, int phantomize){
43	int rid, sz;

44	char z[UUID_SIZE+1];
45
46	sz = strlen(zUuid);
47	if( sz!=UUID_SIZE \|\| !validate16(zUuid, sz) ){
48	return 0;
49	}
50	strcpy(z, zUuid);
51	canonical16(z, sz);
52	rid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", z);







53	if( rid==0 && phantomize ){
54	rid = content_put(0, zUuid, 0);
55	}
56	return rid;
57	}
58
59	/*
60

	--- src/vfile.c
	+++ src/vfile.c
	@@ -39,21 +39,29 @@
39	** If the UUID is not found and phantomize is 1, then attempt to
40	** create a phantom record.
41	*/
42	int uuid_to_rid(const char *zUuid, int phantomize){
43	int rid, sz;
44	static Stmt q;
45	char z[UUID_SIZE+1];
46
47	sz = strlen(zUuid);
48	if( sz!=UUID_SIZE \|\| !validate16(zUuid, sz) ){
49	return 0;
50	}
51	strcpy(z, zUuid);
52	canonical16(z, sz);
53	db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
54	db_bind_text(&q, ":uuid", z);
55	if( db_step(&q)==SQLITE_ROW ){
56	rid = db_column_int(&q, 0);
57	}else{
58	rid = 0;
59	}
60	db_reset(&q);
61	if( rid==0 && phantomize ){
62	rid = content_new(zUuid);
63	}
64	return rid;
65	}
66
67	/*
68

M src/xfer.c

+16 -12

		--- src/xfer.c
		+++ src/xfer.c
		@@ -56,13 +56,23 @@
56	56	**
57	57	** Compare to uuid_to_rid(). This routine takes a blob argument
58	58	** and does less error checking.
59	59	*/
60	60	static int rid_from_uuid(Blob *pUuid, int phantomize){
61		- int rid = db_int(0, "SELECT rid FROM blob WHERE uuid='%b'", pUuid);
	61	+ static Stmt q;
	62	+ int rid;
	63	+
	64	+ db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
	65	+ db_bind_str(&q, ":uuid", pUuid);
	66	+ if( db_step(&q)==SQLITE_ROW ){
	67	+ rid = db_column_int(&q, 0);
	68	+ }else{
	69	+ rid = 0;
	70	+ }
	71	+ db_reset(&q);
62	72	if( rid==0 && phantomize ){
63		- rid = content_put(0, blob_str(pUuid), 0);
	73	+ rid = content_new(blob_str(pUuid));
64	74	}
65	75	return rid;
66	76	}
67	77
68	78	/*
		@@ -840,20 +850,14 @@
840	850	*/
841	851	if( xfer.nToken==2
842	852	&& blob_eq(&xfer.aToken[0], "igot")
843	853	&& blob_is_uuid(&xfer.aToken[1])
844	854	){
845		- int rid = 0;
846		- if( pullFlag \|\| cloneFlag ){
847		- if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
848		- &xfer.aToken[1]) ){
849		- rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
850		- newPhantom = 1;
851		- }
852		- }
853		- if( rid==0 ){
854		- rid = rid_from_uuid(&xfer.aToken[1], 0);
	855	+ int rid = rid_from_uuid(&xfer.aToken[1], 0);
	856	+ if( rid==0 && (pullFlag \|\| cloneFlag) ){
	857	+ rid = content_new(blob_str(&xfer.aToken[1]));
	858	+ newPhantom = 1;
855	859	}
856	860	remote_has(rid);
857	861	}else
858	862
859	863
860	864

	--- src/xfer.c
	+++ src/xfer.c
	@@ -56,13 +56,23 @@
56	**
57	** Compare to uuid_to_rid(). This routine takes a blob argument
58	** and does less error checking.
59	*/
60	static int rid_from_uuid(Blob *pUuid, int phantomize){
61	int rid = db_int(0, "SELECT rid FROM blob WHERE uuid='%b'", pUuid);










62	if( rid==0 && phantomize ){
63	rid = content_put(0, blob_str(pUuid), 0);
64	}
65	return rid;
66	}
67
68	/*
	@@ -840,20 +850,14 @@
840	*/
841	if( xfer.nToken==2
842	&& blob_eq(&xfer.aToken[0], "igot")
843	&& blob_is_uuid(&xfer.aToken[1])
844	){
845	int rid = 0;
846	if( pullFlag \|\| cloneFlag ){
847	if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
848	&xfer.aToken[1]) ){
849	rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
850	newPhantom = 1;
851	}
852	}
853	if( rid==0 ){
854	rid = rid_from_uuid(&xfer.aToken[1], 0);
855	}
856	remote_has(rid);
857	}else
858
859
860

	--- src/xfer.c
	+++ src/xfer.c
	@@ -56,13 +56,23 @@
56	**
57	** Compare to uuid_to_rid(). This routine takes a blob argument
58	** and does less error checking.
59	*/
60	static int rid_from_uuid(Blob *pUuid, int phantomize){
61	static Stmt q;
62	int rid;
63
64	db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
65	db_bind_str(&q, ":uuid", pUuid);
66	if( db_step(&q)==SQLITE_ROW ){
67	rid = db_column_int(&q, 0);
68	}else{
69	rid = 0;
70	}
71	db_reset(&q);
72	if( rid==0 && phantomize ){
73	rid = content_new(blob_str(pUuid));
74	}
75	return rid;
76	}
77
78	/*
	@@ -840,20 +850,14 @@
850	*/
851	if( xfer.nToken==2
852	&& blob_eq(&xfer.aToken[0], "igot")
853	&& blob_is_uuid(&xfer.aToken[1])
854	){
855	int rid = rid_from_uuid(&xfer.aToken[1], 0);
856	if( rid==0 && (pullFlag \|\| cloneFlag) ){
857	rid = content_new(blob_str(&xfer.aToken[1]));
858	newPhantom = 1;






859	}
860	remote_has(rid);
861	}else
862
863
864

Fossil SCM

Keyboard Shortcuts