Fossil SCM

The delta compress on xfer is working better now, but still needs work.

drh 2007-08-09 17:42 trunk
Commit 73bddaebb9ec5d6eea26b46f56d2b82b68420f01
Parent 6eca3132fe0bf28…
5 files changed +4 +1 -1 +10 -5 +6 -2 +107 -71
~ src/content.c ~ src/delta.c ~ src/rebuild.c ~ src/schema.c ~ src/xfer.c
M src/content.c
+4
--- src/content.c
+++ src/content.c
@@ -196,10 +196,11 @@
196196
 g.rcvid, size, rid
197197
 );
198198
 blob_compress(pBlob, &cmpr);
199199
 db_bind_blob(&s1, ":data", &cmpr);
200200
 db_exec(&s1);
201
+ db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
201202
 }else{
202203
 /* We are creating a new entry */
203204
 db_prepare(&s1,
204205
 "INSERT INTO blob(rcvid,size,uuid,content)"
205206
 "VALUES(%d,%d,'%s',:data)",
@@ -209,10 +210,13 @@
209210
 blob_compress(pBlob, &cmpr);
210211
 db_bind_blob(&s1, ":data", &cmpr);
211212
 }
212213
 db_exec(&s1);
213214
 rid = db_last_insert_rowid();
215
+ if( !pBlob ){
216
+ db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
217
+ }
214218
 }
215219
216220
217221
 /* Finish the transaction and cleanup */
218222
 db_finalize(&s1);
219223
--- src/content.c
+++ src/content.c
@@ -196,10 +196,11 @@
196 g.rcvid, size, rid
197 );
198 blob_compress(pBlob, &cmpr);
199 db_bind_blob(&s1, ":data", &cmpr);
200 db_exec(&s1);
 
201 }else{
202 /* We are creating a new entry */
203 db_prepare(&s1,
204 "INSERT INTO blob(rcvid,size,uuid,content)"
205 "VALUES(%d,%d,'%s',:data)",
@@ -209,10 +210,13 @@
209 blob_compress(pBlob, &cmpr);
210 db_bind_blob(&s1, ":data", &cmpr);
211 }
212 db_exec(&s1);
213 rid = db_last_insert_rowid();
 
 
 
214 }
215
216
217 /* Finish the transaction and cleanup */
218 db_finalize(&s1);
219
--- src/content.c
+++ src/content.c
@@ -196,10 +196,11 @@
196 g.rcvid, size, rid
197 );
198 blob_compress(pBlob, &cmpr);
199 db_bind_blob(&s1, ":data", &cmpr);
200 db_exec(&s1);
201 db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
202 }else{
203 /* We are creating a new entry */
204 db_prepare(&s1,
205 "INSERT INTO blob(rcvid,size,uuid,content)"
206 "VALUES(%d,%d,'%s',:data)",
@@ -209,10 +210,13 @@
210 blob_compress(pBlob, &cmpr);
211 db_bind_blob(&s1, ":data", &cmpr);
212 }
213 db_exec(&s1);
214 rid = db_last_insert_rowid();
215 if( !pBlob ){
216 db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
217 }
218 }
219
220
221 /* Finish the transaction and cleanup */
222 db_finalize(&s1);
223
M src/delta.c
+1 -1
--- src/delta.c
+++ src/delta.c
@@ -333,11 +333,11 @@
333333
334334
 /* Begin scanning the target file and generating copy commands and
335335
 ** literal sections of the delta.
336336
 */
337337
 base = 0; /* We have already generated everything before zOut[base] */
338
- while( base<lenOut-NHASH ){
338
+ while( base+NHASH<lenOut ){
339339
 int iSrc, iBlock;
340340
 unsigned int bestCnt, bestOfst, bestLitsz;
341341
 hash_init(&h, &zOut[base]);
342342
 i = 0; /* Trying to match a landmark against zOut[base+i] */
343343
 bestCnt = 0;
344344
--- src/delta.c
+++ src/delta.c
@@ -333,11 +333,11 @@
333
334 /* Begin scanning the target file and generating copy commands and
335 ** literal sections of the delta.
336 */
337 base = 0; /* We have already generated everything before zOut[base] */
338 while( base<lenOut-NHASH ){
339 int iSrc, iBlock;
340 unsigned int bestCnt, bestOfst, bestLitsz;
341 hash_init(&h, &zOut[base]);
342 i = 0; /* Trying to match a landmark against zOut[base+i] */
343 bestCnt = 0;
344
--- src/delta.c
+++ src/delta.c
@@ -333,11 +333,11 @@
333
334 /* Begin scanning the target file and generating copy commands and
335 ** literal sections of the delta.
336 */
337 base = 0; /* We have already generated everything before zOut[base] */
338 while( base+NHASH<lenOut ){
339 int iSrc, iBlock;
340 unsigned int bestCnt, bestOfst, bestLitsz;
341 hash_init(&h, &zOut[base]);
342 i = 0; /* Trying to match a landmark against zOut[base+i] */
343 bestCnt = 0;
344
M src/rebuild.c
+10 -5
--- src/rebuild.c
+++ src/rebuild.c
@@ -60,17 +60,22 @@
6060
 db_multi_exec("DROP TABLE %Q", zTable);
6161
 free(zTable);
6262
 }
6363
 db_multi_exec(zRepositorySchema2);
6464
65
- db_prepare(&s, "SELECT rid FROM blob");
65
+ db_prepare(&s, "SELECT rid, size FROM blob");
6666
 while( db_step(&s)==SQLITE_ROW ){
6767
 int rid = db_column_int(&s, 0);
68
- Blob content;
69
- content_get(rid, &content);
70
- manifest_crosslink(rid, &content);
71
- blob_reset(&content);
68
+ int size = db_column_int(&s, 1);
69
+ if( size>=0 ){
70
+ Blob content;
71
+ content_get(rid, &content);
72
+ manifest_crosslink(rid, &content);
73
+ blob_reset(&content);
74
+ }else{
75
+ db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
76
+ }
7277
 }
7378
7479
 if( errCnt && !forceFlag ){
7580
 printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
7681
 errCnt);
7782
--- src/rebuild.c
+++ src/rebuild.c
@@ -60,17 +60,22 @@
60 db_multi_exec("DROP TABLE %Q", zTable);
61 free(zTable);
62 }
63 db_multi_exec(zRepositorySchema2);
64
65 db_prepare(&s, "SELECT rid FROM blob");
66 while( db_step(&s)==SQLITE_ROW ){
67 int rid = db_column_int(&s, 0);
68 Blob content;
69 content_get(rid, &content);
70 manifest_crosslink(rid, &content);
71 blob_reset(&content);
 
 
 
 
 
72 }
73
74 if( errCnt && !forceFlag ){
75 printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
76 errCnt);
77
--- src/rebuild.c
+++ src/rebuild.c
@@ -60,17 +60,22 @@
60 db_multi_exec("DROP TABLE %Q", zTable);
61 free(zTable);
62 }
63 db_multi_exec(zRepositorySchema2);
64
65 db_prepare(&s, "SELECT rid, size FROM blob");
66 while( db_step(&s)==SQLITE_ROW ){
67 int rid = db_column_int(&s, 0);
68 int size = db_column_int(&s, 1);
69 if( size>=0 ){
70 Blob content;
71 content_get(rid, &content);
72 manifest_crosslink(rid, &content);
73 blob_reset(&content);
74 }else{
75 db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
76 }
77 }
78
79 if( errCnt && !forceFlag ){
80 printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
81 errCnt);
82
M src/schema.c
+6 -2
--- src/schema.c
+++ src/schema.c
@@ -135,10 +135,12 @@
135135
 @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest
136136
 @ fnid INTEGER REFERENCES filename -- Name of the file
137137
 @ );
138138
 @ CREATE INDEX mlink_i1 ON mlink(mid);
139139
 @ CREATE INDEX mlink_i2 ON mlink(fnid);
140
+@ CREATE INDEX mlink_i3 ON mlink(fid);
141
+@ CREATE INDEX mlink_i4 ON mlink(pid);
140142
 @
141143
 @ -- Parent/child linkages
142144
 @ --
143145
 @ CREATE TABLE plink(
144146
 @ pid INTEGER REFERENCES blob, -- Parent manifest
@@ -160,13 +162,15 @@
160162
 @ comment TEXT
161163
 @ );
162164
 @ CREATE INDEX event_i1 ON event(mtime);
163165
 @ CREATE INDEX event_i2 ON event(objid);
164166
 @
165
-@ -- Make sure reading DELTA by SRCID is efficient
167
+@ -- A record of phantoms
166168
 @ --
167
-@ CREATE INDEX IF NOT EXISTS delta_srcid ON delta(srcid, rid);
169
+@ CREATE TABLE phantom(
170
+@ rid INTEGER PRIMARY KEY -- Record ID of the phantom
171
+@ );
168172
 @
169173
 @ -- Aggregated ticket information
170174
 @ --
171175
 @ CREATE TABLE tkt(
172176
 @ tktid INTEGER PRIMARY KEY, -- Internal ticket ID
173177
--- src/schema.c
+++ src/schema.c
@@ -135,10 +135,12 @@
135 @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest
136 @ fnid INTEGER REFERENCES filename -- Name of the file
137 @ );
138 @ CREATE INDEX mlink_i1 ON mlink(mid);
139 @ CREATE INDEX mlink_i2 ON mlink(fnid);
 
 
140 @
141 @ -- Parent/child linkages
142 @ --
143 @ CREATE TABLE plink(
144 @ pid INTEGER REFERENCES blob, -- Parent manifest
@@ -160,13 +162,15 @@
160 @ comment TEXT
161 @ );
162 @ CREATE INDEX event_i1 ON event(mtime);
163 @ CREATE INDEX event_i2 ON event(objid);
164 @
165 @ -- Make sure reading DELTA by SRCID is efficient
166 @ --
167 @ CREATE INDEX IF NOT EXISTS delta_srcid ON delta(srcid, rid);
 
 
168 @
169 @ -- Aggregated ticket information
170 @ --
171 @ CREATE TABLE tkt(
172 @ tktid INTEGER PRIMARY KEY, -- Internal ticket ID
173
--- src/schema.c
+++ src/schema.c
@@ -135,10 +135,12 @@
135 @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest
136 @ fnid INTEGER REFERENCES filename -- Name of the file
137 @ );
138 @ CREATE INDEX mlink_i1 ON mlink(mid);
139 @ CREATE INDEX mlink_i2 ON mlink(fnid);
140 @ CREATE INDEX mlink_i3 ON mlink(fid);
141 @ CREATE INDEX mlink_i4 ON mlink(pid);
142 @
143 @ -- Parent/child linkages
144 @ --
145 @ CREATE TABLE plink(
146 @ pid INTEGER REFERENCES blob, -- Parent manifest
@@ -160,13 +162,15 @@
162 @ comment TEXT
163 @ );
164 @ CREATE INDEX event_i1 ON event(mtime);
165 @ CREATE INDEX event_i2 ON event(objid);
166 @
167 @ -- A record of phantoms
168 @ --
169 @ CREATE TABLE phantom(
170 @ rid INTEGER PRIMARY KEY -- Record ID of the phantom
171 @ );
172 @
173 @ -- Aggregated ticket information
174 @ --
175 @ CREATE TABLE tkt(
176 @ tktid INTEGER PRIMARY KEY, -- Internal ticket ID
177
M src/xfer.c
+107 -71
--- src/xfer.c
+++ src/xfer.c
@@ -34,51 +34,79 @@
3434
 ** Return the integer record ID of the similar record. Or return
3535
 ** 0 if none is found.
3636
 */
3737
 static int similar_record(int rid, int traceFlag){
3838
 int inCnt, outCnt;
39
+ int i;
3940
 Stmt q;
4041
 int queue[100];
41
-
42
-return 0;
43
-
44
- db_prepare(&q,
42
+ static const char *azQuery[] = {
43
+ /* Scan the delta table first */
4544
 "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
4645
 " FROM delta"
4746
 " WHERE rid=:x"
47
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=srcid)"
4848
 " UNION ALL "
4949
 "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
5050
 " FROM delta"
5151
 " WHERE srcid=:x"
52
- );
53
- queue[0] = rid;
54
- inCnt = 1;
55
- outCnt = 0;
56
- while( outCnt<inCnt ){
57
- int xid = queue[outCnt%64];
58
- outCnt++;
59
- db_bind_int(&q, ":x", xid);
60
- if( traceFlag ) printf("xid=%d\n", xid);
61
- while( db_step(&q)==SQLITE_ROW ){
62
- int nid = db_column_int(&q, 0);
63
- int hit = db_column_int(&q, 1);
64
- if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
65
- if( hit ){
66
- db_finalize(&q);
67
- return nid;
68
- }
69
- if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
70
- int i;
71
- for(i=0; i<inCnt && queue[i]!=nid; i++){}
72
- if( i>=inCnt ){
73
- queue[inCnt++] = nid;
74
- }
75
- }
76
- }
77
- db_reset(&q);
78
- }
79
- db_finalize(&q);
52
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=delta.rid)",
53
+
54
+ /* Then the plink table */
55
+ "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
56
+ " FROM plink"
57
+ " WHERE cid=:x"
58
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
59
+ " UNION ALL "
60
+ "SELECT cid, EXISTS(SELECT 1 FROM onremote WHERE rid=cid)"
61
+ " FROM plink"
62
+ " WHERE pid=:x"
63
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=cid)",
64
+
65
+ /* Finally the mlink table */
66
+ "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
67
+ " FROM mlink"
68
+ " WHERE fid=:x AND pid>0"
69
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
70
+ " UNION ALL "
71
+ "SELECT fid, EXISTS(SELECT 1 FROM onremote WHERE rid=fid)"
72
+ " FROM mlink"
73
+ " WHERE pid=:x AND fid>0"
74
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=fid)",
75
+ };
76
+
77
+ for(i=0; i<sizeof(azQuery)/sizeof(azQuery[0]); i++){
78
+ db_prepare(&q, azQuery[i]);
79
+ queue[0] = rid;
80
+ inCnt = 1;
81
+ outCnt = 0;
82
+ if( traceFlag ) printf("PASS %d\n", i+1);
83
+ while( outCnt<inCnt ){
84
+ int xid = queue[outCnt%64];
85
+ outCnt++;
86
+ db_bind_int(&q, ":x", xid);
87
+ if( traceFlag ) printf("xid=%d\n", xid);
88
+ while( db_step(&q)==SQLITE_ROW ){
89
+ int nid = db_column_int(&q, 0);
90
+ int hit = db_column_int(&q, 1);
91
+ if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
92
+ if( hit ){
93
+ db_finalize(&q);
94
+ return nid;
95
+ }
96
+ if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
97
+ int i;
98
+ for(i=0; i<inCnt && queue[i]!=nid; i++){}
99
+ if( i>=inCnt ){
100
+ queue[inCnt++] = nid;
101
+ }
102
+ }
103
+ }
104
+ db_reset(&q);
105
+ }
106
+ db_finalize(&q);
107
+ }
80108
 return 0;
81109
 }
82110
83111
 /*
84112
 ** COMMAND: test-similar-record
@@ -172,43 +200,37 @@
172200
 if( blob_size(&uuid)==0 ){
173201
 return 0;
174202
 }
175203
 content_get(rid, &content);
176204
177
- srcid = similar_record(rid, 0);
178
- if( srcid ){
179
- Blob src, delta;
180
- Blob srcuuid;
181
- content_get(srcid, &src);
182
- blob_delta_create(&src, &content, &delta);
183
- blob_reset(&src);
184
- blob_reset(&content);
185
- blob_zero(&srcuuid);
186
- db_blob(&srcuuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
187
- size = blob_size(&delta);
188
- if( pOut ){
189
- blob_appendf(pOut, "file %b %b %d\n", &uuid, &srcuuid, size);
190
- blob_append(pOut, blob_buffer(&delta), size);
191
- }else{
192
- cgi_printf("file %b %b %d\n", &uuid, &srcuuid, size);
193
- cgi_append_content(blob_buffer(&delta), size);
194
- }
195
- blob_reset(&delta);
196
- blob_reset(&srcuuid);
197
- blob_reset(&uuid);
198
- }else{
199
- size = blob_size(&content);
200
- if( pOut ){
201
- blob_appendf(pOut, "file %b %d\n", &uuid, size);
202
- blob_append(pOut, blob_buffer(&content), size);
203
- }else{
204
- cgi_printf("file %b %d\n", &uuid, size);
205
- cgi_append_content(blob_buffer(&content), size);
206
- }
207
- blob_reset(&content);
208
- blob_reset(&uuid);
209
- }
205
+ if( blob_size(&content)>100 ){
206
+ srcid = similar_record(rid, 0);
207
+ if( srcid ){
208
+ Blob src;
209
+ content_get(srcid, &src);
210
+ if( blob_size(&src)>100 ){
211
+ Blob delta;
212
+ blob_delta_create(&src, &content, &delta);
213
+ blob_reset(&content);
214
+ content = delta;
215
+ blob_append(&uuid, " ", 1);
216
+ blob_append(&content, "\n", 1);
217
+ db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
218
+ }
219
+ blob_reset(&src);
220
+ }
221
+ }
222
+ size = blob_size(&content);
223
+ if( pOut ){
224
+ blob_appendf(pOut, "file %b %d\n", &uuid, size);
225
+ blob_append(pOut, blob_buffer(&content), size);
226
+ }else{
227
+ cgi_printf("file %b %d\n", &uuid, size);
228
+ cgi_append_content(blob_buffer(&content), size);
229
+ }
230
+ blob_reset(&content);
231
+ blob_reset(&uuid);
210232
 db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
211233
 return size;
212234
 }
213235
214236
@@ -467,11 +489,11 @@
467489
 }
468490
 isPull = 1;
469491
 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
470492
 db_multi_exec(
471493
 "INSERT OR IGNORE INTO pending(rid) "
472
- "SELECT rid FROM blob WHERE size>=0"
494
+ "SELECT mid FROM mlink JOIN blob ON mid=rid"
473495
 );
474496
 }else
475497
476498
 /* login USER NONCE SIGNATURE
477499
 **
@@ -495,14 +517,22 @@
495517
 }
496518
497519
 /* The input message has now been processed. Generate a reply. */
498520
 if( isPush ){
499521
 Stmt q;
500
- db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
501
- while( db_step(&q)==SQLITE_ROW ){
522
+ int nReq = 0;
523
+ db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
524
+ while( db_step(&q)==SQLITE_ROW && nReq++ < 200 ){
502525
 const char *zUuid = db_column_text(&q, 0);
526
+ int rid = db_column_int(&q, 1);
527
+ int xid = similar_record(rid, 0);
503528
 @ gimme %s(zUuid)
529
+ if( xid ){
530
+ char *zXUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid);
531
+ @ igot %s(zXUuid);
532
+ free(zXUuid);
533
+ }
504534
 }
505535
 db_finalize(&q);
506536
 }
507537
 if( isPull ){
508538
 send_all_pending(0);
@@ -622,15 +652,21 @@
622652
623653
 if( pullFlag ){
624654
 /* Send gimme message for every phantom that we hold.
625655
 */
626656
 Stmt q;
627
- db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
628
- while( db_step(&q)==SQLITE_ROW ){
657
+ db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
658
+ while( db_step(&q)==SQLITE_ROW && nReq<200 ){
629659
 const char *zUuid = db_column_text(&q, 0);
660
+ int rid = db_column_int(&q, 1);
661
+ int xid = similar_record(rid, 0);
630662
 blob_appendf(&send,"gimme %s\n", zUuid);
631663
 nReq++;
664
+ if( xid ){
665
+ blob_appendf(&send, "igot %z\n",
666
+ db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid));
667
+ }
632668
 }
633669
 db_finalize(&q);
634670
 }
635671
636672
 if( pushFlag ){
@@ -716,11 +752,11 @@
716752
 if( db_changes()>0 ){
717753
 go = 1;
718754
 }
719755
 }
720756
 if( pullFlag && !go &&
721
- db_exists("SELECT 1 FROM blob WHERE rid=%d AND size<0", rid) ){
757
+ db_exists("SELECT 1 FROM phantom WHERE rid=%d", rid) ){
722758
 go = 1;
723759
 }
724760
 }else if( pullFlag ){
725761
 go = 1;
726762
 content_put(0, blob_str(&aToken[1]));
727763
--- src/xfer.c
+++ src/xfer.c
@@ -34,51 +34,79 @@
34 ** Return the integer record ID of the similar record. Or return
35 ** 0 if none is found.
36 */
37 static int similar_record(int rid, int traceFlag){
38 int inCnt, outCnt;
 
39 Stmt q;
40 int queue[100];
41
42 return 0;
43
44 db_prepare(&q,
45 "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
46 " FROM delta"
47 " WHERE rid=:x"
 
48 " UNION ALL "
49 "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
50 " FROM delta"
51 " WHERE srcid=:x"
52 );
53 queue[0] = rid;
54 inCnt = 1;
55 outCnt = 0;
56 while( outCnt<inCnt ){
57 int xid = queue[outCnt%64];
58 outCnt++;
59 db_bind_int(&q, ":x", xid);
60 if( traceFlag ) printf("xid=%d\n", xid);
61 while( db_step(&q)==SQLITE_ROW ){
62 int nid = db_column_int(&q, 0);
63 int hit = db_column_int(&q, 1);
64 if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
65 if( hit ){
66 db_finalize(&q);
67 return nid;
68 }
69 if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
70 int i;
71 for(i=0; i<inCnt && queue[i]!=nid; i++){}
72 if( i>=inCnt ){
73 queue[inCnt++] = nid;
74 }
75 }
76 }
77 db_reset(&q);
78 }
79 db_finalize(&q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80 return 0;
81 }
82
83 /*
84 ** COMMAND: test-similar-record
@@ -172,43 +200,37 @@
172 if( blob_size(&uuid)==0 ){
173 return 0;
174 }
175 content_get(rid, &content);
176
177 srcid = similar_record(rid, 0);
178 if( srcid ){
179 Blob src, delta;
180 Blob srcuuid;
181 content_get(srcid, &src);
182 blob_delta_create(&src, &content, &delta);
183 blob_reset(&src);
184 blob_reset(&content);
185 blob_zero(&srcuuid);
186 db_blob(&srcuuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
187 size = blob_size(&delta);
188 if( pOut ){
189 blob_appendf(pOut, "file %b %b %d\n", &uuid, &srcuuid, size);
190 blob_append(pOut, blob_buffer(&delta), size);
191 }else{
192 cgi_printf("file %b %b %d\n", &uuid, &srcuuid, size);
193 cgi_append_content(blob_buffer(&delta), size);
194 }
195 blob_reset(&delta);
196 blob_reset(&srcuuid);
197 blob_reset(&uuid);
198 }else{
199 size = blob_size(&content);
200 if( pOut ){
201 blob_appendf(pOut, "file %b %d\n", &uuid, size);
202 blob_append(pOut, blob_buffer(&content), size);
203 }else{
204 cgi_printf("file %b %d\n", &uuid, size);
205 cgi_append_content(blob_buffer(&content), size);
206 }
207 blob_reset(&content);
208 blob_reset(&uuid);
209 }
210 db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
211 return size;
212 }
213
214
@@ -467,11 +489,11 @@
467 }
468 isPull = 1;
469 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
470 db_multi_exec(
471 "INSERT OR IGNORE INTO pending(rid) "
472 "SELECT rid FROM blob WHERE size>=0"
473 );
474 }else
475
476 /* login USER NONCE SIGNATURE
477 **
@@ -495,14 +517,22 @@
495 }
496
497 /* The input message has now been processed. Generate a reply. */
498 if( isPush ){
499 Stmt q;
500 db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
501 while( db_step(&q)==SQLITE_ROW ){
 
502 const char *zUuid = db_column_text(&q, 0);
 
 
503 @ gimme %s(zUuid)
 
 
 
 
 
504 }
505 db_finalize(&q);
506 }
507 if( isPull ){
508 send_all_pending(0);
@@ -622,15 +652,21 @@
622
623 if( pullFlag ){
624 /* Send gimme message for every phantom that we hold.
625 */
626 Stmt q;
627 db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
628 while( db_step(&q)==SQLITE_ROW ){
629 const char *zUuid = db_column_text(&q, 0);
 
 
630 blob_appendf(&send,"gimme %s\n", zUuid);
631 nReq++;
 
 
 
 
632 }
633 db_finalize(&q);
634 }
635
636 if( pushFlag ){
@@ -716,11 +752,11 @@
716 if( db_changes()>0 ){
717 go = 1;
718 }
719 }
720 if( pullFlag && !go &&
721 db_exists("SELECT 1 FROM blob WHERE rid=%d AND size<0", rid) ){
722 go = 1;
723 }
724 }else if( pullFlag ){
725 go = 1;
726 content_put(0, blob_str(&aToken[1]));
727
--- src/xfer.c
+++ src/xfer.c
@@ -34,51 +34,79 @@
34 ** Return the integer record ID of the similar record. Or return
35 ** 0 if none is found.
36 */
37 static int similar_record(int rid, int traceFlag){
38 int inCnt, outCnt;
39 int i;
40 Stmt q;
41 int queue[100];
42 static const char *azQuery[] = {
43 /* Scan the delta table first */
 
 
44 "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
45 " FROM delta"
46 " WHERE rid=:x"
47 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=srcid)"
48 " UNION ALL "
49 "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
50 " FROM delta"
51 " WHERE srcid=:x"
52 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=delta.rid)",
53
54 /* Then the plink table */
55 "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
56 " FROM plink"
57 " WHERE cid=:x"
58 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
59 " UNION ALL "
60 "SELECT cid, EXISTS(SELECT 1 FROM onremote WHERE rid=cid)"
61 " FROM plink"
62 " WHERE pid=:x"
63 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=cid)",
64
65 /* Finally the mlink table */
66 "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
67 " FROM mlink"
68 " WHERE fid=:x AND pid>0"
69 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
70 " UNION ALL "
71 "SELECT fid, EXISTS(SELECT 1 FROM onremote WHERE rid=fid)"
72 " FROM mlink"
73 " WHERE pid=:x AND fid>0"
74 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=fid)",
75 };
76
77 for(i=0; i<sizeof(azQuery)/sizeof(azQuery[0]); i++){
78 db_prepare(&q, azQuery[i]);
79 queue[0] = rid;
80 inCnt = 1;
81 outCnt = 0;
82 if( traceFlag ) printf("PASS %d\n", i+1);
83 while( outCnt<inCnt ){
84 int xid = queue[outCnt%64];
85 outCnt++;
86 db_bind_int(&q, ":x", xid);
87 if( traceFlag ) printf("xid=%d\n", xid);
88 while( db_step(&q)==SQLITE_ROW ){
89 int nid = db_column_int(&q, 0);
90 int hit = db_column_int(&q, 1);
91 if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
92 if( hit ){
93 db_finalize(&q);
94 return nid;
95 }
96 if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
97 int i;
98 for(i=0; i<inCnt && queue[i]!=nid; i++){}
99 if( i>=inCnt ){
100 queue[inCnt++] = nid;
101 }
102 }
103 }
104 db_reset(&q);
105 }
106 db_finalize(&q);
107 }
108 return 0;
109 }
110
111 /*
112 ** COMMAND: test-similar-record
@@ -172,43 +200,37 @@
200 if( blob_size(&uuid)==0 ){
201 return 0;
202 }
203 content_get(rid, &content);
204
205 if( blob_size(&content)>100 ){
206 srcid = similar_record(rid, 0);
207 if( srcid ){
208 Blob src;
209 content_get(srcid, &src);
210 if( blob_size(&src)>100 ){
211 Blob delta;
212 blob_delta_create(&src, &content, &delta);
213 blob_reset(&content);
214 content = delta;
215 blob_append(&uuid, " ", 1);
216 blob_append(&content, "\n", 1);
217 db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
218 }
219 blob_reset(&src);
220 }
221 }
222 size = blob_size(&content);
223 if( pOut ){
224 blob_appendf(pOut, "file %b %d\n", &uuid, size);
225 blob_append(pOut, blob_buffer(&content), size);
226 }else{
227 cgi_printf("file %b %d\n", &uuid, size);
228 cgi_append_content(blob_buffer(&content), size);
229 }
230 blob_reset(&content);
231 blob_reset(&uuid);
 
 
 
 
 
 
232 db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
233 return size;
234 }
235
236
@@ -467,11 +489,11 @@
489 }
490 isPull = 1;
491 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
492 db_multi_exec(
493 "INSERT OR IGNORE INTO pending(rid) "
494 "SELECT mid FROM mlink JOIN blob ON mid=rid"
495 );
496 }else
497
498 /* login USER NONCE SIGNATURE
499 **
@@ -495,14 +517,22 @@
517 }
518
519 /* The input message has now been processed. Generate a reply. */
520 if( isPush ){
521 Stmt q;
522 int nReq = 0;
523 db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
524 while( db_step(&q)==SQLITE_ROW && nReq++ < 200 ){
525 const char *zUuid = db_column_text(&q, 0);
526 int rid = db_column_int(&q, 1);
527 int xid = similar_record(rid, 0);
528 @ gimme %s(zUuid)
529 if( xid ){
530 char *zXUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid);
531 @ igot %s(zXUuid);
532 free(zXUuid);
533 }
534 }
535 db_finalize(&q);
536 }
537 if( isPull ){
538 send_all_pending(0);
@@ -622,15 +652,21 @@
652
653 if( pullFlag ){
654 /* Send gimme message for every phantom that we hold.
655 */
656 Stmt q;
657 db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
658 while( db_step(&q)==SQLITE_ROW && nReq<200 ){
659 const char *zUuid = db_column_text(&q, 0);
660 int rid = db_column_int(&q, 1);
661 int xid = similar_record(rid, 0);
662 blob_appendf(&send,"gimme %s\n", zUuid);
663 nReq++;
664 if( xid ){
665 blob_appendf(&send, "igot %z\n",
666 db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid));
667 }
668 }
669 db_finalize(&q);
670 }
671
672 if( pushFlag ){
@@ -716,11 +752,11 @@
752 if( db_changes()>0 ){
753 go = 1;
754 }
755 }
756 if( pullFlag && !go &&
757 db_exists("SELECT 1 FROM phantom WHERE rid=%d", rid) ){
758 go = 1;
759 }
760 }else if( pullFlag ){
761 go = 1;
762 content_put(0, blob_str(&aToken[1]));
763

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button