Fossil SCM

An attempt to reduce the depth of recursion in order to run better on systems with limited stack spack. Ticket [2a1e8e3c4b0b39e08fdde0]. This check-in compiles and runs but has issues.

drh 2010-10-05 02:46 UTC trunk
Commit 9664989c0f7bcada3ceb400697db00d64ffc53f1
Parent e6c8df3bb73faa4…
2 files changed +204 -94 +73 -66
~ src/content.c ~ src/rebuild.c
M src/content.c
+204 -94
--- src/content.c
+++ src/content.c
@@ -22,30 +22,31 @@
2222
 #include <assert.h>
2323
2424
 /*
2525
 ** Macros for debugging
2626
 */
27
-#if 0
27
+#if 1
2828
 # define CONTENT_TRACE(X) printf X;
2929
 #else
3030
 # define CONTENT_TRACE(X)
3131
 #endif
3232
3333
 /*
3434
 ** The artifact retrival cache
3535
 */
36
-#define MX_CACHE_CNT 50 /* Maximum number of positive cache entries */
37
-#define EXPELL_INTERVAL 5 /* How often to expell from a full cache */
3836
 static struct {
39
- int n; /* Current number of positive cache entries */
37
+ i64 szTotal; /* Total size of all entries in the cache */
38
+ int n; /* Current number of eache entries */
39
+ int nAlloc; /* Number of slots allocated in a[] */
4040
 int nextAge; /* Age counter for implementing LRU */
4141
 int skipCnt; /* Used to limit entries expelled from cache */
42
- struct { /* One instance of this for each cache entry */
42
+ struct cacheLine { /* One instance of this for each cache entry */
4343
 int rid; /* Artifact id */
4444
 int age; /* Age. Newer is larger */
4545
 Blob content; /* Content of the artifact */
46
- } a[MX_CACHE_CNT]; /* The positive cache */
46
+ } *a; /* The positive cache */
47
+ Bag inCache; /* Set of artifacts currently in cache */
4748
4849
 /*
4950
 ** The missing artifact cache.
5051
 **
5152
 ** Artifacts whose record ID are in missingCache cannot be retrieved
@@ -56,10 +57,69 @@
5657
 */
5758
 Bag missing; /* Cache of artifacts that are incomplete */
5859
 Bag available; /* Cache of artifacts that are complete */
5960
 } contentCache;
6061
62
+/*
63
+** Remove the oldest element from the content cache
64
+*/
65
+static void content_cache_expire_oldest(void){
66
+ int i;
67
+ int mnAge = contentCache.nextAge;
68
+ int mn = -1;
69
+ for(i=0; i<contentCache.n; i++){
70
+ if( contentCache.a[i].age<mnAge ){
71
+ mnAge = contentCache.a[i].age;
72
+ mn = i;
73
+ }
74
+ }
75
+ if( mn>=0 ){
76
+ bag_remove(&contentCache.inCache, contentCache.a[mn].rid);
77
+ contentCache.szTotal -= blob_size(&contentCache.a[mn].content);
78
+ blob_reset(&contentCache.a[mn].content);
79
+ contentCache.n--;
80
+ contentCache.a[mn] = contentCache.a[contentCache.n];
81
+ }
82
+}
83
+
84
+/*
85
+** Add an entry to the content cache
86
+*/
87
+void content_cache_insert(int rid, Blob *pBlob){
88
+ struct cacheLine *p;
89
+ if( contentCache.n>500 || contentCache.szTotal>50000000 ){
90
+ content_cache_expire_oldest();
91
+ }
92
+ if( contentCache.n>=contentCache.nAlloc ){
93
+ contentCache.nAlloc = contentCache.nAlloc*2 + 10;
94
+ contentCache.a = realloc(contentCache.a,
95
+ contentCache.nAlloc*sizeof(contentCache.a[0]));
96
+ if( contentCache.a==0 ) fossil_panic("out of memory");
97
+ }
98
+ p = &contentCache.a[contentCache.n++];
99
+ p->rid = rid;
100
+ p->age = contentCache.nextAge++;
101
+ contentCache.szTotal += blob_size(pBlob);
102
+ p->content = *pBlob;
103
+ blob_zero(pBlob);
104
+ bag_insert(&contentCache.inCache, rid);
105
+}
106
+
107
+#if 0
108
+/*
109
+** Remove an entry from the content cache
110
+*/
111
+void content_cache_remove(int rid){
112
+ int i;
113
+ for(i=0; i<contentCache.n && contentCache.a[i].rid!=rid; i++){}
114
+ if( i>=contentCache.n ) return;
115
+ contentCache.szTotal -= blob_size(&contentCache.a[i].content);
116
+ blob_reset(&contentCache.a[i].content);
117
+ contentCache.n--;
118
+ contentCache.a[i] = contentCache.a[contentCache.n];
119
+}
120
+#endif
61121
62122
 /*
63123
 ** Clear the content cache.
64124
 */
65125
 void content_clear_cache(void){
@@ -67,11 +127,13 @@
67127
 for(i=0; i<contentCache.n; i++){
68128
 blob_reset(&contentCache.a[i].content);
69129
 }
70130
 bag_clear(&contentCache.missing);
71131
 bag_clear(&contentCache.available);
132
+ bag_clear(&contentCache.inCache);
72133
 contentCache.n = 0;
134
+ contentCache.szTotal = 0;
73135
 }
74136
75137
 /*
76138
 ** Return the srcid associated with rid. Or return 0 if rid is
77139
 ** original content and not a delta.
@@ -95,32 +157,31 @@
95157
 ** true if it is. Return false if rid is a phantom or depends on
96158
 ** a phantom.
97159
 */
98160
 int content_is_available(int rid){
99161
 int srcid;
100
- if( bag_find(&contentCache.missing, rid) ){
101
- return 0;
102
- }
103
- if( bag_find(&contentCache.available, rid) ){
104
- return 1;
105
- }
106
- if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
107
- bag_insert(&contentCache.missing, rid);
108
- return 0;
109
- }
110
- srcid = findSrcid(rid);
111
- if( srcid==0 ){
112
- bag_insert(&contentCache.available, rid);
113
- return 1;
114
- }
115
- if( content_is_available(srcid) ){
116
- bag_insert(&contentCache.available, rid);
117
- return 1;
118
- }else{
119
- bag_insert(&contentCache.missing, rid);
120
- return 0;
121
- }
162
+ int depth = 0; /* Limit to recursion depth */
163
+ while( depth++ < 10000000 ){
164
+ if( bag_find(&contentCache.missing, rid) ){
165
+ return 0;
166
+ }
167
+ if( bag_find(&contentCache.available, rid) ){
168
+ return 1;
169
+ }
170
+ if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
171
+ bag_insert(&contentCache.missing, rid);
172
+ return 0;
173
+ }
174
+ srcid = findSrcid(rid);
175
+ if( srcid==0 ){
176
+ bag_insert(&contentCache.available, rid);
177
+ return 1;
178
+ }
179
+ rid = srcid;
180
+ }
181
+ fossil_panic("delta-loop in repository");
182
+ return 0;
122183
 }
123184
124185
 /*
125186
 ** Mark artifact rid as being available now. Update the cache to
126187
 ** show that everything that was formerly unavailable because rid
@@ -163,17 +224,95 @@
163224
 }
164225
 db_reset(&q);
165226
 return rc;
166227
 }
167228
168
-
169229
 /*
170230
 ** Extract the content for ID rid and put it into the
171231
 ** uninitialized blob. Return 1 on success. If the record
172232
 ** is a phantom, zero pBlob and return 0.
173233
 */
174234
 int content_get(int rid, Blob *pBlob){
235
+ int rc;
236
+ int i;
237
+ int nextRid;
238
+
239
+ assert( g.repositoryOpen );
240
+ blob_zero(pBlob);
241
+ if( rid==0 ) return 0;
242
+
243
+ /* Early out if we know the content is not available */
244
+ if( bag_find(&contentCache.missing, rid) ){
245
+ return 0;
246
+ }
247
+
248
+ /* Look for the artifact in the cache first */
249
+ if( bag_find(&contentCache.inCache, rid) ){
250
+ for(i=0; i<contentCache.n; i++){
251
+ if( contentCache.a[i].rid==rid ){
252
+ blob_copy(pBlob, &contentCache.a[i].content);
253
+ contentCache.a[i].age = contentCache.nextAge++;
254
+ return 1;
255
+ }
256
+ }
257
+ }
258
+
259
+ nextRid = findSrcid(rid);
260
+ if( nextRid==0 ){
261
+ rc = content_of_blob(rid, pBlob);
262
+ }else{
263
+ int n = 1;
264
+ int nAlloc = 10;
265
+ int *a = 0;
266
+ int mx;
267
+ Blob delta, next;
268
+
269
+ a = malloc( sizeof(a[0])*nAlloc );
270
+ if( a==0 ) fossil_panic("out of memory");
271
+ a[0] = rid;
272
+ a[1] = nextRid;
273
+ n = 1;
274
+ while( !bag_find(&contentCache.inCache, nextRid)
275
+ && (nextRid = findSrcid(nextRid))>0 ){
276
+ n++;
277
+ if( n>=nAlloc ){
278
+ nAlloc = nAlloc*2 + 10;
279
+ a = realloc(a, nAlloc*sizeof(a[0]));
280
+ if( a==0 ) fossil_panic("out of memory");
281
+ }
282
+ a[n] = nextRid;
283
+ }
284
+ mx = n;
285
+ rc = content_of_blob(a[n], pBlob);
286
+ n--;
287
+ while( rc && n>=0 ){
288
+ rc = content_of_blob(a[n], &delta);
289
+ if( rc ){
290
+ blob_delta_apply(pBlob, &delta, &next);
291
+ blob_reset(&delta);
292
+ if( (mx-n)%8==0 ){
293
+ content_cache_insert(a[n+1], pBlob);
294
+ }else{
295
+ blob_reset(pBlob);
296
+ }
297
+ *pBlob = next;
298
+ }
299
+ n--;
300
+ }
301
+ free(a);
302
+ if( !rc ) blob_reset(pBlob);
303
+ }
304
+ return rc;
305
+}
306
+
307
+#if 0
308
+/*
309
+** Extract the content for ID rid and put it into the
310
+** uninitialized blob. Return 1 on success. If the record
311
+** is a phantom, zero pBlob and return 0.
312
+*/
313
+int old_content_get(int rid, Blob *pBlob){
175314
 Blob src;
176315
 int srcid;
177316
 int rc = 0;
178317
 int i;
179318
 static Bag inProcess;
@@ -182,36 +321,24 @@
182321
 blob_zero(pBlob);
183322
 if( rid==0 ) return 0;
184323
185324
 /* Early out if we know the content is not available */
186325
 if( bag_find(&contentCache.missing, rid) ){
187
- CONTENT_TRACE(("%*smiss from cache: %d\n",
188
- bag_count(&inProcess), "", rid))
189326
 return 0;
190327
 }
191328
192329
 /* Look for the artifact in the cache first */
193330
 for(i=0; i<contentCache.n; i++){
194331
 if( contentCache.a[i].rid==rid ){
195
- *pBlob = contentCache.a[i].content;
196
- blob_zero(&contentCache.a[i].content);
197
- contentCache.n--;
198
- if( i<contentCache.n ){
199
- contentCache.a[i] = contentCache.a[contentCache.n];
200
- }
201
- CONTENT_TRACE(("%*scache: %d\n",
202
- bag_count(&inProcess), "", rid))
332
+ blob_copy(pBlob, &contentCache.a[i].content);
333
+ contentCache.a[i].age = contentCache.nextAge++;
203334
 return 1;
204335
 }
205336
 }
206337
207338
 /* See if we need to apply a delta to find this artifact */
208339
 srcid = findSrcid(rid);
209
- CONTENT_TRACE(("%*ssearching for %d. Need %d.\n",
210
- bag_count(&inProcess), "", rid, srcid))
211
-
212
-
213340
 if( srcid ){
214341
 /* Yes, a delta is required */
215342
 if( bag_find(&inProcess, srcid) ){
216343
 db_multi_exec(
217344
 "UPDATE blob SET content=NULL, size=-1 WHERE rid=%d;"
@@ -222,47 +349,19 @@
222349
 blob_zero(pBlob);
223350
 return 0;
224351
 }
225352
 bag_insert(&inProcess, srcid);
226353
354
+ cacheSrcid(rid);
227355
 if( content_get(srcid, &src) ){
228356
 Blob delta;
229357
 if( content_of_blob(rid, &delta) ){
230358
 blob_init(pBlob,0,0);
231359
 blob_delta_apply(&src, &delta, pBlob);
232360
 blob_reset(&delta);
233361
 rc = 1;
234362
 }
235
-
236
- /* Save the srcid artifact in the cache */
237
- if( contentCache.n<MX_CACHE_CNT ){
238
- i = contentCache.n++;
239
- }else if( ((contentCache.skipCnt++)%EXPELL_INTERVAL)!=0 ){
240
- i = -1;
241
- }else{
242
- int j, best;
243
- best = contentCache.nextAge+1;
244
- i = -1;
245
- for(j=0; j<contentCache.n; j++){
246
- if( contentCache.a[j].age<best ){
247
- i = j;
248
- best = contentCache.a[j].age;
249
- }
250
- }
251
- CONTENT_TRACE(("%*sexpell %d from cache\n",
252
- bag_count(&inProcess), "", contentCache.a[i].rid))
253
- blob_reset(&contentCache.a[i].content);
254
- }
255
- if( i>=0 ){
256
- contentCache.a[i].content = src;
257
- contentCache.a[i].age = contentCache.nextAge++;
258
- contentCache.a[i].rid = srcid;
259
- CONTENT_TRACE(("%*sadd %d to cache\n",
260
- bag_count(&inProcess), "", srcid))
261
- }else{
262
- blob_reset(&src);
263
- }
264363
 }
265364
 bag_remove(&inProcess, srcid);
266365
 }else{
267366
 /* No delta required. Read content directly from the database */
268367
 if( content_of_blob(rid, pBlob) ){
@@ -274,10 +373,11 @@
274373
 }else{
275374
 bag_insert(&contentCache.available, rid);
276375
 }
277376
 return rc;
278377
 }
378
+#endif
279379
280380
 /*
281381
 ** COMMAND: artifact
282382
 **
283383
 ** Usage: %fossil artifact ARTIFACT-ID ?OUTPUT-FILENAME?
@@ -320,35 +420,45 @@
320420
321421
 /*
322422
 ** When a record is converted from a phantom to a real record,
323423
 ** if that record has other records that are derived by delta,
324424
 ** then call manifest_crosslink() on those other records.
425
+**
426
+** Tail recursion is used to minimize stack depth.
325427
 */
326428
 void after_dephantomize(int rid, int linkFlag){
327429
 Stmt q;
328
- int prevTid = 0;
329
-
330
- /* The prevTid variable is used to delay invoking this routine
331
- ** recursively, if possible, until after the query has finalized,
332
- ** in order to avoid having an excessive number of prepared statements.
333
- ** This is most effective in the common case where the query returns
334
- ** just one row.
335
- */
336
- db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
337
- while( db_step(&q)==SQLITE_ROW ){
338
- int tid = db_column_int(&q, 0);
339
- if( prevTid ) after_dephantomize(prevTid, 1);
340
- prevTid = tid;
341
- }
342
- db_finalize(&q);
343
- if( prevTid ) after_dephantomize(prevTid, 1);
344
- if( linkFlag ){
345
- Blob content;
346
- content_get(rid, &content);
347
- manifest_crosslink(rid, &content);
348
- blob_reset(&content);
349
- }
430
+ int nChildAlloc = 0;
431
+ int *aChild = 0;
432
+
433
+ while( rid ){
434
+ int nChildUsed = 0;
435
+ int i;
436
+ if( linkFlag ){
437
+ Blob content;
438
+ content_get(rid, &content);
439
+ manifest_crosslink(rid, &content);
440
+ blob_reset(&content);
441
+ }
442
+ db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
443
+ while( db_step(&q)==SQLITE_ROW ){
444
+ int child = db_column_int(&q, 0);
445
+ if( nChildUsed>=nChildAlloc ){
446
+ nChildAlloc = nChildAlloc*2 + 10;
447
+ aChild = realloc(aChild, nChildAlloc*sizeof(aChild));
448
+ if( aChild==0 ) fossil_panic("out of memory");
449
+ }
450
+ aChild[nChildUsed++] = child;
451
+ }
452
+ db_finalize(&q);
453
+ for(i=1; i<nChildUsed; i++){
454
+ after_dephantomize(aChild[i], 1);
455
+ }
456
+ rid = nChildUsed>0 ? aChild[0] : 0;
457
+ linkFlag = 1;
458
+ }
459
+ free(aChild);
350460
 }
351461
352462
 /*
353463
 ** Write content into the database. Return the record ID. If the
354464
 ** content is already in the database, just return the record ID.
355465
--- src/content.c
+++ src/content.c
@@ -22,30 +22,31 @@
22 #include <assert.h>
23
24 /*
25 ** Macros for debugging
26 */
27 #if 0
28 # define CONTENT_TRACE(X) printf X;
29 #else
30 # define CONTENT_TRACE(X)
31 #endif
32
33 /*
34 ** The artifact retrival cache
35 */
36 #define MX_CACHE_CNT 50 /* Maximum number of positive cache entries */
37 #define EXPELL_INTERVAL 5 /* How often to expell from a full cache */
38 static struct {
39 int n; /* Current number of positive cache entries */
 
 
40 int nextAge; /* Age counter for implementing LRU */
41 int skipCnt; /* Used to limit entries expelled from cache */
42 struct { /* One instance of this for each cache entry */
43 int rid; /* Artifact id */
44 int age; /* Age. Newer is larger */
45 Blob content; /* Content of the artifact */
46 } a[MX_CACHE_CNT]; /* The positive cache */
 
47
48 /*
49 ** The missing artifact cache.
50 **
51 ** Artifacts whose record ID are in missingCache cannot be retrieved
@@ -56,10 +57,69 @@
56 */
57 Bag missing; /* Cache of artifacts that are incomplete */
58 Bag available; /* Cache of artifacts that are complete */
59 } contentCache;
60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
62 /*
63 ** Clear the content cache.
64 */
65 void content_clear_cache(void){
@@ -67,11 +127,13 @@
67 for(i=0; i<contentCache.n; i++){
68 blob_reset(&contentCache.a[i].content);
69 }
70 bag_clear(&contentCache.missing);
71 bag_clear(&contentCache.available);
 
72 contentCache.n = 0;
 
73 }
74
75 /*
76 ** Return the srcid associated with rid. Or return 0 if rid is
77 ** original content and not a delta.
@@ -95,32 +157,31 @@
95 ** true if it is. Return false if rid is a phantom or depends on
96 ** a phantom.
97 */
98 int content_is_available(int rid){
99 int srcid;
100 if( bag_find(&contentCache.missing, rid) ){
101 return 0;
102 }
103 if( bag_find(&contentCache.available, rid) ){
104 return 1;
105 }
106 if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
107 bag_insert(&contentCache.missing, rid);
108 return 0;
109 }
110 srcid = findSrcid(rid);
111 if( srcid==0 ){
112 bag_insert(&contentCache.available, rid);
113 return 1;
114 }
115 if( content_is_available(srcid) ){
116 bag_insert(&contentCache.available, rid);
117 return 1;
118 }else{
119 bag_insert(&contentCache.missing, rid);
120 return 0;
121 }
122 }
123
124 /*
125 ** Mark artifact rid as being available now. Update the cache to
126 ** show that everything that was formerly unavailable because rid
@@ -163,17 +224,95 @@
163 }
164 db_reset(&q);
165 return rc;
166 }
167
168
169 /*
170 ** Extract the content for ID rid and put it into the
171 ** uninitialized blob. Return 1 on success. If the record
172 ** is a phantom, zero pBlob and return 0.
173 */
174 int content_get(int rid, Blob *pBlob){
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175 Blob src;
176 int srcid;
177 int rc = 0;
178 int i;
179 static Bag inProcess;
@@ -182,36 +321,24 @@
182 blob_zero(pBlob);
183 if( rid==0 ) return 0;
184
185 /* Early out if we know the content is not available */
186 if( bag_find(&contentCache.missing, rid) ){
187 CONTENT_TRACE(("%*smiss from cache: %d\n",
188 bag_count(&inProcess), "", rid))
189 return 0;
190 }
191
192 /* Look for the artifact in the cache first */
193 for(i=0; i<contentCache.n; i++){
194 if( contentCache.a[i].rid==rid ){
195 *pBlob = contentCache.a[i].content;
196 blob_zero(&contentCache.a[i].content);
197 contentCache.n--;
198 if( i<contentCache.n ){
199 contentCache.a[i] = contentCache.a[contentCache.n];
200 }
201 CONTENT_TRACE(("%*scache: %d\n",
202 bag_count(&inProcess), "", rid))
203 return 1;
204 }
205 }
206
207 /* See if we need to apply a delta to find this artifact */
208 srcid = findSrcid(rid);
209 CONTENT_TRACE(("%*ssearching for %d. Need %d.\n",
210 bag_count(&inProcess), "", rid, srcid))
211
212
213 if( srcid ){
214 /* Yes, a delta is required */
215 if( bag_find(&inProcess, srcid) ){
216 db_multi_exec(
217 "UPDATE blob SET content=NULL, size=-1 WHERE rid=%d;"
@@ -222,47 +349,19 @@
222 blob_zero(pBlob);
223 return 0;
224 }
225 bag_insert(&inProcess, srcid);
226
 
227 if( content_get(srcid, &src) ){
228 Blob delta;
229 if( content_of_blob(rid, &delta) ){
230 blob_init(pBlob,0,0);
231 blob_delta_apply(&src, &delta, pBlob);
232 blob_reset(&delta);
233 rc = 1;
234 }
235
236 /* Save the srcid artifact in the cache */
237 if( contentCache.n<MX_CACHE_CNT ){
238 i = contentCache.n++;
239 }else if( ((contentCache.skipCnt++)%EXPELL_INTERVAL)!=0 ){
240 i = -1;
241 }else{
242 int j, best;
243 best = contentCache.nextAge+1;
244 i = -1;
245 for(j=0; j<contentCache.n; j++){
246 if( contentCache.a[j].age<best ){
247 i = j;
248 best = contentCache.a[j].age;
249 }
250 }
251 CONTENT_TRACE(("%*sexpell %d from cache\n",
252 bag_count(&inProcess), "", contentCache.a[i].rid))
253 blob_reset(&contentCache.a[i].content);
254 }
255 if( i>=0 ){
256 contentCache.a[i].content = src;
257 contentCache.a[i].age = contentCache.nextAge++;
258 contentCache.a[i].rid = srcid;
259 CONTENT_TRACE(("%*sadd %d to cache\n",
260 bag_count(&inProcess), "", srcid))
261 }else{
262 blob_reset(&src);
263 }
264 }
265 bag_remove(&inProcess, srcid);
266 }else{
267 /* No delta required. Read content directly from the database */
268 if( content_of_blob(rid, pBlob) ){
@@ -274,10 +373,11 @@
274 }else{
275 bag_insert(&contentCache.available, rid);
276 }
277 return rc;
278 }
 
279
280 /*
281 ** COMMAND: artifact
282 **
283 ** Usage: %fossil artifact ARTIFACT-ID ?OUTPUT-FILENAME?
@@ -320,35 +420,45 @@
320
321 /*
322 ** When a record is converted from a phantom to a real record,
323 ** if that record has other records that are derived by delta,
324 ** then call manifest_crosslink() on those other records.
 
 
325 */
326 void after_dephantomize(int rid, int linkFlag){
327 Stmt q;
328 int prevTid = 0;
329
330 /* The prevTid variable is used to delay invoking this routine
331 ** recursively, if possible, until after the query has finalized,
332 ** in order to avoid having an excessive number of prepared statements.
333 ** This is most effective in the common case where the query returns
334 ** just one row.
335 */
336 db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
337 while( db_step(&q)==SQLITE_ROW ){
338 int tid = db_column_int(&q, 0);
339 if( prevTid ) after_dephantomize(prevTid, 1);
340 prevTid = tid;
341 }
342 db_finalize(&q);
343 if( prevTid ) after_dephantomize(prevTid, 1);
344 if( linkFlag ){
345 Blob content;
346 content_get(rid, &content);
347 manifest_crosslink(rid, &content);
348 blob_reset(&content);
349 }
 
 
 
 
 
 
 
 
350 }
351
352 /*
353 ** Write content into the database. Return the record ID. If the
354 ** content is already in the database, just return the record ID.
355
--- src/content.c
+++ src/content.c
@@ -22,30 +22,31 @@
22 #include <assert.h>
23
24 /*
25 ** Macros for debugging
26 */
27 #if 1
28 # define CONTENT_TRACE(X) printf X;
29 #else
30 # define CONTENT_TRACE(X)
31 #endif
32
33 /*
34 ** The artifact retrival cache
35 */
 
 
36 static struct {
37 i64 szTotal; /* Total size of all entries in the cache */
38 int n; /* Current number of eache entries */
39 int nAlloc; /* Number of slots allocated in a[] */
40 int nextAge; /* Age counter for implementing LRU */
41 int skipCnt; /* Used to limit entries expelled from cache */
42 struct cacheLine { /* One instance of this for each cache entry */
43 int rid; /* Artifact id */
44 int age; /* Age. Newer is larger */
45 Blob content; /* Content of the artifact */
46 } *a; /* The positive cache */
47 Bag inCache; /* Set of artifacts currently in cache */
48
49 /*
50 ** The missing artifact cache.
51 **
52 ** Artifacts whose record ID are in missingCache cannot be retrieved
@@ -56,10 +57,69 @@
57 */
58 Bag missing; /* Cache of artifacts that are incomplete */
59 Bag available; /* Cache of artifacts that are complete */
60 } contentCache;
61
62 /*
63 ** Remove the oldest element from the content cache
64 */
65 static void content_cache_expire_oldest(void){
66 int i;
67 int mnAge = contentCache.nextAge;
68 int mn = -1;
69 for(i=0; i<contentCache.n; i++){
70 if( contentCache.a[i].age<mnAge ){
71 mnAge = contentCache.a[i].age;
72 mn = i;
73 }
74 }
75 if( mn>=0 ){
76 bag_remove(&contentCache.inCache, contentCache.a[mn].rid);
77 contentCache.szTotal -= blob_size(&contentCache.a[mn].content);
78 blob_reset(&contentCache.a[mn].content);
79 contentCache.n--;
80 contentCache.a[mn] = contentCache.a[contentCache.n];
81 }
82 }
83
84 /*
85 ** Add an entry to the content cache
86 */
87 void content_cache_insert(int rid, Blob *pBlob){
88 struct cacheLine *p;
89 if( contentCache.n>500 || contentCache.szTotal>50000000 ){
90 content_cache_expire_oldest();
91 }
92 if( contentCache.n>=contentCache.nAlloc ){
93 contentCache.nAlloc = contentCache.nAlloc*2 + 10;
94 contentCache.a = realloc(contentCache.a,
95 contentCache.nAlloc*sizeof(contentCache.a[0]));
96 if( contentCache.a==0 ) fossil_panic("out of memory");
97 }
98 p = &contentCache.a[contentCache.n++];
99 p->rid = rid;
100 p->age = contentCache.nextAge++;
101 contentCache.szTotal += blob_size(pBlob);
102 p->content = *pBlob;
103 blob_zero(pBlob);
104 bag_insert(&contentCache.inCache, rid);
105 }
106
107 #if 0
108 /*
109 ** Remove an entry from the content cache
110 */
111 void content_cache_remove(int rid){
112 int i;
113 for(i=0; i<contentCache.n && contentCache.a[i].rid!=rid; i++){}
114 if( i>=contentCache.n ) return;
115 contentCache.szTotal -= blob_size(&contentCache.a[i].content);
116 blob_reset(&contentCache.a[i].content);
117 contentCache.n--;
118 contentCache.a[i] = contentCache.a[contentCache.n];
119 }
120 #endif
121
122 /*
123 ** Clear the content cache.
124 */
125 void content_clear_cache(void){
@@ -67,11 +127,13 @@
127 for(i=0; i<contentCache.n; i++){
128 blob_reset(&contentCache.a[i].content);
129 }
130 bag_clear(&contentCache.missing);
131 bag_clear(&contentCache.available);
132 bag_clear(&contentCache.inCache);
133 contentCache.n = 0;
134 contentCache.szTotal = 0;
135 }
136
137 /*
138 ** Return the srcid associated with rid. Or return 0 if rid is
139 ** original content and not a delta.
@@ -95,32 +157,31 @@
157 ** true if it is. Return false if rid is a phantom or depends on
158 ** a phantom.
159 */
160 int content_is_available(int rid){
161 int srcid;
162 int depth = 0; /* Limit to recursion depth */
163 while( depth++ < 10000000 ){
164 if( bag_find(&contentCache.missing, rid) ){
165 return 0;
166 }
167 if( bag_find(&contentCache.available, rid) ){
168 return 1;
169 }
170 if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
171 bag_insert(&contentCache.missing, rid);
172 return 0;
173 }
174 srcid = findSrcid(rid);
175 if( srcid==0 ){
176 bag_insert(&contentCache.available, rid);
177 return 1;
178 }
179 rid = srcid;
180 }
181 fossil_panic("delta-loop in repository");
182 return 0;
 
183 }
184
185 /*
186 ** Mark artifact rid as being available now. Update the cache to
187 ** show that everything that was formerly unavailable because rid
@@ -163,17 +224,95 @@
224 }
225 db_reset(&q);
226 return rc;
227 }
228
 
229 /*
230 ** Extract the content for ID rid and put it into the
231 ** uninitialized blob. Return 1 on success. If the record
232 ** is a phantom, zero pBlob and return 0.
233 */
234 int content_get(int rid, Blob *pBlob){
235 int rc;
236 int i;
237 int nextRid;
238
239 assert( g.repositoryOpen );
240 blob_zero(pBlob);
241 if( rid==0 ) return 0;
242
243 /* Early out if we know the content is not available */
244 if( bag_find(&contentCache.missing, rid) ){
245 return 0;
246 }
247
248 /* Look for the artifact in the cache first */
249 if( bag_find(&contentCache.inCache, rid) ){
250 for(i=0; i<contentCache.n; i++){
251 if( contentCache.a[i].rid==rid ){
252 blob_copy(pBlob, &contentCache.a[i].content);
253 contentCache.a[i].age = contentCache.nextAge++;
254 return 1;
255 }
256 }
257 }
258
259 nextRid = findSrcid(rid);
260 if( nextRid==0 ){
261 rc = content_of_blob(rid, pBlob);
262 }else{
263 int n = 1;
264 int nAlloc = 10;
265 int *a = 0;
266 int mx;
267 Blob delta, next;
268
269 a = malloc( sizeof(a[0])*nAlloc );
270 if( a==0 ) fossil_panic("out of memory");
271 a[0] = rid;
272 a[1] = nextRid;
273 n = 1;
274 while( !bag_find(&contentCache.inCache, nextRid)
275 && (nextRid = findSrcid(nextRid))>0 ){
276 n++;
277 if( n>=nAlloc ){
278 nAlloc = nAlloc*2 + 10;
279 a = realloc(a, nAlloc*sizeof(a[0]));
280 if( a==0 ) fossil_panic("out of memory");
281 }
282 a[n] = nextRid;
283 }
284 mx = n;
285 rc = content_of_blob(a[n], pBlob);
286 n--;
287 while( rc && n>=0 ){
288 rc = content_of_blob(a[n], &delta);
289 if( rc ){
290 blob_delta_apply(pBlob, &delta, &next);
291 blob_reset(&delta);
292 if( (mx-n)%8==0 ){
293 content_cache_insert(a[n+1], pBlob);
294 }else{
295 blob_reset(pBlob);
296 }
297 *pBlob = next;
298 }
299 n--;
300 }
301 free(a);
302 if( !rc ) blob_reset(pBlob);
303 }
304 return rc;
305 }
306
307 #if 0
308 /*
309 ** Extract the content for ID rid and put it into the
310 ** uninitialized blob. Return 1 on success. If the record
311 ** is a phantom, zero pBlob and return 0.
312 */
313 int old_content_get(int rid, Blob *pBlob){
314 Blob src;
315 int srcid;
316 int rc = 0;
317 int i;
318 static Bag inProcess;
@@ -182,36 +321,24 @@
321 blob_zero(pBlob);
322 if( rid==0 ) return 0;
323
324 /* Early out if we know the content is not available */
325 if( bag_find(&contentCache.missing, rid) ){
 
 
326 return 0;
327 }
328
329 /* Look for the artifact in the cache first */
330 for(i=0; i<contentCache.n; i++){
331 if( contentCache.a[i].rid==rid ){
332 blob_copy(pBlob, &contentCache.a[i].content);
333 contentCache.a[i].age = contentCache.nextAge++;
 
 
 
 
 
 
334 return 1;
335 }
336 }
337
338 /* See if we need to apply a delta to find this artifact */
339 srcid = findSrcid(rid);
 
 
 
 
340 if( srcid ){
341 /* Yes, a delta is required */
342 if( bag_find(&inProcess, srcid) ){
343 db_multi_exec(
344 "UPDATE blob SET content=NULL, size=-1 WHERE rid=%d;"
@@ -222,47 +349,19 @@
349 blob_zero(pBlob);
350 return 0;
351 }
352 bag_insert(&inProcess, srcid);
353
354 cacheSrcid(rid);
355 if( content_get(srcid, &src) ){
356 Blob delta;
357 if( content_of_blob(rid, &delta) ){
358 blob_init(pBlob,0,0);
359 blob_delta_apply(&src, &delta, pBlob);
360 blob_reset(&delta);
361 rc = 1;
362 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
363 }
364 bag_remove(&inProcess, srcid);
365 }else{
366 /* No delta required. Read content directly from the database */
367 if( content_of_blob(rid, pBlob) ){
@@ -274,10 +373,11 @@
373 }else{
374 bag_insert(&contentCache.available, rid);
375 }
376 return rc;
377 }
378 #endif
379
380 /*
381 ** COMMAND: artifact
382 **
383 ** Usage: %fossil artifact ARTIFACT-ID ?OUTPUT-FILENAME?
@@ -320,35 +420,45 @@
420
421 /*
422 ** When a record is converted from a phantom to a real record,
423 ** if that record has other records that are derived by delta,
424 ** then call manifest_crosslink() on those other records.
425 **
426 ** Tail recursion is used to minimize stack depth.
427 */
428 void after_dephantomize(int rid, int linkFlag){
429 Stmt q;
430 int nChildAlloc = 0;
431 int *aChild = 0;
432
433 while( rid ){
434 int nChildUsed = 0;
435 int i;
436 if( linkFlag ){
437 Blob content;
438 content_get(rid, &content);
439 manifest_crosslink(rid, &content);
440 blob_reset(&content);
441 }
442 db_prepare(&q, "SELECT rid FROM delta WHERE srcid=%d", rid);
443 while( db_step(&q)==SQLITE_ROW ){
444 int child = db_column_int(&q, 0);
445 if( nChildUsed>=nChildAlloc ){
446 nChildAlloc = nChildAlloc*2 + 10;
447 aChild = realloc(aChild, nChildAlloc*sizeof(aChild));
448 if( aChild==0 ) fossil_panic("out of memory");
449 }
450 aChild[nChildUsed++] = child;
451 }
452 db_finalize(&q);
453 for(i=1; i<nChildUsed; i++){
454 after_dephantomize(aChild[i], 1);
455 }
456 rid = nChildUsed>0 ? aChild[0] : 0;
457 linkFlag = 1;
458 }
459 free(aChild);
460 }
461
462 /*
463 ** Write content into the database. Return the record ID. If the
464 ** content is already in the database, just return the record ID.
465
M src/rebuild.c
+73 -66
--- src/rebuild.c
+++ src/rebuild.c
@@ -121,76 +121,83 @@
121121
 Bag children;
122122
 Blob copy;
123123
 Blob *pUse;
124124
 int nChild, i, cid;
125125
126
- /* Fix up the "blob.size" field if needed. */
127
- if( size!=blob_size(pBase) ){
128
- db_multi_exec(
129
- "UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid
130
- );
131
- }
132
-
133
- /* Find all children of artifact rid */
134
- db_static_prepare(&q1, "SELECT rid FROM delta WHERE srcid=:rid");
135
- db_bind_int(&q1, ":rid", rid);
136
- bag_init(&children);
137
- while( db_step(&q1)==SQLITE_ROW ){
138
- int cid = db_column_int(&q1, 0);
139
- if( !bag_find(&bagDone, cid) ){
140
- bag_insert(&children, cid);
141
- }
142
- }
143
- nChild = bag_count(&children);
144
- db_reset(&q1);
145
-
146
- /* Crosslink the artifact */
147
- if( nChild==0 ){
148
- pUse = pBase;
149
- }else{
150
- blob_copy(&copy, pBase);
151
- pUse = &copy;
152
- }
153
- if( zFNameFormat==0 ){
154
- /* We are doing "fossil rebuild" */
155
- manifest_crosslink(rid, pUse);
156
- }else{
157
- /* We are doing "fossil deconstruct" */
158
- char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
159
- char *zFile = mprintf(zFNameFormat, zUuid, zUuid+prefixLength);
160
- blob_write_to_file(pUse,zFile);
161
- free(zFile);
162
- free(zUuid);
163
- }
164
- blob_reset(pUse);
165
- rebuild_step_done(rid);
166
-
167
- /* Call all children recursively */
168
- for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){
169
- Stmt q2;
170
- int sz;
171
- if( nChild==i ){
172
- pUse = pBase;
173
- }else{
174
- blob_copy(&copy, pBase);
175
- pUse = &copy;
176
- }
177
- db_prepare(&q2, "SELECT content, size FROM blob WHERE rid=%d", cid);
178
- if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){
179
- Blob delta;
180
- db_ephemeral_blob(&q2, 0, &delta);
181
- blob_uncompress(&delta, &delta);
182
- blob_delta_apply(pUse, &delta, pUse);
183
- blob_reset(&delta);
184
- db_finalize(&q2);
185
- rebuild_step(cid, sz, pUse);
186
- }else{
187
- db_finalize(&q2);
188
- blob_reset(pUse);
189
- }
190
- }
191
- bag_clear(&children);
126
+ while( rid>0 ){
127
+
128
+ /* Fix up the "blob.size" field if needed. */
129
+ if( size!=blob_size(pBase) ){
130
+ db_multi_exec(
131
+ "UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid
132
+ );
133
+ }
134
+
135
+ /* Find all children of artifact rid */
136
+ db_static_prepare(&q1, "SELECT rid FROM delta WHERE srcid=:rid");
137
+ db_bind_int(&q1, ":rid", rid);
138
+ bag_init(&children);
139
+ while( db_step(&q1)==SQLITE_ROW ){
140
+ int cid = db_column_int(&q1, 0);
141
+ if( !bag_find(&bagDone, cid) ){
142
+ bag_insert(&children, cid);
143
+ }
144
+ }
145
+ nChild = bag_count(&children);
146
+ db_reset(&q1);
147
+
148
+ /* Crosslink the artifact */
149
+ if( nChild==0 ){
150
+ pUse = pBase;
151
+ }else{
152
+ blob_copy(&copy, pBase);
153
+ pUse = &copy;
154
+ }
155
+ if( zFNameFormat==0 ){
156
+ /* We are doing "fossil rebuild" */
157
+ manifest_crosslink(rid, pUse);
158
+ }else{
159
+ /* We are doing "fossil deconstruct" */
160
+ char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
161
+ char *zFile = mprintf(zFNameFormat, zUuid, zUuid+prefixLength);
162
+// blob_write_to_file(pUse,zFile);
163
+printf("%d - %s\n", rid, zUuid);
164
+ free(zFile);
165
+ free(zUuid);
166
+ }
167
+ blob_reset(pUse);
168
+ rebuild_step_done(rid);
169
+
170
+ /* Call all children recursively */
171
+ rid = 0;
172
+ for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){
173
+ Stmt q2;
174
+ int sz;
175
+ db_prepare(&q2, "SELECT content, size FROM blob WHERE rid=%d", cid);
176
+ if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){
177
+ Blob delta, next;
178
+ db_ephemeral_blob(&q2, 0, &delta);
179
+ blob_uncompress(&delta, &delta);
180
+ blob_delta_apply(pBase, &delta, &next);
181
+ blob_reset(&delta);
182
+ db_finalize(&q2);
183
+ if( i<nChild ){
184
+ rebuild_step(cid, sz, &next);
185
+ }else{
186
+ /* Tail recursion */
187
+ rid = cid;
188
+ size = sz;
189
+ blob_reset(pBase);
190
+ *pBase = next;
191
+ }
192
+ }else{
193
+ db_finalize(&q2);
194
+ blob_reset(pBase);
195
+ }
196
+ }
197
+ bag_clear(&children);
198
+ }
192199
 }
193200
194201
 /*
195202
 ** Check to see if the "sym-trunk" tag exists. If not, create it
196203
 ** and attach it to the very first check-in.
197204
--- src/rebuild.c
+++ src/rebuild.c
@@ -121,76 +121,83 @@
121 Bag children;
122 Blob copy;
123 Blob *pUse;
124 int nChild, i, cid;
125
126 /* Fix up the "blob.size" field if needed. */
127 if( size!=blob_size(pBase) ){
128 db_multi_exec(
129 "UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid
130 );
131 }
132
133 /* Find all children of artifact rid */
134 db_static_prepare(&q1, "SELECT rid FROM delta WHERE srcid=:rid");
135 db_bind_int(&q1, ":rid", rid);
136 bag_init(&children);
137 while( db_step(&q1)==SQLITE_ROW ){
138 int cid = db_column_int(&q1, 0);
139 if( !bag_find(&bagDone, cid) ){
140 bag_insert(&children, cid);
141 }
142 }
143 nChild = bag_count(&children);
144 db_reset(&q1);
145
146 /* Crosslink the artifact */
147 if( nChild==0 ){
148 pUse = pBase;
149 }else{
150 blob_copy(&copy, pBase);
151 pUse = &copy;
152 }
153 if( zFNameFormat==0 ){
154 /* We are doing "fossil rebuild" */
155 manifest_crosslink(rid, pUse);
156 }else{
157 /* We are doing "fossil deconstruct" */
158 char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
159 char *zFile = mprintf(zFNameFormat, zUuid, zUuid+prefixLength);
160 blob_write_to_file(pUse,zFile);
161 free(zFile);
162 free(zUuid);
163 }
164 blob_reset(pUse);
165 rebuild_step_done(rid);
166
167 /* Call all children recursively */
168 for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){
169 Stmt q2;
170 int sz;
171 if( nChild==i ){
172 pUse = pBase;
173 }else{
174 blob_copy(&copy, pBase);
175 pUse = &copy;
176 }
177 db_prepare(&q2, "SELECT content, size FROM blob WHERE rid=%d", cid);
178 if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){
179 Blob delta;
180 db_ephemeral_blob(&q2, 0, &delta);
181 blob_uncompress(&delta, &delta);
182 blob_delta_apply(pUse, &delta, pUse);
183 blob_reset(&delta);
184 db_finalize(&q2);
185 rebuild_step(cid, sz, pUse);
186 }else{
187 db_finalize(&q2);
188 blob_reset(pUse);
189 }
190 }
191 bag_clear(&children);
 
 
 
 
 
 
 
192 }
193
194 /*
195 ** Check to see if the "sym-trunk" tag exists. If not, create it
196 ** and attach it to the very first check-in.
197
--- src/rebuild.c
+++ src/rebuild.c
@@ -121,76 +121,83 @@
121 Bag children;
122 Blob copy;
123 Blob *pUse;
124 int nChild, i, cid;
125
126 while( rid>0 ){
127
128 /* Fix up the "blob.size" field if needed. */
129 if( size!=blob_size(pBase) ){
130 db_multi_exec(
131 "UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid
132 );
133 }
134
135 /* Find all children of artifact rid */
136 db_static_prepare(&q1, "SELECT rid FROM delta WHERE srcid=:rid");
137 db_bind_int(&q1, ":rid", rid);
138 bag_init(&children);
139 while( db_step(&q1)==SQLITE_ROW ){
140 int cid = db_column_int(&q1, 0);
141 if( !bag_find(&bagDone, cid) ){
142 bag_insert(&children, cid);
143 }
144 }
145 nChild = bag_count(&children);
146 db_reset(&q1);
147
148 /* Crosslink the artifact */
149 if( nChild==0 ){
150 pUse = pBase;
151 }else{
152 blob_copy(&copy, pBase);
153 pUse = &copy;
154 }
155 if( zFNameFormat==0 ){
156 /* We are doing "fossil rebuild" */
157 manifest_crosslink(rid, pUse);
158 }else{
159 /* We are doing "fossil deconstruct" */
160 char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
161 char *zFile = mprintf(zFNameFormat, zUuid, zUuid+prefixLength);
162 // blob_write_to_file(pUse,zFile);
163 printf("%d - %s\n", rid, zUuid);
164 free(zFile);
165 free(zUuid);
166 }
167 blob_reset(pUse);
168 rebuild_step_done(rid);
169
170 /* Call all children recursively */
171 rid = 0;
172 for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){
173 Stmt q2;
174 int sz;
175 db_prepare(&q2, "SELECT content, size FROM blob WHERE rid=%d", cid);
176 if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){
177 Blob delta, next;
178 db_ephemeral_blob(&q2, 0, &delta);
179 blob_uncompress(&delta, &delta);
180 blob_delta_apply(pBase, &delta, &next);
181 blob_reset(&delta);
182 db_finalize(&q2);
183 if( i<nChild ){
184 rebuild_step(cid, sz, &next);
185 }else{
186 /* Tail recursion */
187 rid = cid;
188 size = sz;
189 blob_reset(pBase);
190 *pBase = next;
191 }
192 }else{
193 db_finalize(&q2);
194 blob_reset(pBase);
195 }
196 }
197 bag_clear(&children);
198 }
199 }
200
201 /*
202 ** Check to see if the "sym-trunk" tag exists. If not, create it
203 ** and attach it to the very first check-in.
204

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button