Fossil SCM

Improvements to clone performance. Skip cross-linking during the clone the automatically rebuild after the clone. Fixes to rebuild to make sure all artifacts are crosslinked.

drh 2008-05-18 17:18 trunk
Commit 791a513c289ded077964c0c1332e089a0e65587d
Parent 243e02bfbdf5629…
2 files changed +28 -30 +27 -4
~ src/db.c ~ src/rebuild.c
M src/db.c
+28 -30
--- src/db.c
+++ src/db.c
@@ -50,10 +50,11 @@
5050
 ** structure.
5151
 */
5252
 struct Stmt {
5353
 Blob sql; /* The SQL for this statement */
5454
 sqlite3_stmt *pStmt; /* The results of sqlite3_prepare() */
55
+ Stmt *pNext, *pPrev; /* List of all unfinalized statements */
5556
 };
5657
 #endif /* INTERFACE */
5758
5859
 /*
5960
 ** Call this routine when a database error occurs.
@@ -82,10 +83,11 @@
8283
 static int nCommitHook = 0; /* Number of commit hooks */
8384
 static struct sCommitHook {
8485
 int (*xHook)(void); /* Functions to call at db_end_transaction() */
8586
 int sequence; /* Call functions in sequence order */
8687
 } aHook[5];
88
+static Stmt *pAllStmt = 0; /* List of all unfinalized statements */
8789
8890
 /*
8991
 ** This routine is called by the SQLite commit-hook mechanism
9092
 ** just prior to each omit. All this routine does is verify
9193
 ** that nBegin really is zero. That insures that transactions
@@ -163,36 +165,25 @@
163165
 aHook[nCommitHook].sequence = sequence;
164166
 aHook[nCommitHook].xHook = x;
165167
 nCommitHook++;
166168
 }
167169
168
-/*
169
-** Prepare or reprepare the sqlite3 statement from the raw SQL text.
170
-*/
171
-static void reprepare(Stmt *pStmt){
172
- sqlite3_stmt *pNew;
173
- if( sqlite3_prepare(g.db, blob_buffer(&pStmt->sql), -1, &pNew, 0)!=0 ){
174
- db_err("%s\n%s", blob_str(&pStmt->sql), sqlite3_errmsg(g.db));
175
- }
176
- if( pStmt->pStmt ){
177
- sqlite3_transfer_bindings(pStmt->pStmt, pNew);
178
- sqlite3_finalize(pStmt->pStmt);
179
- }
180
- pStmt->pStmt = pNew;
181
-}
182
-
183170
 /*
184171
 ** Prepare a Stmt. Assume that the Stmt is previously uninitialized.
185172
 ** If the input string contains multiple SQL statements, only the first
186173
 ** one is processed. All statements beyond the first are silently ignored.
187174
 */
188175
 int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){
176
+ char *zSql;
189177
 blob_zero(&pStmt->sql);
190178
 blob_vappendf(&pStmt->sql, zFormat, ap);
191179
 va_end(ap);
192
- pStmt->pStmt = 0;
193
- reprepare(pStmt);
180
+ zSql = blob_str(&pStmt->sql);
181
+ if( sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0)!=0 ){
182
+ db_err("%s\n%s", zSql, sqlite3_errmsg(g.db));
183
+ }
184
+ pStmt->pNext = pStmt->pPrev = 0;
194185
 return 0;
195186
 }
196187
 int db_prepare(Stmt *pStmt, const char *zFormat, ...){
197188
 int rc;
198189
 va_list ap;
@@ -205,10 +196,14 @@
205196
 int rc = SQLITE_OK;
206197
 if( blob_size(&pStmt->sql)==0 ){
207198
 va_list ap;
208199
 va_start(ap, zFormat);
209200
 rc = db_vprepare(pStmt, zFormat, ap);
201
+ pStmt->pNext = pAllStmt;
202
+ pStmt->pPrev = 0;
203
+ if( pAllStmt ) pAllStmt->pPrev = pStmt;
204
+ pAllStmt = pStmt;
210205
 va_end(ap);
211206
 }
212207
 return rc;
213208
 }
214209
@@ -258,23 +253,12 @@
258253
 /*
259254
 ** Step the SQL statement. Return either SQLITE_ROW or an error code
260255
 ** or SQLITE_OK if the statement finishes successfully.
261256
 */
262257
 int db_step(Stmt *pStmt){
263
- int rc = SQLITE_OK;
264
- int limit = 3;
265
- while( limit-- ){
266
- rc = sqlite3_step(pStmt->pStmt);
267
- if( rc==SQLITE_ERROR ){
268
- rc = sqlite3_reset(pStmt->pStmt);
269
- }
270
- if( rc==SQLITE_SCHEMA ){
271
- reprepare(pStmt);
272
- }else{
273
- break;
274
- }
275
- }
258
+ int rc;
259
+ rc = sqlite3_step(pStmt->pStmt);
276260
 return rc;
277261
 }
278262
279263
 /*
280264
 ** Reset or finalize a statement.
@@ -287,10 +271,21 @@
287271
 int db_finalize(Stmt *pStmt){
288272
 int rc;
289273
 blob_reset(&pStmt->sql);
290274
 rc = sqlite3_finalize(pStmt->pStmt);
291275
 db_check_result(rc);
276
+ pStmt->pStmt = 0;
277
+ if( pStmt->pNext ){
278
+ pStmt->pNext->pPrev = pStmt->pPrev;
279
+ }
280
+ if( pStmt->pPrev ){
281
+ pStmt->pPrev->pNext = pStmt->pNext;
282
+ }else if( pAllStmt==pStmt ){
283
+ pAllStmt = pStmt->pNext;
284
+ }
285
+ pStmt->pNext = 0;
286
+ pStmt->pPrev = 0;
292287
 return rc;
293288
 }
294289
295290
 /*
296291
 ** Return the rowid of the most recent insert
@@ -714,10 +709,13 @@
714709
 /*
715710
 ** Close the database connection.
716711
 */
717712
 void db_close(void){
718713
 if( g.db==0 ) return;
714
+ while( pAllStmt ){
715
+ db_finalize(pAllStmt);
716
+ }
719717
 g.repositoryOpen = 0;
720718
 g.localOpen = 0;
721719
 g.configOpen = 0;
722720
 sqlite3_close(g.db);
723721
 g.db = 0;
724722
--- src/db.c
+++ src/db.c
@@ -50,10 +50,11 @@
50 ** structure.
51 */
52 struct Stmt {
53 Blob sql; /* The SQL for this statement */
54 sqlite3_stmt *pStmt; /* The results of sqlite3_prepare() */
 
55 };
56 #endif /* INTERFACE */
57
58 /*
59 ** Call this routine when a database error occurs.
@@ -82,10 +83,11 @@
82 static int nCommitHook = 0; /* Number of commit hooks */
83 static struct sCommitHook {
84 int (*xHook)(void); /* Functions to call at db_end_transaction() */
85 int sequence; /* Call functions in sequence order */
86 } aHook[5];
 
87
88 /*
89 ** This routine is called by the SQLite commit-hook mechanism
90 ** just prior to each omit. All this routine does is verify
91 ** that nBegin really is zero. That insures that transactions
@@ -163,36 +165,25 @@
163 aHook[nCommitHook].sequence = sequence;
164 aHook[nCommitHook].xHook = x;
165 nCommitHook++;
166 }
167
168 /*
169 ** Prepare or reprepare the sqlite3 statement from the raw SQL text.
170 */
171 static void reprepare(Stmt *pStmt){
172 sqlite3_stmt *pNew;
173 if( sqlite3_prepare(g.db, blob_buffer(&pStmt->sql), -1, &pNew, 0)!=0 ){
174 db_err("%s\n%s", blob_str(&pStmt->sql), sqlite3_errmsg(g.db));
175 }
176 if( pStmt->pStmt ){
177 sqlite3_transfer_bindings(pStmt->pStmt, pNew);
178 sqlite3_finalize(pStmt->pStmt);
179 }
180 pStmt->pStmt = pNew;
181 }
182
183 /*
184 ** Prepare a Stmt. Assume that the Stmt is previously uninitialized.
185 ** If the input string contains multiple SQL statements, only the first
186 ** one is processed. All statements beyond the first are silently ignored.
187 */
188 int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){
 
189 blob_zero(&pStmt->sql);
190 blob_vappendf(&pStmt->sql, zFormat, ap);
191 va_end(ap);
192 pStmt->pStmt = 0;
193 reprepare(pStmt);
 
 
 
194 return 0;
195 }
196 int db_prepare(Stmt *pStmt, const char *zFormat, ...){
197 int rc;
198 va_list ap;
@@ -205,10 +196,14 @@
205 int rc = SQLITE_OK;
206 if( blob_size(&pStmt->sql)==0 ){
207 va_list ap;
208 va_start(ap, zFormat);
209 rc = db_vprepare(pStmt, zFormat, ap);
 
 
 
 
210 va_end(ap);
211 }
212 return rc;
213 }
214
@@ -258,23 +253,12 @@
258 /*
259 ** Step the SQL statement. Return either SQLITE_ROW or an error code
260 ** or SQLITE_OK if the statement finishes successfully.
261 */
262 int db_step(Stmt *pStmt){
263 int rc = SQLITE_OK;
264 int limit = 3;
265 while( limit-- ){
266 rc = sqlite3_step(pStmt->pStmt);
267 if( rc==SQLITE_ERROR ){
268 rc = sqlite3_reset(pStmt->pStmt);
269 }
270 if( rc==SQLITE_SCHEMA ){
271 reprepare(pStmt);
272 }else{
273 break;
274 }
275 }
276 return rc;
277 }
278
279 /*
280 ** Reset or finalize a statement.
@@ -287,10 +271,21 @@
287 int db_finalize(Stmt *pStmt){
288 int rc;
289 blob_reset(&pStmt->sql);
290 rc = sqlite3_finalize(pStmt->pStmt);
291 db_check_result(rc);
 
 
 
 
 
 
 
 
 
 
 
292 return rc;
293 }
294
295 /*
296 ** Return the rowid of the most recent insert
@@ -714,10 +709,13 @@
714 /*
715 ** Close the database connection.
716 */
717 void db_close(void){
718 if( g.db==0 ) return;
 
 
 
719 g.repositoryOpen = 0;
720 g.localOpen = 0;
721 g.configOpen = 0;
722 sqlite3_close(g.db);
723 g.db = 0;
724
--- src/db.c
+++ src/db.c
@@ -50,10 +50,11 @@
50 ** structure.
51 */
52 struct Stmt {
53 Blob sql; /* The SQL for this statement */
54 sqlite3_stmt *pStmt; /* The results of sqlite3_prepare() */
55 Stmt *pNext, *pPrev; /* List of all unfinalized statements */
56 };
57 #endif /* INTERFACE */
58
59 /*
60 ** Call this routine when a database error occurs.
@@ -82,10 +83,11 @@
83 static int nCommitHook = 0; /* Number of commit hooks */
84 static struct sCommitHook {
85 int (*xHook)(void); /* Functions to call at db_end_transaction() */
86 int sequence; /* Call functions in sequence order */
87 } aHook[5];
88 static Stmt *pAllStmt = 0; /* List of all unfinalized statements */
89
90 /*
91 ** This routine is called by the SQLite commit-hook mechanism
92 ** just prior to each omit. All this routine does is verify
93 ** that nBegin really is zero. That insures that transactions
@@ -163,36 +165,25 @@
165 aHook[nCommitHook].sequence = sequence;
166 aHook[nCommitHook].xHook = x;
167 nCommitHook++;
168 }
169
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170 /*
171 ** Prepare a Stmt. Assume that the Stmt is previously uninitialized.
172 ** If the input string contains multiple SQL statements, only the first
173 ** one is processed. All statements beyond the first are silently ignored.
174 */
175 int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){
176 char *zSql;
177 blob_zero(&pStmt->sql);
178 blob_vappendf(&pStmt->sql, zFormat, ap);
179 va_end(ap);
180 zSql = blob_str(&pStmt->sql);
181 if( sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0)!=0 ){
182 db_err("%s\n%s", zSql, sqlite3_errmsg(g.db));
183 }
184 pStmt->pNext = pStmt->pPrev = 0;
185 return 0;
186 }
187 int db_prepare(Stmt *pStmt, const char *zFormat, ...){
188 int rc;
189 va_list ap;
@@ -205,10 +196,14 @@
196 int rc = SQLITE_OK;
197 if( blob_size(&pStmt->sql)==0 ){
198 va_list ap;
199 va_start(ap, zFormat);
200 rc = db_vprepare(pStmt, zFormat, ap);
201 pStmt->pNext = pAllStmt;
202 pStmt->pPrev = 0;
203 if( pAllStmt ) pAllStmt->pPrev = pStmt;
204 pAllStmt = pStmt;
205 va_end(ap);
206 }
207 return rc;
208 }
209
@@ -258,23 +253,12 @@
253 /*
254 ** Step the SQL statement. Return either SQLITE_ROW or an error code
255 ** or SQLITE_OK if the statement finishes successfully.
256 */
257 int db_step(Stmt *pStmt){
258 int rc;
259 rc = sqlite3_step(pStmt->pStmt);
 
 
 
 
 
 
 
 
 
 
 
260 return rc;
261 }
262
263 /*
264 ** Reset or finalize a statement.
@@ -287,10 +271,21 @@
271 int db_finalize(Stmt *pStmt){
272 int rc;
273 blob_reset(&pStmt->sql);
274 rc = sqlite3_finalize(pStmt->pStmt);
275 db_check_result(rc);
276 pStmt->pStmt = 0;
277 if( pStmt->pNext ){
278 pStmt->pNext->pPrev = pStmt->pPrev;
279 }
280 if( pStmt->pPrev ){
281 pStmt->pPrev->pNext = pStmt->pNext;
282 }else if( pAllStmt==pStmt ){
283 pAllStmt = pStmt->pNext;
284 }
285 pStmt->pNext = 0;
286 pStmt->pPrev = 0;
287 return rc;
288 }
289
290 /*
291 ** Return the rowid of the most recent insert
@@ -714,10 +709,13 @@
709 /*
710 ** Close the database connection.
711 */
712 void db_close(void){
713 if( g.db==0 ) return;
714 while( pAllStmt ){
715 db_finalize(pAllStmt);
716 }
717 g.repositoryOpen = 0;
718 g.localOpen = 0;
719 g.configOpen = 0;
720 sqlite3_close(g.db);
721 g.db = 0;
722
M src/rebuild.c
+27 -4
--- src/rebuild.c
+++ src/rebuild.c
@@ -68,15 +68,18 @@
6868
 ** Variables used for progress information
6969
 */
7070
 static int totalSize; /* Total number of artifacts to process */
7171
 static int processCnt; /* Number processed so far */
7272
 static int ttyOutput; /* Do progress output */
73
+static Bag bagDone; /* Bag of records rebuilt */
7374
7475
 /*
7576
 ** Called after each artifact is processed
7677
 */
77
-static void rebuild_step_done(void){
78
+static void rebuild_step_done(rid){
79
+ assert( bag_find(&bagDone, rid)==0 );
80
+ bag_insert(&bagDone, rid);
7881
 if( ttyOutput ){
7982
 processCnt++;
8083
 printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
8184
 fflush(stdout);
8285
 }
@@ -103,11 +106,14 @@
103106
104107
 /* Find all children of artifact rid */
105108
 db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
106109
 bag_init(&children);
107110
 while( db_step(&q1)==SQLITE_ROW ){
108
- bag_insert(&children, db_column_int(&q1, 0));
111
+ int cid = db_column_int(&q1, 0);
112
+ if( !bag_find(&bagDone, cid) ){
113
+ bag_insert(&children, cid);
114
+ }
109115
 }
110116
 nChild = bag_count(&children);
111117
 db_finalize(&q1);
112118
113119
 /* Crosslink the artifact */
@@ -143,11 +149,11 @@
143149
 db_finalize(&q2);
144150
 blob_reset(pUse);
145151
 }
146152
 }
147153
 bag_clear(&children);
148
- rebuild_step_done();
154
+ rebuild_step_done(rid);
149155
 }
150156
151157
 /*
152158
 ** Core function to rebuild the infomration in the derived tables of a
153159
 ** fossil repository from the blobs. This function is shared between
@@ -163,10 +169,11 @@
163169
 int rebuild_db(int randomize, int doOut){
164170
 Stmt s;
165171
 int errCnt = 0;
166172
 char *zTable;
167173
174
+ bag_init(&bagDone);
168175
 ttyOutput = doOut;
169176
 processCnt = 0;
170177
 db_multi_exec(zSchemaUpdates);
171178
 for(;;){
172179
 zTable = db_text(0,
@@ -204,13 +211,29 @@
204211
 int size = db_column_int(&s, 1);
205212
 if( size>=0 ){
206213
 Blob content;
207214
 content_get(rid, &content);
208215
 rebuild_step(rid, size, &content);
216
+ }
217
+ }
218
+ db_finalize(&s);
219
+ db_prepare(&s,
220
+ "SELECT rid, size FROM blob"
221
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
222
+ );
223
+ while( db_step(&s)==SQLITE_ROW ){
224
+ int rid = db_column_int(&s, 0);
225
+ int size = db_column_int(&s, 1);
226
+ if( size>=0 ){
227
+ if( !bag_find(&bagDone, rid) ){
228
+ Blob content;
229
+ content_get(rid, &content);
230
+ rebuild_step(rid, size, &content);
231
+ }
209232
 }else{
210233
 db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
211
- rebuild_step_done();
234
+ rebuild_step_done(rid);
212235
 }
213236
 }
214237
 db_finalize(&s);
215238
 if( ttyOutput ){
216239
 printf("\n");
217240
--- src/rebuild.c
+++ src/rebuild.c
@@ -68,15 +68,18 @@
68 ** Variables used for progress information
69 */
70 static int totalSize; /* Total number of artifacts to process */
71 static int processCnt; /* Number processed so far */
72 static int ttyOutput; /* Do progress output */
 
73
74 /*
75 ** Called after each artifact is processed
76 */
77 static void rebuild_step_done(void){
 
 
78 if( ttyOutput ){
79 processCnt++;
80 printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
81 fflush(stdout);
82 }
@@ -103,11 +106,14 @@
103
104 /* Find all children of artifact rid */
105 db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
106 bag_init(&children);
107 while( db_step(&q1)==SQLITE_ROW ){
108 bag_insert(&children, db_column_int(&q1, 0));
 
 
 
109 }
110 nChild = bag_count(&children);
111 db_finalize(&q1);
112
113 /* Crosslink the artifact */
@@ -143,11 +149,11 @@
143 db_finalize(&q2);
144 blob_reset(pUse);
145 }
146 }
147 bag_clear(&children);
148 rebuild_step_done();
149 }
150
151 /*
152 ** Core function to rebuild the infomration in the derived tables of a
153 ** fossil repository from the blobs. This function is shared between
@@ -163,10 +169,11 @@
163 int rebuild_db(int randomize, int doOut){
164 Stmt s;
165 int errCnt = 0;
166 char *zTable;
167
 
168 ttyOutput = doOut;
169 processCnt = 0;
170 db_multi_exec(zSchemaUpdates);
171 for(;;){
172 zTable = db_text(0,
@@ -204,13 +211,29 @@
204 int size = db_column_int(&s, 1);
205 if( size>=0 ){
206 Blob content;
207 content_get(rid, &content);
208 rebuild_step(rid, size, &content);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209 }else{
210 db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
211 rebuild_step_done();
212 }
213 }
214 db_finalize(&s);
215 if( ttyOutput ){
216 printf("\n");
217
--- src/rebuild.c
+++ src/rebuild.c
@@ -68,15 +68,18 @@
68 ** Variables used for progress information
69 */
70 static int totalSize; /* Total number of artifacts to process */
71 static int processCnt; /* Number processed so far */
72 static int ttyOutput; /* Do progress output */
73 static Bag bagDone; /* Bag of records rebuilt */
74
75 /*
76 ** Called after each artifact is processed
77 */
78 static void rebuild_step_done(rid){
79 assert( bag_find(&bagDone, rid)==0 );
80 bag_insert(&bagDone, rid);
81 if( ttyOutput ){
82 processCnt++;
83 printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
84 fflush(stdout);
85 }
@@ -103,11 +106,14 @@
106
107 /* Find all children of artifact rid */
108 db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
109 bag_init(&children);
110 while( db_step(&q1)==SQLITE_ROW ){
111 int cid = db_column_int(&q1, 0);
112 if( !bag_find(&bagDone, cid) ){
113 bag_insert(&children, cid);
114 }
115 }
116 nChild = bag_count(&children);
117 db_finalize(&q1);
118
119 /* Crosslink the artifact */
@@ -143,11 +149,11 @@
149 db_finalize(&q2);
150 blob_reset(pUse);
151 }
152 }
153 bag_clear(&children);
154 rebuild_step_done(rid);
155 }
156
157 /*
158 ** Core function to rebuild the infomration in the derived tables of a
159 ** fossil repository from the blobs. This function is shared between
@@ -163,10 +169,11 @@
169 int rebuild_db(int randomize, int doOut){
170 Stmt s;
171 int errCnt = 0;
172 char *zTable;
173
174 bag_init(&bagDone);
175 ttyOutput = doOut;
176 processCnt = 0;
177 db_multi_exec(zSchemaUpdates);
178 for(;;){
179 zTable = db_text(0,
@@ -204,13 +211,29 @@
211 int size = db_column_int(&s, 1);
212 if( size>=0 ){
213 Blob content;
214 content_get(rid, &content);
215 rebuild_step(rid, size, &content);
216 }
217 }
218 db_finalize(&s);
219 db_prepare(&s,
220 "SELECT rid, size FROM blob"
221 " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
222 );
223 while( db_step(&s)==SQLITE_ROW ){
224 int rid = db_column_int(&s, 0);
225 int size = db_column_int(&s, 1);
226 if( size>=0 ){
227 if( !bag_find(&bagDone, rid) ){
228 Blob content;
229 content_get(rid, &content);
230 rebuild_step(rid, size, &content);
231 }
232 }else{
233 db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
234 rebuild_step_done(rid);
235 }
236 }
237 db_finalize(&s);
238 if( ttyOutput ){
239 printf("\n");
240

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button