Fossil SCM

fossil-scm / src / rebuild.c
Blame History Raw 1546 lines
1
/*
2
** Copyright (c) 2007 D. Richard Hipp
3
**
4
** This program is free software; you can redistribute it and/or
5
** modify it under the terms of the Simplified BSD License (also
6
** known as the "2-Clause License" or "FreeBSD License".)
7
8
** This program is distributed in the hope that it will be useful,
9
** but without any warranty; without even the implied warranty of
10
** merchantability or fitness for a particular purpose.
11
**
12
** Author contact information:
13
** [email protected]
14
** http://www.hwaci.com/drh/
15
**
16
*******************************************************************************
17
**
18
** This file contains code used to rebuild the database.
19
*/
20
#include "config.h"
21
#include "rebuild.h"
22
#include <assert.h>
23
#include <errno.h>
24
25
/*
26
** Update the schema as necessary
27
*/
28
static void rebuild_update_schema(void){
29
/* Verify that the PLINK table has a new column added by the
30
** 2014-11-28 schema change. Create it if necessary. This code
31
** can be removed in the future, once all users have upgraded to the
32
** 2014-11-28 or later schema.
33
*/
34
if( !db_table_has_column("repository","plink","baseid") ){
35
db_multi_exec(
36
"ALTER TABLE repository.plink ADD COLUMN baseid;"
37
);
38
}
39
40
/* Verify that the MLINK table has the newer columns added by the
41
** 2015-01-24 schema change. Create them if necessary. This code
42
** can be removed in the future, once all users have upgraded to the
43
** 2015-01-24 or later schema.
44
*/
45
if( !db_table_has_column("repository","mlink","isaux") ){
46
db_begin_transaction();
47
db_multi_exec(
48
"ALTER TABLE repository.mlink ADD COLUMN pmid INTEGER DEFAULT 0;"
49
"ALTER TABLE repository.mlink ADD COLUMN isaux BOOLEAN DEFAULT 0;"
50
);
51
db_end_transaction(0);
52
}
53
54
/* Add the user.mtime column if it is missing. (2011-04-27)
55
*/
56
if( !db_table_has_column("repository", "user", "mtime") ){
57
db_unprotect(PROTECT_ALL);
58
db_multi_exec(
59
"CREATE TEMP TABLE temp_user AS SELECT * FROM user;"
60
"DROP TABLE user;"
61
"CREATE TABLE user(\n"
62
" uid INTEGER PRIMARY KEY,\n"
63
" login TEXT UNIQUE,\n"
64
" pw TEXT,\n"
65
" cap TEXT,\n"
66
" cookie TEXT,\n"
67
" ipaddr TEXT,\n"
68
" cexpire DATETIME,\n"
69
" info TEXT,\n"
70
" mtime DATE,\n"
71
" photo BLOB\n"
72
");"
73
"INSERT OR IGNORE INTO user"
74
" SELECT uid, login, pw, cap, cookie,"
75
" ipaddr, cexpire, info, now(), photo FROM temp_user;"
76
"DROP TABLE temp_user;"
77
);
78
db_protect_pop();
79
}
80
81
/* Add the config.mtime column if it is missing. (2011-04-27)
82
*/
83
if( !db_table_has_column("repository", "config", "mtime") ){
84
db_unprotect(PROTECT_CONFIG);
85
db_multi_exec(
86
"ALTER TABLE config ADD COLUMN mtime INTEGER;"
87
"UPDATE config SET mtime=now();"
88
);
89
db_protect_pop();
90
}
91
92
/* Add the shun.mtime and shun.scom columns if they are missing.
93
** (2011-04-27)
94
*/
95
if( !db_table_has_column("repository", "shun", "mtime") ){
96
db_multi_exec(
97
"ALTER TABLE shun ADD COLUMN mtime INTEGER;"
98
"ALTER TABLE shun ADD COLUMN scom TEXT;"
99
"UPDATE shun SET mtime=now();"
100
);
101
}
102
103
/* Add the reportfmt.mtime column if it is missing. (2011-04-27)
104
*/
105
if( !db_table_has_column("repository", "reportfmt", "mtime") ){
106
static const char zCreateReportFmtTable[] =
107
@ -- An entry in this table describes a database query that generates a
108
@ -- table of tickets.
109
@ --
110
@ CREATE TABLE IF NOT EXISTS reportfmt(
111
@ rn INTEGER PRIMARY KEY, -- Report number
112
@ owner TEXT, -- Owner of this report format (not used)
113
@ title TEXT UNIQUE, -- Title of this report
114
@ mtime INTEGER, -- Time last modified. Seconds since 1970
115
@ cols TEXT, -- A color-key specification
116
@ sqlcode TEXT -- An SQL SELECT statement for this report
117
@ );
118
;
119
db_multi_exec(
120
"CREATE TEMP TABLE old_fmt AS SELECT * FROM reportfmt;"
121
"DROP TABLE reportfmt;"
122
);
123
db_multi_exec("%s", zCreateReportFmtTable/*safe-for-%s*/);
124
db_multi_exec(
125
"INSERT OR IGNORE INTO reportfmt(rn,owner,title,cols,sqlcode,mtime)"
126
" SELECT rn, owner, title, cols, sqlcode, now() FROM old_fmt;"
127
"INSERT OR IGNORE INTO reportfmt(rn,owner,title,cols,sqlcode,mtime)"
128
" SELECT rn, owner, title || ' (' || rn || ')', cols, sqlcode, now()"
129
" FROM old_fmt;"
130
);
131
}
132
133
/* Add the concealed.mtime column if it is missing. (2011-04-27)
134
*/
135
if( !db_table_has_column("repository", "concealed", "mtime") ){
136
db_multi_exec(
137
"ALTER TABLE concealed ADD COLUMN mtime INTEGER;"
138
"UPDATE concealed SET mtime=now();"
139
);
140
}
141
142
/* Do the fossil-2.0 updates to the schema. (2017-02-28)
143
*/
144
rebuild_schema_update_2_0();
145
146
/* Add the user.jx and reportfmt.jx columns if they are missing. (2022-11-18)
147
*/
148
user_update_user_table();
149
report_update_reportfmt_table();
150
}
151
152
/*
153
** Update the repository schema for Fossil version 2.0. (2017-02-28)
154
** (1) Change the CHECK constraint on BLOB.UUID so that the length
155
** is greater than or equal to 40, not exactly equal to 40.
156
*/
157
void rebuild_schema_update_2_0(void){
158
char *z = db_text(0, "SELECT sql FROM repository.sqlite_schema"
159
" WHERE name='blob'");
160
if( z ){
161
/* Search for: length(uuid)==40
162
** 0123456789 12345 */
163
int i;
164
for(i=10; z[i]; i++){
165
if( z[i]=='=' && strncmp(&z[i-6],"(uuid)==40",10)==0 ){
166
int rc = 0;
167
z[i] = '>';
168
sqlite3_db_config(g.db, SQLITE_DBCONFIG_DEFENSIVE, 0, &rc);
169
db_multi_exec(
170
"PRAGMA writable_schema=ON;"
171
"UPDATE repository.sqlite_schema SET sql=%Q WHERE name LIKE 'blob';"
172
"PRAGMA writable_schema=OFF;",
173
z
174
);
175
sqlite3_db_config(g.db, SQLITE_DBCONFIG_DEFENSIVE, 1, &rc);
176
break;
177
}
178
}
179
fossil_free(z);
180
}
181
db_multi_exec(
182
"CREATE VIEW IF NOT EXISTS "
183
" repository.artifact(rid,rcvid,size,atype,srcid,hash,content) AS "
184
" SELECT blob.rid,rcvid,size,1,srcid,uuid,content"
185
" FROM blob LEFT JOIN delta ON (blob.rid=delta.rid);"
186
);
187
}
188
189
/*
190
** Variables used to store state information about an on-going "rebuild"
191
** or "deconstruct".
192
*/
193
static int totalSize; /* Total number of artifacts to process */
194
static int processCnt; /* Number processed so far */
195
static int ttyOutput; /* Do progress output */
196
static Bag bagDone = Bag_INIT; /* Bag of records rebuilt */
197
198
static char *zFNameFormat; /* Format string for filenames on deconstruct */
199
static int cchFNamePrefix; /* Length of directory prefix in zFNameFormat */
200
static const char *zDestDir;/* Destination directory on deconstruct */
201
static int prefixLength; /* Length of directory prefix for deconstruct */
202
static int fKeepRid1; /* Flag to preserve RID=1 on de- and reconstruct */
203
204
205
/*
206
** Draw the percent-complete message.
207
** The input is actually the permill complete.
208
*/
209
static void percent_complete(int permill){
210
static int lastOutput = -1;
211
if( permill>lastOutput ){
212
fossil_print(" %d.%d%% complete...\r", permill/10, permill%10);
213
fflush(stdout);
214
lastOutput = permill;
215
}
216
}
217
218
/*
219
** Frees rebuild-level cached state. Intended only to be called by the
220
** app-level atexit() handler.
221
*/
222
void rebuild_clear_cache(){
223
bag_clear(&bagDone);
224
}
225
226
/*
227
** Called after each artifact is processed
228
*/
229
static void rebuild_step_done(int rid){
230
/* assert( bag_find(&bagDone, rid)==0 ); */
231
bag_insert(&bagDone, rid);
232
if( ttyOutput ){
233
processCnt++;
234
if (!g.fQuiet && totalSize>0) {
235
percent_complete((processCnt*1000)/totalSize);
236
}
237
}
238
}
239
240
/*
241
** Rebuild cross-referencing information for the artifact
242
** rid with content pBase and all of its descendants. This
243
** routine clears the content buffer before returning.
244
**
245
** If the zFNameFormat variable is set, then this routine is
246
** called to run "fossil deconstruct" instead of the usual
247
** "fossil rebuild". In that case, instead of rebuilding the
248
** cross-referencing information, write the file content out
249
** to the appropriate directory.
250
**
251
** In both cases, this routine automatically recurses to process
252
** other artifacts that are deltas off of the current artifact.
253
** This is the most efficient way to extract all of the original
254
** artifact content from the Fossil repository.
255
*/
256
static void rebuild_step(int rid, int size, Blob *pBase){
257
static Stmt q1;
258
Bag children;
259
Blob copy;
260
Blob *pUse;
261
int nChild, i, cid;
262
263
while( rid>0 ){
264
265
/* Fix up the "blob.size" field if needed. */
266
if( size!=(int)blob_size(pBase) ){
267
db_multi_exec(
268
"UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid
269
);
270
}
271
272
/* Find all children of artifact rid */
273
db_static_prepare(&q1, "SELECT rid FROM delta WHERE srcid=:rid");
274
db_bind_int(&q1, ":rid", rid);
275
bag_init(&children);
276
while( db_step(&q1)==SQLITE_ROW ){
277
int cid = db_column_int(&q1, 0);
278
if( !bag_find(&bagDone, cid) ){
279
bag_insert(&children, cid);
280
}
281
}
282
nChild = bag_count(&children);
283
db_reset(&q1);
284
285
/* Crosslink the artifact */
286
if( nChild==0 ){
287
pUse = pBase;
288
}else{
289
blob_copy(&copy, pBase);
290
pUse = &copy;
291
}
292
if( zFNameFormat==0 ){
293
/* We are doing "fossil rebuild" */
294
manifest_crosslink(rid, pUse, MC_NONE);
295
}else{
296
/* We are doing "fossil deconstruct" */
297
char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
298
char *zFile = mprintf(zFNameFormat /*works-like:"%s:%s"*/,
299
zUuid, zUuid+prefixLength);
300
blob_write_to_file(pUse,zFile);
301
if( rid==1 && fKeepRid1!=0 ){
302
char *zFnDotRid1 = mprintf("%s/.rid1", zDestDir);
303
char *zFnRid1 = zFile + cchFNamePrefix + 1; /* Skip directory slash */
304
Blob bFileContents = empty_blob;
305
blob_appendf(&bFileContents,
306
"# The file holding the artifact with RID=1\n"
307
"%s\n", zFnRid1);
308
blob_write_to_file(&bFileContents, zFnDotRid1);
309
blob_reset(&bFileContents);
310
free(zFnDotRid1);
311
}
312
free(zFile);
313
free(zUuid);
314
blob_reset(pUse);
315
}
316
assert( blob_is_reset(pUse) );
317
rebuild_step_done(rid);
318
319
/* Call all children recursively */
320
rid = 0;
321
for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){
322
static Stmt q2;
323
int sz;
324
db_static_prepare(&q2, "SELECT content, size FROM blob WHERE rid=:rid");
325
db_bind_int(&q2, ":rid", cid);
326
if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){
327
Blob delta, next;
328
db_ephemeral_blob(&q2, 0, &delta);
329
blob_uncompress(&delta, &delta);
330
blob_delta_apply(pBase, &delta, &next);
331
blob_reset(&delta);
332
db_reset(&q2);
333
if( i<nChild ){
334
rebuild_step(cid, sz, &next);
335
}else{
336
/* Tail recursion */
337
rid = cid;
338
size = sz;
339
blob_reset(pBase);
340
*pBase = next;
341
}
342
}else{
343
db_reset(&q2);
344
blob_reset(pBase);
345
}
346
}
347
bag_clear(&children);
348
}
349
}
350
351
/*
352
** Check to see if the "sym-trunk" tag exists. If not, create it
353
** and attach it to the very first check-in.
354
*/
355
static void rebuild_tag_trunk(void){
356
const char *zMainBranch = db_main_branch();
357
int tagid = db_int(0, "SELECT 1 FROM tag WHERE tagname='sym-%q'",zMainBranch);
358
int rid;
359
char *zUuid;
360
361
if( tagid>0 ) return;
362
rid = db_int(0, "SELECT pid FROM plink AS x WHERE NOT EXISTS("
363
" SELECT 1 FROM plink WHERE cid=x.pid)");
364
if( rid==0 ) return;
365
366
/* Add the trunk tag to the root of the whole tree */
367
zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
368
if( zUuid==0 ) return;
369
tag_add_artifact("sym-", zMainBranch, zUuid, 0, 2, 0, 0);
370
tag_add_artifact("", "branch", zUuid, zMainBranch, 2, 0, 0);
371
}
372
373
/*
374
** Core function to rebuild the information in the derived tables of a
375
** fossil repository from the blobs. This function is shared between
376
** 'rebuild_database' ('rebuild') and 'reconstruct_cmd'
377
** ('reconstruct'), both of which have to regenerate this information
378
** from scratch.
379
*/
380
int rebuild_db(int doOut, int doClustering){
381
Stmt s, q;
382
int errCnt = 0;
383
int incrSize;
384
Blob sql;
385
386
bag_clear(&bagDone);
387
ttyOutput = doOut;
388
processCnt = 0;
389
if (ttyOutput && !g.fQuiet) {
390
percent_complete(0);
391
}
392
manifest_disable_event_triggers();
393
rebuild_update_schema();
394
blob_init(&sql, 0, 0);
395
db_unprotect(PROTECT_ALL);
396
#ifndef SQLITE_PREPARE_DONT_LOG
397
g.dbIgnoreErrors++;
398
#endif
399
db_prepare(&q,
400
"SELECT name FROM pragma_table_list /*scan*/"
401
" WHERE schema='repository' AND type IN ('table','virtual')"
402
" AND name NOT IN ('admin_log', 'blob','delta','rcvfrom','user','alias',"
403
"'config','shun','private','reportfmt',"
404
"'concealed','accesslog','modreq',"
405
"'purgeevent','purgeitem','unversioned',"
406
"'subscriber','pending_alert','chat')"
407
" AND name NOT GLOB 'sqlite_*'"
408
" AND name NOT GLOB 'fx_*';"
409
);
410
while( db_step(&q)==SQLITE_ROW ){
411
blob_appendf(&sql, "DROP TABLE IF EXISTS \"%w\";\n", db_column_text(&q,0));
412
}
413
db_finalize(&q);
414
#ifndef SQLITE_PREPARE_DONT_LOG
415
g.dbIgnoreErrors--;
416
#endif
417
db_multi_exec("%s", blob_str(&sql)/*safe-for-%s*/);
418
blob_reset(&sql);
419
db_multi_exec("%s", zRepositorySchema2/*safe-for-%s*/);
420
ticket_create_table(0);
421
shun_artifacts();
422
423
db_multi_exec(
424
"INSERT INTO unclustered"
425
" SELECT rid FROM blob EXCEPT SELECT rid FROM private"
426
);
427
db_multi_exec(
428
"DELETE FROM unclustered"
429
" WHERE rid IN (SELECT rid FROM shun JOIN blob USING(uuid))"
430
);
431
db_multi_exec(
432
"DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')"
433
);
434
db_multi_exec(
435
"UPDATE user SET mtime=strftime('%%s','now') WHERE mtime IS NULL"
436
);
437
438
/* The following should be count(*) instead of max(rid). max(rid) is
439
** an adequate approximation, however, and is much faster for large
440
** repositories. */
441
totalSize = db_int(0, "SELECT max(rid) FROM blob");
442
incrSize = totalSize/100;
443
totalSize += incrSize*2;
444
db_prepare(&s,
445
"SELECT rid, size FROM blob /*scan*/"
446
" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
447
" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
448
);
449
manifest_crosslink_begin();
450
while( db_step(&s)==SQLITE_ROW ){
451
int rid = db_column_int(&s, 0);
452
int size = db_column_int(&s, 1);
453
if( size>=0 ){
454
Blob content;
455
content_get(rid, &content);
456
rebuild_step(rid, size, &content);
457
}
458
}
459
db_finalize(&s);
460
db_prepare(&s,
461
"SELECT rid, size FROM blob"
462
" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
463
);
464
while( db_step(&s)==SQLITE_ROW ){
465
int rid = db_column_int(&s, 0);
466
int size = db_column_int(&s, 1);
467
if( size>=0 ){
468
if( !bag_find(&bagDone, rid) ){
469
Blob content;
470
content_get(rid, &content);
471
rebuild_step(rid, size, &content);
472
}
473
}else{
474
db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
475
rebuild_step_done(rid);
476
}
477
}
478
db_finalize(&s);
479
manifest_crosslink_end(MC_NONE);
480
rebuild_tag_trunk();
481
if( ttyOutput && !g.fQuiet && totalSize>0 ){
482
processCnt += incrSize;
483
percent_complete((processCnt*1000)/totalSize);
484
}
485
if( doClustering ) create_cluster();
486
if( ttyOutput && !g.fQuiet && totalSize>0 ){
487
processCnt += incrSize;
488
percent_complete((processCnt*1000)/totalSize);
489
}
490
if(!g.fQuiet && ttyOutput ){
491
percent_complete(1000);
492
fossil_print("\n");
493
}
494
db_protect_pop();
495
return errCnt;
496
}
497
498
/*
499
** Number of neighbors to search
500
*/
501
#define N_NEIGHBOR 5
502
503
/*
504
** Attempt to convert more full-text blobs into delta-blobs for
505
** storage efficiency. Return the number of bytes of storage space
506
** saved.
507
*/
508
i64 extra_deltification(int *pnDelta){
509
Stmt q;
510
int aPrev[N_NEIGHBOR];
511
int nPrev;
512
int rid;
513
int prevfnid, fnid;
514
int nDelta = 0;
515
i64 nByte = 0;
516
int nSaved;
517
db_begin_transaction();
518
519
/* Look for manifests that have not been deltaed and try to make them
520
** children of one of the 5 chronologically subsequent check-ins
521
*/
522
db_prepare(&q,
523
"SELECT rid FROM event, blob"
524
" WHERE blob.rid=event.objid"
525
" AND event.type='ci'"
526
" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
527
" ORDER BY event.mtime DESC"
528
);
529
nPrev = 0;
530
while( db_step(&q)==SQLITE_ROW ){
531
rid = db_column_int(&q, 0);
532
if( nPrev>0 ){
533
nSaved = content_deltify(rid, aPrev, nPrev, 0);
534
if( nSaved>0 ){
535
nDelta++;
536
nByte += nSaved;
537
}
538
}
539
if( nPrev<N_NEIGHBOR ){
540
aPrev[nPrev++] = rid;
541
}else{
542
int i;
543
for(i=0; i<N_NEIGHBOR-1; i++) aPrev[i] = aPrev[i+1];
544
aPrev[N_NEIGHBOR-1] = rid;
545
}
546
}
547
db_finalize(&q);
548
549
/* For individual files that have not been deltaed, try to find
550
** a parent which is an undeltaed file with the same name in a
551
** more recent branch.
552
*/
553
db_prepare(&q,
554
"SELECT DISTINCT blob.rid, mlink.fnid FROM blob, mlink, plink"
555
" WHERE NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
556
" AND mlink.fid=blob.rid"
557
" AND mlink.mid=plink.cid"
558
" AND plink.cid=mlink.mid"
559
" ORDER BY mlink.fnid, plink.mtime DESC"
560
);
561
prevfnid = 0;
562
while( db_step(&q)==SQLITE_ROW ){
563
rid = db_column_int(&q, 0);
564
fnid = db_column_int(&q, 1);
565
if( fnid!=prevfnid ) nPrev = 0;
566
prevfnid = fnid;
567
if( nPrev>0 ){
568
nSaved = content_deltify(rid, aPrev, nPrev, 0);
569
if( nSaved>0 ){
570
nDelta++;
571
nByte += nSaved;
572
}
573
}
574
if( nPrev<N_NEIGHBOR ){
575
aPrev[nPrev++] = rid;
576
}else{
577
int i;
578
for(i=0; i<N_NEIGHBOR-1; i++) aPrev[i] = aPrev[i+1];
579
aPrev[N_NEIGHBOR-1] = rid;
580
}
581
}
582
db_finalize(&q);
583
584
db_end_transaction(0);
585
if( pnDelta!=0 ) *pnDelta = nDelta;
586
return nByte;
587
}
588
589
590
/* Reconstruct the private table. The private table contains the rid
591
** of every manifest that is tagged with "private" and every file that
592
** is not used by a manifest that is not private.
593
*/
594
static void reconstruct_private_table(void){
595
db_multi_exec(
596
"CREATE TEMP TABLE private_ckin(rid INTEGER PRIMARY KEY);"
597
"INSERT INTO private_ckin "
598
" SELECT rid FROM tagxref WHERE tagid=%d AND tagtype>0;"
599
"INSERT OR IGNORE INTO private"
600
" SELECT fid FROM mlink"
601
" EXCEPT SELECT fid FROM mlink WHERE mid NOT IN private_ckin;"
602
"INSERT OR IGNORE INTO private SELECT rid FROM private_ckin;"
603
"DROP TABLE private_ckin;", TAG_PRIVATE
604
);
605
fix_private_blob_dependencies(0);
606
}
607
608
/*
609
** COMMAND: repack
610
**
611
** Usage: %fossil repack ?REPOSITORY?
612
**
613
** Perform extra delta-compression to try to minimize the size of the
614
** repository. This command is simply a short-hand for:
615
**
616
** fossil rebuild --compress-only
617
**
618
** The name for this command is stolen from the "git repack" command that
619
** does approximately the same thing in Git.
620
*/
621
void repack_command(void){
622
i64 nByte = 0;
623
int nDelta = 0;
624
int runVacuum = 0;
625
verify_all_options();
626
if( g.argc==3 ){
627
db_open_repository(g.argv[2]);
628
}else if( g.argc==2 ){
629
db_find_and_open_repository(OPEN_ANY_SCHEMA, 0);
630
if( g.argc!=2 ){
631
usage("?REPOSITORY-FILENAME?");
632
}
633
db_close(1);
634
db_open_repository(g.zRepositoryName);
635
}else{
636
usage("?REPOSITORY-FILENAME?");
637
}
638
db_unprotect(PROTECT_ALL);
639
nByte = extra_deltification(&nDelta);
640
if( nDelta>0 ){
641
if( nDelta==1 ){
642
fossil_print("1 new delta saves %,lld bytes\n", nByte);
643
}else{
644
fossil_print("%d new deltas save %,lld bytes\n", nDelta, nByte);
645
}
646
runVacuum = 1;
647
}else{
648
fossil_print("no new compression opportunities found\n");
649
runVacuum = db_int(0, "PRAGMA repository.freelist_count")>0;
650
}
651
if( runVacuum ){
652
fossil_print("Vacuuming the database... "); fflush(stdout);
653
db_multi_exec("VACUUM");
654
fossil_print("done\n");
655
}
656
}
657
658
659
/*
660
** COMMAND: rebuild
661
**
662
** Usage: %fossil rebuild ?REPOSITORY? ?OPTIONS?
663
**
664
** Reconstruct the named repository database from the core
665
** records. Run this command after updating the fossil
666
** executable in a way that changes the database schema.
667
**
668
** Options:
669
** --analyze Run ANALYZE on the database after rebuilding
670
** --cluster Compute clusters for unclustered artifacts
671
** --compress Strive to make the database as small as possible
672
** --compress-only Skip the rebuilding step. Do --compress only
673
** --force Force the rebuild to complete even if errors are seen
674
** --ifneeded Only do the rebuild if it would change the schema version
675
** --index Always add in the full-text search index
676
** --noverify Skip the verification of changes to the BLOB table
677
** --noindex Always omit the full-text search index
678
** --pagesize N Set the database pagesize to N (512..65536, power of 2)
679
** --quiet Only show output if there are errors
680
** --stats Show artifact statistics after rebuilding
681
** --vacuum Run VACUUM on the database after rebuilding
682
** --wal Set Write-Ahead-Log journalling mode on the database
683
*/
684
void rebuild_database(void){
685
int forceFlag;
686
int errCnt = 0;
687
int omitVerify;
688
int doClustering;
689
const char *zPagesize;
690
int newPagesize = 0;
691
int activateWal;
692
int runVacuum;
693
int runDeanalyze;
694
int runAnalyze;
695
int runCompress;
696
int showStats;
697
int runReindex;
698
int optNoIndex;
699
int optIndex;
700
int optIfNeeded;
701
int compressOnlyFlag;
702
703
omitVerify = find_option("noverify",0,0)!=0;
704
forceFlag = find_option("force","f",0)!=0;
705
doClustering = find_option("cluster", 0, 0)!=0;
706
runVacuum = find_option("vacuum",0,0)!=0;
707
runDeanalyze = find_option("deanalyze",0,0)!=0; /* Deprecated */
708
runAnalyze = find_option("analyze",0,0)!=0;
709
runCompress = find_option("compress",0,0)!=0;
710
zPagesize = find_option("pagesize",0,1);
711
showStats = find_option("stats",0,0)!=0;
712
optIndex = find_option("index",0,0)!=0;
713
optNoIndex = find_option("noindex",0,0)!=0;
714
optIfNeeded = find_option("ifneeded",0,0)!=0;
715
compressOnlyFlag = find_option("compress-only",0,0)!=0;
716
if( compressOnlyFlag ) runCompress = 1;
717
if( zPagesize ){
718
newPagesize = atoi(zPagesize);
719
if( newPagesize<512 || newPagesize>65536
720
|| (newPagesize&(newPagesize-1))!=0
721
){
722
fossil_fatal("page size must be a power of two between 512 and 65536");
723
}
724
}
725
activateWal = find_option("wal",0,0)!=0;
726
if( g.argc==3 ){
727
db_open_repository(g.argv[2]);
728
}else{
729
db_find_and_open_repository(OPEN_ANY_SCHEMA, 0);
730
if( g.argc!=2 ){
731
usage("?REPOSITORY-FILENAME?");
732
}
733
db_close(1);
734
db_open_repository(g.zRepositoryName);
735
}
736
runReindex = search_index_exists() && !compressOnlyFlag;
737
if( optIndex ) runReindex = 1;
738
if( optNoIndex ) runReindex = 0;
739
if( optIfNeeded && fossil_strcmp(db_get("aux-schema",""),AUX_SCHEMA_MAX)==0 ){
740
return;
741
}
742
743
/* We should be done with options.. */
744
verify_all_options();
745
746
db_begin_transaction();
747
db_unprotect(PROTECT_ALL);
748
if( !compressOnlyFlag ){
749
search_drop_index();
750
ttyOutput = 1;
751
errCnt = rebuild_db(1, doClustering);
752
reconstruct_private_table();
753
}
754
db_multi_exec(
755
"REPLACE INTO config(name,value,mtime) VALUES('content-schema',%Q,now());"
756
"REPLACE INTO config(name,value,mtime) VALUES('aux-schema',%Q,now());"
757
"REPLACE INTO config(name,value,mtime) VALUES('rebuilt',%Q,now());",
758
CONTENT_SCHEMA, AUX_SCHEMA_MAX, get_version()
759
);
760
if( errCnt && !forceFlag ){
761
fossil_print(
762
"%d errors. Rolling back changes. Use --force to force a commit.\n",
763
errCnt
764
);
765
db_end_transaction(1);
766
}else{
767
if( runCompress ){
768
i64 nByte = 0;
769
int nDelta = 0;
770
fossil_print("Extra delta compression... "); fflush(stdout);
771
nByte = extra_deltification(&nDelta);
772
if( nDelta>0 ){
773
if( nDelta==1 ){
774
fossil_print("1 new delta saves %,lld bytes", nByte);
775
}else{
776
fossil_print("%d new deltas save %,lld bytes", nDelta, nByte);
777
}
778
runVacuum = 1;
779
}else{
780
fossil_print("none found");
781
}
782
fflush(stdout);
783
}
784
if( omitVerify ) verify_cancel();
785
db_end_transaction(0);
786
if( runCompress ) fossil_print("\n");
787
db_close(0);
788
db_open_repository(g.zRepositoryName);
789
if( newPagesize ){
790
db_multi_exec("PRAGMA page_size=%d", newPagesize);
791
runVacuum = 1;
792
}
793
if( runDeanalyze ){
794
db_multi_exec("DROP TABLE IF EXISTS sqlite_stat1;"
795
"DROP TABLE IF EXISTS sqlite_stat3;"
796
"DROP TABLE IF EXISTS sqlite_stat4;");
797
}
798
if( runAnalyze ){
799
fossil_print("Analyzing the database... "); fflush(stdout);
800
db_multi_exec("ANALYZE;");
801
fossil_print("done\n");
802
}
803
if( runVacuum ){
804
fossil_print("Vacuuming the database... "); fflush(stdout);
805
db_multi_exec("VACUUM");
806
fossil_print("done\n");
807
}
808
if( activateWal ){
809
db_multi_exec("PRAGMA journal_mode=WAL;");
810
}
811
}
812
if( runReindex ) search_rebuild_index();
813
db_protect_pop();
814
if( showStats ){
815
static const struct { int idx; const char *zLabel; } aStat[] = {
816
{ CFTYPE_ANY, "Artifacts:" },
817
{ CFTYPE_MANIFEST, "Manifests:" },
818
{ CFTYPE_CLUSTER, "Clusters:" },
819
{ CFTYPE_CONTROL, "Tags:" },
820
{ CFTYPE_WIKI, "Wikis:" },
821
{ CFTYPE_TICKET, "Tickets:" },
822
{ CFTYPE_ATTACHMENT,"Attachments:" },
823
{ CFTYPE_EVENT, "Events:" },
824
};
825
int i;
826
int subtotal = 0;
827
for(i=0; i<count(aStat); i++){
828
int k = aStat[i].idx;
829
fossil_print("%-15s %6d\n", aStat[i].zLabel, g.parseCnt[k]);
830
if( k>0 ) subtotal += g.parseCnt[k];
831
}
832
fossil_print("%-15s %6d\n", "Other:", g.parseCnt[CFTYPE_ANY] - subtotal);
833
}
834
}
835
836
/*
837
** COMMAND: detach*
838
**
839
** Usage: %fossil detach ?REPOSITORY?
840
**
841
** Change the project-code and make other changes to REPOSITORY so that
842
** it becomes a new and distinct child project. After being detached,
843
** REPOSITORY will not longer be able to push and pull from other clones
844
** of the original project. However REPOSITORY will still be able to pull
845
** from those other clones using the --from-parent-project option of the
846
** "fossil pull" command.
847
**
848
** This is an experts-only command. You should not use this command unless
849
** you fully understand what you are doing.
850
**
851
** The original use-case for this command was to create test repositories
852
** from real-world working repositories that could be safely altered by
853
** making strange commits or other changes, without having to worry that
854
** those test changes would leak back into the original project via an
855
** accidental auto-sync.
856
*/
857
void test_detach_cmd(void){
858
const char *zXfer[] = {
859
"project-name", "parent-project-name",
860
"project-code", "parent-project-code",
861
"last-sync-url", "parent-project-url",
862
"last-sync-pw", "parent-project-pw"
863
};
864
int i;
865
Blob ans;
866
char cReply;
867
db_find_and_open_repository(0, 2);
868
prompt_user("This change will be difficult to undo. Are you sure (y/N)? ",
869
&ans);
870
cReply = blob_str(&ans)[0];
871
if( cReply!='y' && cReply!='Y' ) return;
872
db_begin_transaction();
873
db_unprotect(PROTECT_CONFIG);
874
for(i=0; i<ArraySize(zXfer)-1; i+=2 ){
875
db_multi_exec(
876
"REPLACE INTO config(name,value,mtime)"
877
" SELECT %Q, value, now() FROM config WHERE name=%Q",
878
zXfer[i+1], zXfer[i]
879
);
880
}
881
db_multi_exec(
882
"DELETE FROM config WHERE name IN"
883
"(WITH pattern(x) AS (VALUES"
884
" ('baseurl:*'),"
885
" ('cert:*'),"
886
" ('ckout:*'),"
887
" ('gitpush:*'),"
888
" ('http-auth:*'),"
889
" ('last-sync-*'),"
890
" ('link:*'),"
891
" ('login-group-*'),"
892
" ('peer-*'),"
893
" ('subrepo:*'),"
894
" ('sync-*'),"
895
" ('syncfrom:*'),"
896
" ('syncwith:*'),"
897
" ('ssl-*')"
898
") SELECT name FROM config, pattern WHERE name GLOB x);"
899
"UPDATE config SET value=lower(hex(randomblob(20)))"
900
" WHERE name='project-code';"
901
"UPDATE config SET value='detached-' || value"
902
" WHERE name='project-name' AND value NOT GLOB 'detached-*';"
903
);
904
db_protect_pop();
905
db_end_transaction(0);
906
fossil_print("New project code: %s\n", db_get("project-code",""));
907
}
908
909
/*
910
** COMMAND: test-create-clusters
911
**
912
** Create clusters for all unclustered artifacts if the number of unclustered
913
** artifacts exceeds the current clustering threshold.
914
*/
915
void test_createcluster_cmd(void){
916
if( g.argc==3 ){
917
db_open_repository(g.argv[2]);
918
}else{
919
db_find_and_open_repository(0, 0);
920
if( g.argc!=2 ){
921
usage("?REPOSITORY-FILENAME?");
922
}
923
db_close(1);
924
db_open_repository(g.zRepositoryName);
925
}
926
db_begin_transaction();
927
create_cluster();
928
db_end_transaction(0);
929
}
930
931
/*
932
** COMMAND: test-clusters
933
**
934
** Verify that all non-private and non-shunned artifacts are accessible
935
** through the cluster chain.
936
*/
937
void test_clusters_cmd(void){
938
Bag pending;
939
Stmt q;
940
int n;
941
942
db_find_and_open_repository(0, 2);
943
bag_init(&pending);
944
db_multi_exec(
945
"CREATE TEMP TABLE xdone(x INTEGER PRIMARY KEY);"
946
"INSERT INTO xdone SELECT rid FROM unclustered;"
947
"INSERT OR IGNORE INTO xdone SELECT rid FROM private;"
948
"INSERT OR IGNORE INTO xdone"
949
" SELECT blob.rid FROM shun JOIN blob USING(uuid);"
950
);
951
db_prepare(&q,
952
"SELECT rid FROM unclustered WHERE rid IN"
953
" (SELECT rid FROM tagxref WHERE tagid=%d)", TAG_CLUSTER
954
);
955
while( db_step(&q)==SQLITE_ROW ){
956
bag_insert(&pending, db_column_int(&q, 0));
957
}
958
db_finalize(&q);
959
while( bag_count(&pending)>0 ){
960
Manifest *p;
961
int rid = bag_first(&pending);
962
int i;
963
964
bag_remove(&pending, rid);
965
p = manifest_get(rid, CFTYPE_CLUSTER, 0);
966
if( p==0 ){
967
fossil_fatal("bad cluster: rid=%d", rid);
968
}
969
for(i=0; i<p->nCChild; i++){
970
const char *zUuid = p->azCChild[i];
971
int crid = name_to_rid(zUuid);
972
if( crid==0 ){
973
fossil_warning("cluster (rid=%d) references unknown artifact %s",
974
rid, zUuid);
975
continue;
976
}
977
db_multi_exec("INSERT OR IGNORE INTO xdone VALUES(%d)", crid);
978
if( db_exists("SELECT 1 FROM tagxref WHERE tagid=%d AND rid=%d",
979
TAG_CLUSTER, crid) ){
980
bag_insert(&pending, crid);
981
}
982
}
983
manifest_destroy(p);
984
}
985
n = db_int(0, "SELECT count(*) FROM /*scan*/"
986
" (SELECT rid FROM blob EXCEPT SELECT x FROM xdone)");
987
if( n==0 ){
988
fossil_print("all artifacts reachable through clusters\n");
989
}else{
990
fossil_print("%d unreachable artifacts:\n", n);
991
db_prepare(&q, "SELECT rid, uuid FROM blob WHERE rid NOT IN xdone");
992
while( db_step(&q)==SQLITE_ROW ){
993
fossil_print(" %3d %s\n", db_column_int(&q,0), db_column_text(&q,1));
994
}
995
db_finalize(&q);
996
}
997
}
998
999
/*
1000
** COMMAND: scrub*
1001
**
1002
** Usage: %fossil scrub ?OPTIONS? ?REPOSITORY?
1003
**
1004
** The command removes sensitive information (such as passwords) from a
1005
** repository so that the repository can be sent to an untrusted reader.
1006
**
1007
** By default, only passwords are removed. However, if the --verily option
1008
** is added, then private branches, concealed email addresses, IP
1009
** addresses of correspondents, and similar privacy-sensitive fields
1010
** are also purged. If the --private option is used, then only private
1011
** branches are removed and all other information is left intact.
1012
**
1013
** This command permanently deletes the scrubbed information. THE EFFECTS
1014
** OF THIS COMMAND ARE IRREVERSIBLE. USE WITH CAUTION!
1015
**
1016
** The user is prompted to confirm the scrub unless the --force option
1017
** is used.
1018
**
1019
** Options:
1020
** --force Do not prompt for confirmation
1021
** --private Only private branches are removed from the repository
1022
** --verily Scrub real thoroughly (see above)
1023
*/
1024
void scrub_cmd(void){
1025
int bVerily = find_option("verily",0,0)!=0;
1026
int bForce = find_option("force", "f", 0)!=0;
1027
int privateOnly = find_option("private",0,0)!=0;
1028
int bNeedRebuild = 0;
1029
db_find_and_open_repository(OPEN_ANY_SCHEMA, 2);
1030
db_close(1);
1031
db_open_repository(g.zRepositoryName);
1032
1033
/* We should be done with options.. */
1034
verify_all_options();
1035
1036
if( !bForce ){
1037
Blob ans;
1038
char cReply;
1039
prompt_user(
1040
"Scrubbing the repository will permanently delete information.\n"
1041
"Changes cannot be undone. Continue (y/N)? ", &ans);
1042
cReply = blob_str(&ans)[0];
1043
if( cReply!='y' && cReply!='Y' ){
1044
fossil_exit(1);
1045
}
1046
}
1047
db_begin_transaction();
1048
if( privateOnly || bVerily ){
1049
bNeedRebuild = db_exists("SELECT 1 FROM private");
1050
delete_private_content();
1051
}
1052
if( !privateOnly ){
1053
db_unprotect(PROTECT_ALL);
1054
db_multi_exec(
1055
"PRAGMA secure_delete=ON;"
1056
"UPDATE user SET pw='';"
1057
"DELETE FROM config WHERE name IN"
1058
"(WITH pattern(x) AS (VALUES"
1059
" ('baseurl:*'),"
1060
" ('cert:*'),"
1061
" ('ckout:*'),"
1062
" ('draft[1-9]-*'),"
1063
" ('gitpush:*'),"
1064
" ('http-auth:*'),"
1065
" ('last-sync-*'),"
1066
" ('link:*'),"
1067
" ('login-group-*'),"
1068
" ('parent-project-*'),"
1069
" ('peer-*'),"
1070
" ('skin:*'),"
1071
" ('subrepo:*'),"
1072
" ('sync-*'),"
1073
" ('syncfrom:*'),"
1074
" ('syncwith:*'),"
1075
" ('ssl-*')"
1076
") SELECT name FROM config, pattern WHERE name GLOB x);"
1077
);
1078
if( bVerily ){
1079
db_multi_exec(
1080
"DELETE FROM concealed;\n"
1081
"UPDATE rcvfrom SET ipaddr='unknown';\n"
1082
"DROP TABLE IF EXISTS accesslog;\n"
1083
"UPDATE user SET photo=NULL, info='';\n"
1084
"DROP TABLE IF EXISTS purgeevent;\n"
1085
"DROP TABLE IF EXISTS purgeitem;\n"
1086
"DROP TABLE IF EXISTS admin_log;\n"
1087
"DROP TABLE IF EXISTS vcache;\n"
1088
"DROP TABLE IF EXISTS chat;\n"
1089
);
1090
}
1091
db_protect_pop();
1092
}
1093
if( !bNeedRebuild ){
1094
db_end_transaction(0);
1095
db_unprotect(PROTECT_ALL);
1096
db_multi_exec("VACUUM;");
1097
db_protect_pop();
1098
}else{
1099
rebuild_db(1, 0);
1100
db_end_transaction(0);
1101
}
1102
}
1103
1104
/*
1105
** Recursively read all files from the directory zPath and install
1106
** every file read as a new artifact in the repository.
1107
*/
1108
void recon_read_dir(char *zPath){
1109
DIR *d;
1110
struct dirent *pEntry;
1111
Blob aContent; /* content of the just read artifact */
1112
static int nFileRead = 0;
1113
void *zUnicodePath;
1114
char *zUtf8Name;
1115
static int recursionLevel = 0; /* Bookkeeping about the recursion level */
1116
static char *zFnRid1 = 0; /* The file holding the artifact with RID=1 */
1117
static int cchPathInitial = 0; /* The length of zPath on first recursion */
1118
1119
recursionLevel++;
1120
if( recursionLevel==1 ){
1121
cchPathInitial = strlen(zPath);
1122
if( fKeepRid1!=0 ){
1123
char *zFnDotRid1 = mprintf("%s/.rid1", zPath);
1124
Blob bFileContents;
1125
if( blob_read_from_file(&bFileContents, zFnDotRid1, ExtFILE)!=-1 ){
1126
Blob line, value;
1127
while( blob_line(&bFileContents, &line)>0 ){
1128
if( blob_token(&line, &value)==0 ) continue; /* Empty line */
1129
if( blob_buffer(&value)[0]=='#' ) continue; /* Comment */
1130
blob_trim(&value);
1131
zFnRid1 = mprintf("%s/%s", zPath, blob_str(&value));
1132
break;
1133
}
1134
blob_reset(&bFileContents);
1135
if( zFnRid1 ){
1136
if( blob_read_from_file(&aContent, zFnRid1, ExtFILE)==-1 ){
1137
fossil_fatal("some unknown error occurred while reading \"%s\"",
1138
zFnRid1);
1139
}else{
1140
recon_set_hash_policy(0, zFnRid1);
1141
content_put(&aContent);
1142
recon_restore_hash_policy();
1143
blob_reset(&aContent);
1144
fossil_print("\r%d", ++nFileRead);
1145
fflush(stdout);
1146
}
1147
}else{
1148
fossil_fatal("an error occurred while reading or parsing \"%s\"",
1149
zFnDotRid1);
1150
}
1151
}
1152
free(zFnDotRid1);
1153
}
1154
}
1155
zUnicodePath = fossil_utf8_to_path(zPath, 1);
1156
d = opendir(zUnicodePath);
1157
if( d ){
1158
while( (pEntry=readdir(d))!=0 ){
1159
Blob path;
1160
char *zSubpath;
1161
1162
if( pEntry->d_name[0]=='.' ){
1163
continue;
1164
}
1165
zUtf8Name = fossil_path_to_utf8(pEntry->d_name);
1166
zSubpath = mprintf("%s/%s", zPath, zUtf8Name);
1167
fossil_path_free(zUtf8Name);
1168
#ifdef _DIRENT_HAVE_D_TYPE
1169
if( (pEntry->d_type==DT_UNKNOWN || pEntry->d_type==DT_LNK)
1170
? (file_isdir(zSubpath, ExtFILE)==1) : (pEntry->d_type==DT_DIR) )
1171
#else
1172
if( file_isdir(zSubpath, ExtFILE)==1 )
1173
#endif
1174
{
1175
recon_read_dir(zSubpath);
1176
}else if( fossil_strcmp(zSubpath, zFnRid1)!=0 ){
1177
blob_init(&path, 0, 0);
1178
blob_appendf(&path, "%s", zSubpath);
1179
if( blob_read_from_file(&aContent, blob_str(&path), ExtFILE)==-1 ){
1180
fossil_fatal("some unknown error occurred while reading \"%s\"",
1181
blob_str(&path));
1182
}
1183
recon_set_hash_policy(cchPathInitial, blob_str(&path));
1184
content_put(&aContent);
1185
recon_restore_hash_policy();
1186
blob_reset(&path);
1187
blob_reset(&aContent);
1188
fossil_print("\r%d", ++nFileRead);
1189
fflush(stdout);
1190
}
1191
free(zSubpath);
1192
}
1193
closedir(d);
1194
}else {
1195
fossil_fatal("encountered error %d while trying to open \"%s\".",
1196
errno, g.argv[3]);
1197
}
1198
fossil_path_free(zUnicodePath);
1199
if( recursionLevel==1 && zFnRid1!=0 ) free(zFnRid1);
1200
recursionLevel--;
1201
}
1202
1203
/*
1204
** Helper functions called from recon_read_dir() to set and restore the correct
1205
** hash policy for an artifact read from disk, inferred from the length of the
1206
** path name.
1207
*/
1208
static int saved_eHashPolicy = -1;
1209
1210
void recon_set_hash_policy(
1211
const int cchPathPrefix, /* Directory prefix length for zUuidAsFilePath */
1212
const char *zUuidAsFilePath /* Relative, well-formed, from recon_read_dir() */
1213
){
1214
int cchUuidAsFilePath;
1215
const char *zHashPart;
1216
int cchHashPart = 0;
1217
int new_eHashPolicy = -1;
1218
assert( HNAME_COUNT==2 ); /* Review function if new hashes are implemented. */
1219
if( zUuidAsFilePath==0 ) return;
1220
cchUuidAsFilePath = strlen(zUuidAsFilePath);
1221
if( cchUuidAsFilePath==0 ) return;
1222
if( cchPathPrefix>=cchUuidAsFilePath ) return;
1223
for( zHashPart = zUuidAsFilePath + cchPathPrefix; *zHashPart; zHashPart++ ){
1224
if( *zHashPart!='/' ) cchHashPart++;
1225
}
1226
if( cchHashPart>=HNAME_LEN_K256 ){
1227
new_eHashPolicy = HPOLICY_SHA3;
1228
}else if( cchHashPart>=HNAME_LEN_SHA1 ){
1229
new_eHashPolicy = HPOLICY_SHA1;
1230
}
1231
if( new_eHashPolicy!=-1 ){
1232
saved_eHashPolicy = g.eHashPolicy;
1233
g.eHashPolicy = new_eHashPolicy;
1234
}
1235
}
1236
1237
void recon_restore_hash_policy(){
1238
if( saved_eHashPolicy!=-1 ){
1239
g.eHashPolicy = saved_eHashPolicy;
1240
saved_eHashPolicy = -1;
1241
}
1242
}
1243
1244
#if 0
1245
/*
1246
** COMMAND: test-hash-from-path*
1247
**
1248
** Usage: %fossil test-hash-from-path ?OPTIONS? DESTINATION UUID
1249
**
1250
** Generate a sample path name from DESTINATION and UUID, as the `deconstruct'
1251
** command would do. Then try to guess the hash policy from the path name, as
1252
** the `reconstruct' command would do.
1253
**
1254
** No files or directories will be created.
1255
**
1256
** Options:
1257
** -L|--prefixlength N Set the length of the names of the DESTINATION
1258
** subdirectories to N
1259
*/
1260
void test_hash_from_path_cmd(void) {
1261
char *zDest;
1262
char *zUuid;
1263
char *zFile;
1264
const char *zHashPolicy = "unknown";
1265
const char *zPrefixOpt = find_option("prefixlength","L",1);
1266
int iPrefixLength;
1267
if( !zPrefixOpt ){
1268
iPrefixLength = 2;
1269
}else{
1270
iPrefixLength = atoi(zPrefixOpt);
1271
if( iPrefixLength<0 || iPrefixLength>9 ){
1272
fossil_fatal("N(%s) is not a valid prefix length!",zPrefixOpt);
1273
}
1274
}
1275
if( g.argc!=4 ){
1276
usage ("?OPTIONS? DESTINATION UUID");
1277
}
1278
zDest = g.argv[2];
1279
zUuid = g.argv[3];
1280
if( iPrefixLength ){
1281
zFNameFormat = mprintf("%s/%%.%ds/%%s",zDest,iPrefixLength);
1282
}else{
1283
zFNameFormat = mprintf("%s/%%s",zDest);
1284
}
1285
cchFNamePrefix = strlen(zDest);
1286
zFile = mprintf(zFNameFormat /*works-like:"%s:%s"*/,
1287
zUuid, zUuid+iPrefixLength);
1288
recon_set_hash_policy(cchFNamePrefix,zFile);
1289
if( saved_eHashPolicy!=-1 ){
1290
zHashPolicy = hpolicy_name();
1291
}
1292
recon_restore_hash_policy();
1293
fossil_print(
1294
"\nPath Name: %s"
1295
"\nHash Policy: %s\n",
1296
zFile,zHashPolicy);
1297
free(zFile);
1298
free(zFNameFormat);
1299
zFNameFormat = 0;
1300
cchFNamePrefix = 0;
1301
}
1302
#endif
1303
1304
/*
1305
** Helper functions used by the `deconstruct' and `reconstruct' commands to
1306
** save and restore the contents of the PRIVATE table.
1307
*/
1308
void private_export(char *zFileName)
1309
{
1310
Stmt q;
1311
Blob fctx = empty_blob;
1312
blob_append(&fctx, "# The hashes of private artifacts\n", -1);
1313
db_prepare(&q,
1314
"SELECT uuid FROM blob WHERE rid IN ( SELECT rid FROM private );");
1315
while( db_step(&q)==SQLITE_ROW ){
1316
const char *zUuid = db_column_text(&q, 0);
1317
blob_append(&fctx, zUuid, -1);
1318
blob_append(&fctx, "\n", -1);
1319
}
1320
db_finalize(&q);
1321
blob_write_to_file(&fctx, zFileName);
1322
blob_reset(&fctx);
1323
}
1324
void private_import(char *zFileName)
1325
{
1326
Blob fctx;
1327
if( blob_read_from_file(&fctx, zFileName, ExtFILE)!=-1 ){
1328
Blob line, value;
1329
while( blob_line(&fctx, &line)>0 ){
1330
char *zUuid;
1331
int nUuid;
1332
if( blob_token(&line, &value)==0 ) continue; /* Empty line */
1333
if( blob_buffer(&value)[0]=='#' ) continue; /* Comment */
1334
blob_trim(&value);
1335
zUuid = blob_buffer(&value);
1336
nUuid = blob_size(&value);
1337
zUuid[nUuid] = 0;
1338
if( hname_validate(zUuid, nUuid)!=HNAME_ERROR ){
1339
canonical16(zUuid, nUuid);
1340
db_multi_exec(
1341
"INSERT OR IGNORE INTO private"
1342
" SELECT rid FROM blob WHERE uuid = %Q;",
1343
zUuid);
1344
}
1345
}
1346
blob_reset(&fctx);
1347
}
1348
}
1349
1350
/*
1351
** COMMAND: reconstruct*
1352
**
1353
** Usage: %fossil reconstruct ?OPTIONS? FILENAME DIRECTORY
1354
**
1355
** This command studies the artifacts (files) in DIRECTORY and reconstructs the
1356
** Fossil record from them. It places the new Fossil repository in FILENAME.
1357
** Subdirectories are read, files with leading '.' in the filename are ignored.
1358
**
1359
** Options:
1360
** -K|--keep-rid1 Read the filename of the artifact with RID=1 from the
1361
** file .rid in DIRECTORY.
1362
** -P|--keep-private Mark the artifacts listed in the file .private in
1363
** DIRECTORY as private in the new Fossil repository.
1364
*/
1365
void reconstruct_cmd(void) {
1366
char *zPassword;
1367
int fKeepPrivate;
1368
fKeepRid1 = find_option("keep-rid1","K",0)!=0;
1369
fKeepPrivate = find_option("keep-private","P",0)!=0;
1370
if( g.argc!=4 ){
1371
usage("FILENAME DIRECTORY");
1372
}
1373
if( file_isdir(g.argv[3], ExtFILE)!=1 ){
1374
fossil_print("\"%s\" is not a directory\n\n", g.argv[3]);
1375
usage("FILENAME DIRECTORY");
1376
}
1377
db_create_repository(g.argv[2]);
1378
db_open_repository(g.argv[2]);
1379
1380
/* We should be done with options.. */
1381
verify_all_options();
1382
1383
db_open_config(0, 0);
1384
db_begin_transaction();
1385
db_initial_setup(0, 0, 0);
1386
1387
fossil_print("Reading files from directory \"%s\"...\n", g.argv[3]);
1388
recon_read_dir(g.argv[3]);
1389
fossil_print("\nBuilding the Fossil repository...\n");
1390
1391
rebuild_db(1, 1);
1392
1393
/* Backwards compatibility: Mark check-ins with "+private" tags as private. */
1394
reconstruct_private_table();
1395
/* Newer method: Import the list of private artifacts to the PRIVATE table. */
1396
if( fKeepPrivate ){
1397
char *zFnDotPrivate = mprintf("%s/.private", g.argv[3]);
1398
private_import(zFnDotPrivate);
1399
free(zFnDotPrivate);
1400
}
1401
1402
/* Skip the verify_before_commit() step on a reconstruct. Most artifacts
1403
** will have been changed and verification therefore takes a really, really
1404
** long time.
1405
*/
1406
verify_cancel();
1407
1408
db_end_transaction(0);
1409
fossil_print("project-id: %s\n", db_get("project-code", 0));
1410
fossil_print("server-id: %s\n", db_get("server-code", 0));
1411
zPassword = db_text(0, "SELECT pw FROM user WHERE login=%Q", g.zLogin);
1412
fossil_print("admin-user: %s (initial password is \"%s\")\n", g.zLogin,
1413
zPassword);
1414
hash_user_password(g.zLogin);
1415
}
1416
1417
/*
1418
** COMMAND: deconstruct*
1419
**
1420
** Usage: %fossil deconstruct ?OPTIONS? DESTINATION
1421
**
1422
** This command exports all artifacts of a given repository and writes all
1423
** artifacts to the file system. The DESTINATION directory will be populated
1424
** with subdirectories AA and files AA/BBBBBBBBB.., where AABBBBBBBBB.. is the
1425
** 40+ character artifact ID, AA the first 2 characters.
1426
** If -L|--prefixlength is given, the length (default 2) of the directory prefix
1427
** can be set to 0,1,..,9 characters.
1428
**
1429
** Options:
1430
** -R|--repository REPO Deconstruct given REPOSITORY
1431
** -K|--keep-rid1 Save the filename of the artifact with RID=1 to
1432
** the file .rid1 in the DESTINATION directory
1433
** -L|--prefixlength N Set the length of the names of the DESTINATION
1434
** subdirectories to N
1435
** --private Include private artifacts
1436
** -P|--keep-private Save the list of private artifacts to the file
1437
** .private in the DESTINATION directory (implies
1438
** the --private option)
1439
*/
1440
void deconstruct_cmd(void){
1441
const char *zPrefixOpt;
1442
Stmt s;
1443
int privateFlag;
1444
int fKeepPrivate;
1445
1446
fKeepRid1 = find_option("keep-rid1","K",0)!=0;
1447
/* get and check prefix length argument and build format string */
1448
zPrefixOpt=find_option("prefixlength","L",1);
1449
if( !zPrefixOpt ){
1450
prefixLength = 2;
1451
}else{
1452
if( zPrefixOpt[0]>='0' && zPrefixOpt[0]<='9' && !zPrefixOpt[1] ){
1453
prefixLength = (int)(*zPrefixOpt-'0');
1454
}else{
1455
fossil_fatal("N(%s) is not a valid prefix length!",zPrefixOpt);
1456
}
1457
}
1458
/* open repository and open query for all artifacts */
1459
db_find_and_open_repository(OPEN_ANY_SCHEMA, 0);
1460
privateFlag = find_option("private",0,0)!=0;
1461
fKeepPrivate = find_option("keep-private","P",0)!=0;
1462
if( fKeepPrivate ) privateFlag = 1;
1463
verify_all_options();
1464
/* check number of arguments */
1465
if( g.argc!=3 ){
1466
usage ("?OPTIONS? DESTINATION");
1467
}
1468
/* get and check argument destination directory */
1469
zDestDir = g.argv[g.argc-1];
1470
if( !*zDestDir || !file_isdir(zDestDir, ExtFILE)) {
1471
fossil_fatal("DESTINATION(%s) is not a directory!",zDestDir);
1472
}
1473
#ifndef _WIN32
1474
if( file_access(zDestDir, W_OK) ){
1475
fossil_fatal("DESTINATION(%s) is not writeable!",zDestDir);
1476
}
1477
#else
1478
/* write access on windows is not checked, errors will be
1479
** detected on blob_write_to_file
1480
*/
1481
#endif
1482
if( prefixLength ){
1483
zFNameFormat = mprintf("%s/%%.%ds/%%s",zDestDir,prefixLength);
1484
}else{
1485
zFNameFormat = mprintf("%s/%%s",zDestDir);
1486
}
1487
cchFNamePrefix = strlen(zDestDir);
1488
1489
bag_init(&bagDone);
1490
ttyOutput = 1;
1491
processCnt = 0;
1492
if (!g.fQuiet) {
1493
fossil_print("0 (0%%)...\r");
1494
fflush(stdout);
1495
}
1496
totalSize = db_int(0, "SELECT count(*) FROM blob");
1497
db_prepare(&s,
1498
"SELECT rid, size FROM blob /*scan*/"
1499
" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1500
" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid) %s",
1501
privateFlag==0 ? "AND rid NOT IN private" : ""
1502
);
1503
while( db_step(&s)==SQLITE_ROW ){
1504
int rid = db_column_int(&s, 0);
1505
int size = db_column_int(&s, 1);
1506
if( size>=0 ){
1507
Blob content;
1508
content_get(rid, &content);
1509
rebuild_step(rid, size, &content);
1510
}
1511
}
1512
db_finalize(&s);
1513
db_prepare(&s,
1514
"SELECT rid, size FROM blob"
1515
" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid) %s",
1516
privateFlag==0 ? "AND rid NOT IN private" : ""
1517
);
1518
while( db_step(&s)==SQLITE_ROW ){
1519
int rid = db_column_int(&s, 0);
1520
int size = db_column_int(&s, 1);
1521
if( size>=0 ){
1522
if( !bag_find(&bagDone, rid) ){
1523
Blob content;
1524
content_get(rid, &content);
1525
rebuild_step(rid, size, &content);
1526
}
1527
}
1528
}
1529
db_finalize(&s);
1530
1531
/* Export the list of private artifacts. */
1532
if( fKeepPrivate ){
1533
char *zFnDotPrivate = mprintf("%s/.private", zDestDir);
1534
private_export(zFnDotPrivate);
1535
free(zFnDotPrivate);
1536
}
1537
1538
if(!g.fQuiet && ttyOutput ){
1539
fossil_print("\n");
1540
}
1541
1542
/* free filename format string */
1543
free(zFNameFormat);
1544
zFNameFormat = 0;
1545
}
1546

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button