Fossil SCM

Added and applied permissive-manifest-parser setting to permit parsing of manifests which have F-cards containing now-illegal names. Required for rebuild of historical data and support of repositories we now know to contain such files.

stephan 2020-08-18 16:07 sec2020-deadend
Commit 9e59cf18fccd0ea0be47a1f260d7df339c6d0ec06ba47d35a3d25a1fe1c968fc
+11
--- src/db.c
+++ src/db.c
@@ -3926,10 +3926,21 @@
39263926
** to obtain a check-in lock during auto-sync, the server will
39273927
** send the "pragma avoid-delta-manifests" statement in its reply,
39283928
** which will cause the client to avoid generating a delta
39293929
** manifest.
39303930
*/
3931
+/*
3932
+** SETTING: permissive-manifest-parser boolean default=off
3933
+** By default, fossil fatally fails if any files are found in a
3934
+** manifest which have a name matching a checkout database name. In
3935
+** order to support repositories where such files were inadvertently
3936
+** checked in, this setting, when on, allows such files to be handled
3937
+** as if they were normal files. Only enable this if absolutely
3938
+** necessary to support older repositories which have such files
3939
+** checked in (anywhere in their history). It should never be enabled
3940
+** for new repositories or old ones which do not contain such files.
3941
+*/
39313942
/*
39323943
** SETTING: proxy width=32 default=off
39333944
** URL of the HTTP proxy. If undefined or "off" then
39343945
** the "http_proxy" environment variable is consulted.
39353946
** If the http_proxy environment variable is undefined
39363947
--- src/db.c
+++ src/db.c
@@ -3926,10 +3926,21 @@
3926 ** to obtain a check-in lock during auto-sync, the server will
3927 ** send the "pragma avoid-delta-manifests" statement in its reply,
3928 ** which will cause the client to avoid generating a delta
3929 ** manifest.
3930 */
 
 
 
 
 
 
 
 
 
 
 
3931 /*
3932 ** SETTING: proxy width=32 default=off
3933 ** URL of the HTTP proxy. If undefined or "off" then
3934 ** the "http_proxy" environment variable is consulted.
3935 ** If the http_proxy environment variable is undefined
3936
--- src/db.c
+++ src/db.c
@@ -3926,10 +3926,21 @@
3926 ** to obtain a check-in lock during auto-sync, the server will
3927 ** send the "pragma avoid-delta-manifests" statement in its reply,
3928 ** which will cause the client to avoid generating a delta
3929 ** manifest.
3930 */
3931 /*
3932 ** SETTING: permissive-manifest-parser boolean default=off
3933 ** By default, fossil fatally fails if any files are found in a
3934 ** manifest which have a name matching a checkout database name. In
3935 ** order to support repositories where such files were inadvertently
3936 ** checked in, this setting, when on, allows such files to be handled
3937 ** as if they were normal files. Only enable this if absolutely
3938 ** necessary to support older repositories which have such files
3939 ** checked in (anywhere in their history). It should never be enabled
3940 ** for new repositories or old ones which do not contain such files.
3941 */
3942 /*
3943 ** SETTING: proxy width=32 default=off
3944 ** URL of the HTTP proxy. If undefined or "off" then
3945 ** the "http_proxy" environment variable is consulted.
3946 ** If the http_proxy environment variable is undefined
3947
+1 -1
--- src/file.c
+++ src/file.c
@@ -2516,11 +2516,11 @@
25162516
}
25172517
25182518
/*
25192519
** COMMAND: test-is-reserved-name
25202520
**
2521
-** Usage: %fossil test-is-ckout-db FILENAMES...
2521
+** Usage: %fossil test-is-reserved-name FILENAMES...
25222522
**
25232523
** Passes each given name to file_is_reserved_name() and outputs one
25242524
** line per file: the result value of that function followed by the
25252525
** name.
25262526
*/
25272527
--- src/file.c
+++ src/file.c
@@ -2516,11 +2516,11 @@
2516 }
2517
2518 /*
2519 ** COMMAND: test-is-reserved-name
2520 **
2521 ** Usage: %fossil test-is-ckout-db FILENAMES...
2522 **
2523 ** Passes each given name to file_is_reserved_name() and outputs one
2524 ** line per file: the result value of that function followed by the
2525 ** name.
2526 */
2527
--- src/file.c
+++ src/file.c
@@ -2516,11 +2516,11 @@
2516 }
2517
2518 /*
2519 ** COMMAND: test-is-reserved-name
2520 **
2521 ** Usage: %fossil test-is-reserved-name FILENAMES...
2522 **
2523 ** Passes each given name to file_is_reserved_name() and outputs one
2524 ** line per file: the result value of that function followed by the
2525 ** name.
2526 */
2527
+5
--- src/main.c
+++ src/main.c
@@ -220,10 +220,14 @@
220220
int noPswd; /* Logged in without password (on 127.0.0.1) */
221221
int userUid; /* Integer user id */
222222
int isHuman; /* True if access by a human, not a spider or bot */
223223
int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be
224224
** accessed through get_comment_format(). */
225
+ int permissiveManifest; /* Tells manifest_parser() whether it may run in
226
+ ** "permissive" (compatibilty) mode. <0=not yet determined,
227
+ ** 0=no, >0=yes. MUST be set to a negative value early on
228
+ ** in app-init (before CLI flags are processed). */
225229
226230
/* Information used to populate the RCVFROM table */
227231
int rcvid; /* The rcvid. 0 if not yet defined. */
228232
char *zIpAddr; /* The remote IP address */
229233
char *zNonce; /* The nonce used for login */
@@ -683,10 +687,11 @@
683687
sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
684688
sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0);
685689
memset(&g, 0, sizeof(g));
686690
g.now = time(0);
687691
g.httpHeader = empty_blob;
692
+ g.permissiveManifest = -1;
688693
#ifdef FOSSIL_ENABLE_JSON
689694
#if defined(NDEBUG)
690695
g.json.errorDetailParanoia = 2 /* FIXME: make configurable
691696
One problem we have here is that this
692697
code is needed before the db is opened,
693698
--- src/main.c
+++ src/main.c
@@ -220,10 +220,14 @@
220 int noPswd; /* Logged in without password (on 127.0.0.1) */
221 int userUid; /* Integer user id */
222 int isHuman; /* True if access by a human, not a spider or bot */
223 int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be
224 ** accessed through get_comment_format(). */
 
 
 
 
225
226 /* Information used to populate the RCVFROM table */
227 int rcvid; /* The rcvid. 0 if not yet defined. */
228 char *zIpAddr; /* The remote IP address */
229 char *zNonce; /* The nonce used for login */
@@ -683,10 +687,11 @@
683 sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
684 sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0);
685 memset(&g, 0, sizeof(g));
686 g.now = time(0);
687 g.httpHeader = empty_blob;
 
688 #ifdef FOSSIL_ENABLE_JSON
689 #if defined(NDEBUG)
690 g.json.errorDetailParanoia = 2 /* FIXME: make configurable
691 One problem we have here is that this
692 code is needed before the db is opened,
693
--- src/main.c
+++ src/main.c
@@ -220,10 +220,14 @@
220 int noPswd; /* Logged in without password (on 127.0.0.1) */
221 int userUid; /* Integer user id */
222 int isHuman; /* True if access by a human, not a spider or bot */
223 int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be
224 ** accessed through get_comment_format(). */
225 int permissiveManifest; /* Tells manifest_parser() whether it may run in
226 ** "permissive" (compatibilty) mode. <0=not yet determined,
227 ** 0=no, >0=yes. MUST be set to a negative value early on
228 ** in app-init (before CLI flags are processed). */
229
230 /* Information used to populate the RCVFROM table */
231 int rcvid; /* The rcvid. 0 if not yet defined. */
232 char *zIpAddr; /* The remote IP address */
233 char *zNonce; /* The nonce used for login */
@@ -683,10 +687,11 @@
687 sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
688 sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0);
689 memset(&g, 0, sizeof(g));
690 g.now = time(0);
691 g.httpHeader = empty_blob;
692 g.permissiveManifest = -1;
693 #ifdef FOSSIL_ENABLE_JSON
694 #if defined(NDEBUG)
695 g.json.errorDetailParanoia = 2 /* FIXME: make configurable
696 One problem we have here is that this
697 code is needed before the db is opened,
698
+8 -2
--- src/manifest.c
+++ src/manifest.c
@@ -454,11 +454,14 @@
454454
isRepeat = 1;
455455
}else{
456456
isRepeat = 0;
457457
bag_insert(&seenManifests, rid);
458458
}
459
-
459
+ if(g.permissiveManifest<0){
460
+ g.permissiveManifest =
461
+ db_get_boolean("permissive-manifest-parser", 0);
462
+ }
460463
/* Every structural artifact ends with a '\n' character. Exit early
461464
** if that is not the case for this artifact.
462465
*/
463466
if( !isRepeat ) g.parseCnt[0]++;
464467
z = blob_materialize(pContent);
@@ -630,13 +633,16 @@
630633
case 'F': {
631634
char *zName, *zPerm, *zPriorName;
632635
zName = next_token(&x,0);
633636
if( zName==0 ) SYNTAX("missing filename on F-card");
634637
defossilize(zName);
638
+ assert(g.permissiveManifest>=0
639
+ && "Must have been set at app init");
635640
if( !file_is_simple_pathname_nonstrict(zName) ){
636641
SYNTAX("F-card filename is not a simple path");
637
- }else if( file_is_reserved_name(zName,-1) ){
642
+ }else if( g.permissiveManifest==0
643
+ && file_is_reserved_name(zName,-1) ){
638644
SYNTAX("F-card contains a reserved name");
639645
}
640646
zUuid = next_token(&x, &sz);
641647
if( p->zBaseline==0 || zUuid!=0 ){
642648
if( zUuid==0 ) SYNTAX("missing hash on F-card");
643649
--- src/manifest.c
+++ src/manifest.c
@@ -454,11 +454,14 @@
454 isRepeat = 1;
455 }else{
456 isRepeat = 0;
457 bag_insert(&seenManifests, rid);
458 }
459
 
 
 
460 /* Every structural artifact ends with a '\n' character. Exit early
461 ** if that is not the case for this artifact.
462 */
463 if( !isRepeat ) g.parseCnt[0]++;
464 z = blob_materialize(pContent);
@@ -630,13 +633,16 @@
630 case 'F': {
631 char *zName, *zPerm, *zPriorName;
632 zName = next_token(&x,0);
633 if( zName==0 ) SYNTAX("missing filename on F-card");
634 defossilize(zName);
 
 
635 if( !file_is_simple_pathname_nonstrict(zName) ){
636 SYNTAX("F-card filename is not a simple path");
637 }else if( file_is_reserved_name(zName,-1) ){
 
638 SYNTAX("F-card contains a reserved name");
639 }
640 zUuid = next_token(&x, &sz);
641 if( p->zBaseline==0 || zUuid!=0 ){
642 if( zUuid==0 ) SYNTAX("missing hash on F-card");
643
--- src/manifest.c
+++ src/manifest.c
@@ -454,11 +454,14 @@
454 isRepeat = 1;
455 }else{
456 isRepeat = 0;
457 bag_insert(&seenManifests, rid);
458 }
459 if(g.permissiveManifest<0){
460 g.permissiveManifest =
461 db_get_boolean("permissive-manifest-parser", 0);
462 }
463 /* Every structural artifact ends with a '\n' character. Exit early
464 ** if that is not the case for this artifact.
465 */
466 if( !isRepeat ) g.parseCnt[0]++;
467 z = blob_materialize(pContent);
@@ -630,13 +633,16 @@
633 case 'F': {
634 char *zName, *zPerm, *zPriorName;
635 zName = next_token(&x,0);
636 if( zName==0 ) SYNTAX("missing filename on F-card");
637 defossilize(zName);
638 assert(g.permissiveManifest>=0
639 && "Must have been set at app init");
640 if( !file_is_simple_pathname_nonstrict(zName) ){
641 SYNTAX("F-card filename is not a simple path");
642 }else if( g.permissiveManifest==0
643 && file_is_reserved_name(zName,-1) ){
644 SYNTAX("F-card contains a reserved name");
645 }
646 zUuid = next_token(&x, &sz);
647 if( p->zBaseline==0 || zUuid!=0 ){
648 if( zUuid==0 ) SYNTAX("missing hash on F-card");
649
--- src/rebuild.c
+++ src/rebuild.c
@@ -628,10 +628,17 @@
628628
int optNoIndex;
629629
int optIndex;
630630
int optIfNeeded;
631631
int compressOnlyFlag;
632632
633
+ g.permissiveManifest = 1
634
+ /* We always allow permissive manifest parsing when mass-dealing
635
+ with batches which are likely to include historical, but no
636
+ longer used/relevant, manifests. Though rebuild will not fail
637
+ for bad manifests, it will consider them to be non-manifests,
638
+ so would necessarily elide them from the timeline.
639
+ */;
633640
omitVerify = find_option("noverify",0,0)!=0;
634641
forceFlag = find_option("force","f",0)!=0;
635642
randomizeFlag = find_option("randomize", 0, 0)!=0;
636643
doClustering = find_option("cluster", 0, 0)!=0;
637644
runVacuum = find_option("vacuum",0,0)!=0;
638645
--- src/rebuild.c
+++ src/rebuild.c
@@ -628,10 +628,17 @@
628 int optNoIndex;
629 int optIndex;
630 int optIfNeeded;
631 int compressOnlyFlag;
632
 
 
 
 
 
 
 
633 omitVerify = find_option("noverify",0,0)!=0;
634 forceFlag = find_option("force","f",0)!=0;
635 randomizeFlag = find_option("randomize", 0, 0)!=0;
636 doClustering = find_option("cluster", 0, 0)!=0;
637 runVacuum = find_option("vacuum",0,0)!=0;
638
--- src/rebuild.c
+++ src/rebuild.c
@@ -628,10 +628,17 @@
628 int optNoIndex;
629 int optIndex;
630 int optIfNeeded;
631 int compressOnlyFlag;
632
633 g.permissiveManifest = 1
634 /* We always allow permissive manifest parsing when mass-dealing
635 with batches which are likely to include historical, but no
636 longer used/relevant, manifests. Though rebuild will not fail
637 for bad manifests, it will consider them to be non-manifests,
638 so would necessarily elide them from the timeline.
639 */;
640 omitVerify = find_option("noverify",0,0)!=0;
641 forceFlag = find_option("force","f",0)!=0;
642 randomizeFlag = find_option("randomize", 0, 0)!=0;
643 doClustering = find_option("cluster", 0, 0)!=0;
644 runVacuum = find_option("vacuum",0,0)!=0;
645

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button