Fossil SCM
Added and applied permissive-manifest-parser setting to permit parsing of manifests which have F-cards containing now-illegal names. Required for rebuild of historical data and support of repositories we now know to contain such files.
Commit
9e59cf18fccd0ea0be47a1f260d7df339c6d0ec06ba47d35a3d25a1fe1c968fc
Parent
917917aa5533105…
5 files changed
+11
+1
-1
+5
+8
-2
+7
M
src/db.c
+11
| --- src/db.c | ||
| +++ src/db.c | ||
| @@ -3926,10 +3926,21 @@ | ||
| 3926 | 3926 | ** to obtain a check-in lock during auto-sync, the server will |
| 3927 | 3927 | ** send the "pragma avoid-delta-manifests" statement in its reply, |
| 3928 | 3928 | ** which will cause the client to avoid generating a delta |
| 3929 | 3929 | ** manifest. |
| 3930 | 3930 | */ |
| 3931 | +/* | |
| 3932 | +** SETTING: permissive-manifest-parser boolean default=off | |
| 3933 | +** By default, fossil fatally fails if any files are found in a | |
| 3934 | +** manifest which have a name matching a checkout database name. In | |
| 3935 | +** order to support repositories where such files were inadvertently | |
| 3936 | +** checked in, this setting, when on, allows such files to be handled | |
| 3937 | +** as if they were normal files. Only enable this if absolutely | |
| 3938 | +** necessary to support older repositories which have such files | |
| 3939 | +** checked in (anywhere in their history). It should never be enabled | |
| 3940 | +** for new repositories or old ones which do not contain such files. | |
| 3941 | +*/ | |
| 3931 | 3942 | /* |
| 3932 | 3943 | ** SETTING: proxy width=32 default=off |
| 3933 | 3944 | ** URL of the HTTP proxy. If undefined or "off" then |
| 3934 | 3945 | ** the "http_proxy" environment variable is consulted. |
| 3935 | 3946 | ** If the http_proxy environment variable is undefined |
| 3936 | 3947 |
| --- src/db.c | |
| +++ src/db.c | |
| @@ -3926,10 +3926,21 @@ | |
| 3926 | ** to obtain a check-in lock during auto-sync, the server will |
| 3927 | ** send the "pragma avoid-delta-manifests" statement in its reply, |
| 3928 | ** which will cause the client to avoid generating a delta |
| 3929 | ** manifest. |
| 3930 | */ |
| 3931 | /* |
| 3932 | ** SETTING: proxy width=32 default=off |
| 3933 | ** URL of the HTTP proxy. If undefined or "off" then |
| 3934 | ** the "http_proxy" environment variable is consulted. |
| 3935 | ** If the http_proxy environment variable is undefined |
| 3936 |
| --- src/db.c | |
| +++ src/db.c | |
| @@ -3926,10 +3926,21 @@ | |
| 3926 | ** to obtain a check-in lock during auto-sync, the server will |
| 3927 | ** send the "pragma avoid-delta-manifests" statement in its reply, |
| 3928 | ** which will cause the client to avoid generating a delta |
| 3929 | ** manifest. |
| 3930 | */ |
| 3931 | /* |
| 3932 | ** SETTING: permissive-manifest-parser boolean default=off |
| 3933 | ** By default, fossil fatally fails if any files are found in a |
| 3934 | ** manifest which have a name matching a checkout database name. In |
| 3935 | ** order to support repositories where such files were inadvertently |
| 3936 | ** checked in, this setting, when on, allows such files to be handled |
| 3937 | ** as if they were normal files. Only enable this if absolutely |
| 3938 | ** necessary to support older repositories which have such files |
| 3939 | ** checked in (anywhere in their history). It should never be enabled |
| 3940 | ** for new repositories or old ones which do not contain such files. |
| 3941 | */ |
| 3942 | /* |
| 3943 | ** SETTING: proxy width=32 default=off |
| 3944 | ** URL of the HTTP proxy. If undefined or "off" then |
| 3945 | ** the "http_proxy" environment variable is consulted. |
| 3946 | ** If the http_proxy environment variable is undefined |
| 3947 |
+1
-1
| --- src/file.c | ||
| +++ src/file.c | ||
| @@ -2516,11 +2516,11 @@ | ||
| 2516 | 2516 | } |
| 2517 | 2517 | |
| 2518 | 2518 | /* |
| 2519 | 2519 | ** COMMAND: test-is-reserved-name |
| 2520 | 2520 | ** |
| 2521 | -** Usage: %fossil test-is-ckout-db FILENAMES... | |
| 2521 | +** Usage: %fossil test-is-reserved-name FILENAMES... | |
| 2522 | 2522 | ** |
| 2523 | 2523 | ** Passes each given name to file_is_reserved_name() and outputs one |
| 2524 | 2524 | ** line per file: the result value of that function followed by the |
| 2525 | 2525 | ** name. |
| 2526 | 2526 | */ |
| 2527 | 2527 |
| --- src/file.c | |
| +++ src/file.c | |
| @@ -2516,11 +2516,11 @@ | |
| 2516 | } |
| 2517 | |
| 2518 | /* |
| 2519 | ** COMMAND: test-is-reserved-name |
| 2520 | ** |
| 2521 | ** Usage: %fossil test-is-ckout-db FILENAMES... |
| 2522 | ** |
| 2523 | ** Passes each given name to file_is_reserved_name() and outputs one |
| 2524 | ** line per file: the result value of that function followed by the |
| 2525 | ** name. |
| 2526 | */ |
| 2527 |
| --- src/file.c | |
| +++ src/file.c | |
| @@ -2516,11 +2516,11 @@ | |
| 2516 | } |
| 2517 | |
| 2518 | /* |
| 2519 | ** COMMAND: test-is-reserved-name |
| 2520 | ** |
| 2521 | ** Usage: %fossil test-is-reserved-name FILENAMES... |
| 2522 | ** |
| 2523 | ** Passes each given name to file_is_reserved_name() and outputs one |
| 2524 | ** line per file: the result value of that function followed by the |
| 2525 | ** name. |
| 2526 | */ |
| 2527 |
+5
| --- src/main.c | ||
| +++ src/main.c | ||
| @@ -220,10 +220,14 @@ | ||
| 220 | 220 | int noPswd; /* Logged in without password (on 127.0.0.1) */ |
| 221 | 221 | int userUid; /* Integer user id */ |
| 222 | 222 | int isHuman; /* True if access by a human, not a spider or bot */ |
| 223 | 223 | int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be |
| 224 | 224 | ** accessed through get_comment_format(). */ |
| 225 | + int permissiveManifest; /* Tells manifest_parser() whether it may run in | |
| 226 | + ** "permissive" (compatibilty) mode. <0=not yet determined, | |
| 227 | + ** 0=no, >0=yes. MUST be set to a negative value early on | |
| 228 | + ** in app-init (before CLI flags are processed). */ | |
| 225 | 229 | |
| 226 | 230 | /* Information used to populate the RCVFROM table */ |
| 227 | 231 | int rcvid; /* The rcvid. 0 if not yet defined. */ |
| 228 | 232 | char *zIpAddr; /* The remote IP address */ |
| 229 | 233 | char *zNonce; /* The nonce used for login */ |
| @@ -683,10 +687,11 @@ | ||
| 683 | 687 | sqlite3_config(SQLITE_CONFIG_MULTITHREAD); |
| 684 | 688 | sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0); |
| 685 | 689 | memset(&g, 0, sizeof(g)); |
| 686 | 690 | g.now = time(0); |
| 687 | 691 | g.httpHeader = empty_blob; |
| 692 | + g.permissiveManifest = -1; | |
| 688 | 693 | #ifdef FOSSIL_ENABLE_JSON |
| 689 | 694 | #if defined(NDEBUG) |
| 690 | 695 | g.json.errorDetailParanoia = 2 /* FIXME: make configurable |
| 691 | 696 | One problem we have here is that this |
| 692 | 697 | code is needed before the db is opened, |
| 693 | 698 |
| --- src/main.c | |
| +++ src/main.c | |
| @@ -220,10 +220,14 @@ | |
| 220 | int noPswd; /* Logged in without password (on 127.0.0.1) */ |
| 221 | int userUid; /* Integer user id */ |
| 222 | int isHuman; /* True if access by a human, not a spider or bot */ |
| 223 | int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be |
| 224 | ** accessed through get_comment_format(). */ |
| 225 | |
| 226 | /* Information used to populate the RCVFROM table */ |
| 227 | int rcvid; /* The rcvid. 0 if not yet defined. */ |
| 228 | char *zIpAddr; /* The remote IP address */ |
| 229 | char *zNonce; /* The nonce used for login */ |
| @@ -683,10 +687,11 @@ | |
| 683 | sqlite3_config(SQLITE_CONFIG_MULTITHREAD); |
| 684 | sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0); |
| 685 | memset(&g, 0, sizeof(g)); |
| 686 | g.now = time(0); |
| 687 | g.httpHeader = empty_blob; |
| 688 | #ifdef FOSSIL_ENABLE_JSON |
| 689 | #if defined(NDEBUG) |
| 690 | g.json.errorDetailParanoia = 2 /* FIXME: make configurable |
| 691 | One problem we have here is that this |
| 692 | code is needed before the db is opened, |
| 693 |
| --- src/main.c | |
| +++ src/main.c | |
| @@ -220,10 +220,14 @@ | |
| 220 | int noPswd; /* Logged in without password (on 127.0.0.1) */ |
| 221 | int userUid; /* Integer user id */ |
| 222 | int isHuman; /* True if access by a human, not a spider or bot */ |
| 223 | int comFmtFlags; /* Zero or more "COMMENT_PRINT_*" bit flags, should be |
| 224 | ** accessed through get_comment_format(). */ |
| 225 | int permissiveManifest; /* Tells manifest_parser() whether it may run in |
| 226 | ** "permissive" (compatibilty) mode. <0=not yet determined, |
| 227 | ** 0=no, >0=yes. MUST be set to a negative value early on |
| 228 | ** in app-init (before CLI flags are processed). */ |
| 229 | |
| 230 | /* Information used to populate the RCVFROM table */ |
| 231 | int rcvid; /* The rcvid. 0 if not yet defined. */ |
| 232 | char *zIpAddr; /* The remote IP address */ |
| 233 | char *zNonce; /* The nonce used for login */ |
| @@ -683,10 +687,11 @@ | |
| 687 | sqlite3_config(SQLITE_CONFIG_MULTITHREAD); |
| 688 | sqlite3_config(SQLITE_CONFIG_LOG, fossil_sqlite_log, 0); |
| 689 | memset(&g, 0, sizeof(g)); |
| 690 | g.now = time(0); |
| 691 | g.httpHeader = empty_blob; |
| 692 | g.permissiveManifest = -1; |
| 693 | #ifdef FOSSIL_ENABLE_JSON |
| 694 | #if defined(NDEBUG) |
| 695 | g.json.errorDetailParanoia = 2 /* FIXME: make configurable |
| 696 | One problem we have here is that this |
| 697 | code is needed before the db is opened, |
| 698 |
+8
-2
| --- src/manifest.c | ||
| +++ src/manifest.c | ||
| @@ -454,11 +454,14 @@ | ||
| 454 | 454 | isRepeat = 1; |
| 455 | 455 | }else{ |
| 456 | 456 | isRepeat = 0; |
| 457 | 457 | bag_insert(&seenManifests, rid); |
| 458 | 458 | } |
| 459 | - | |
| 459 | + if(g.permissiveManifest<0){ | |
| 460 | + g.permissiveManifest = | |
| 461 | + db_get_boolean("permissive-manifest-parser", 0); | |
| 462 | + } | |
| 460 | 463 | /* Every structural artifact ends with a '\n' character. Exit early |
| 461 | 464 | ** if that is not the case for this artifact. |
| 462 | 465 | */ |
| 463 | 466 | if( !isRepeat ) g.parseCnt[0]++; |
| 464 | 467 | z = blob_materialize(pContent); |
| @@ -630,13 +633,16 @@ | ||
| 630 | 633 | case 'F': { |
| 631 | 634 | char *zName, *zPerm, *zPriorName; |
| 632 | 635 | zName = next_token(&x,0); |
| 633 | 636 | if( zName==0 ) SYNTAX("missing filename on F-card"); |
| 634 | 637 | defossilize(zName); |
| 638 | + assert(g.permissiveManifest>=0 | |
| 639 | + && "Must have been set at app init"); | |
| 635 | 640 | if( !file_is_simple_pathname_nonstrict(zName) ){ |
| 636 | 641 | SYNTAX("F-card filename is not a simple path"); |
| 637 | - }else if( file_is_reserved_name(zName,-1) ){ | |
| 642 | + }else if( g.permissiveManifest==0 | |
| 643 | + && file_is_reserved_name(zName,-1) ){ | |
| 638 | 644 | SYNTAX("F-card contains a reserved name"); |
| 639 | 645 | } |
| 640 | 646 | zUuid = next_token(&x, &sz); |
| 641 | 647 | if( p->zBaseline==0 || zUuid!=0 ){ |
| 642 | 648 | if( zUuid==0 ) SYNTAX("missing hash on F-card"); |
| 643 | 649 |
| --- src/manifest.c | |
| +++ src/manifest.c | |
| @@ -454,11 +454,14 @@ | |
| 454 | isRepeat = 1; |
| 455 | }else{ |
| 456 | isRepeat = 0; |
| 457 | bag_insert(&seenManifests, rid); |
| 458 | } |
| 459 | |
| 460 | /* Every structural artifact ends with a '\n' character. Exit early |
| 461 | ** if that is not the case for this artifact. |
| 462 | */ |
| 463 | if( !isRepeat ) g.parseCnt[0]++; |
| 464 | z = blob_materialize(pContent); |
| @@ -630,13 +633,16 @@ | |
| 630 | case 'F': { |
| 631 | char *zName, *zPerm, *zPriorName; |
| 632 | zName = next_token(&x,0); |
| 633 | if( zName==0 ) SYNTAX("missing filename on F-card"); |
| 634 | defossilize(zName); |
| 635 | if( !file_is_simple_pathname_nonstrict(zName) ){ |
| 636 | SYNTAX("F-card filename is not a simple path"); |
| 637 | }else if( file_is_reserved_name(zName,-1) ){ |
| 638 | SYNTAX("F-card contains a reserved name"); |
| 639 | } |
| 640 | zUuid = next_token(&x, &sz); |
| 641 | if( p->zBaseline==0 || zUuid!=0 ){ |
| 642 | if( zUuid==0 ) SYNTAX("missing hash on F-card"); |
| 643 |
| --- src/manifest.c | |
| +++ src/manifest.c | |
| @@ -454,11 +454,14 @@ | |
| 454 | isRepeat = 1; |
| 455 | }else{ |
| 456 | isRepeat = 0; |
| 457 | bag_insert(&seenManifests, rid); |
| 458 | } |
| 459 | if(g.permissiveManifest<0){ |
| 460 | g.permissiveManifest = |
| 461 | db_get_boolean("permissive-manifest-parser", 0); |
| 462 | } |
| 463 | /* Every structural artifact ends with a '\n' character. Exit early |
| 464 | ** if that is not the case for this artifact. |
| 465 | */ |
| 466 | if( !isRepeat ) g.parseCnt[0]++; |
| 467 | z = blob_materialize(pContent); |
| @@ -630,13 +633,16 @@ | |
| 633 | case 'F': { |
| 634 | char *zName, *zPerm, *zPriorName; |
| 635 | zName = next_token(&x,0); |
| 636 | if( zName==0 ) SYNTAX("missing filename on F-card"); |
| 637 | defossilize(zName); |
| 638 | assert(g.permissiveManifest>=0 |
| 639 | && "Must have been set at app init"); |
| 640 | if( !file_is_simple_pathname_nonstrict(zName) ){ |
| 641 | SYNTAX("F-card filename is not a simple path"); |
| 642 | }else if( g.permissiveManifest==0 |
| 643 | && file_is_reserved_name(zName,-1) ){ |
| 644 | SYNTAX("F-card contains a reserved name"); |
| 645 | } |
| 646 | zUuid = next_token(&x, &sz); |
| 647 | if( p->zBaseline==0 || zUuid!=0 ){ |
| 648 | if( zUuid==0 ) SYNTAX("missing hash on F-card"); |
| 649 |
+7
| --- src/rebuild.c | ||
| +++ src/rebuild.c | ||
| @@ -628,10 +628,17 @@ | ||
| 628 | 628 | int optNoIndex; |
| 629 | 629 | int optIndex; |
| 630 | 630 | int optIfNeeded; |
| 631 | 631 | int compressOnlyFlag; |
| 632 | 632 | |
| 633 | + g.permissiveManifest = 1 | |
| 634 | + /* We always allow permissive manifest parsing when mass-dealing | |
| 635 | + with batches which are likely to include historical, but no | |
| 636 | + longer used/relevant, manifests. Though rebuild will not fail | |
| 637 | + for bad manifests, it will consider them to be non-manifests, | |
| 638 | + so would necessarily elide them from the timeline. | |
| 639 | + */; | |
| 633 | 640 | omitVerify = find_option("noverify",0,0)!=0; |
| 634 | 641 | forceFlag = find_option("force","f",0)!=0; |
| 635 | 642 | randomizeFlag = find_option("randomize", 0, 0)!=0; |
| 636 | 643 | doClustering = find_option("cluster", 0, 0)!=0; |
| 637 | 644 | runVacuum = find_option("vacuum",0,0)!=0; |
| 638 | 645 |
| --- src/rebuild.c | |
| +++ src/rebuild.c | |
| @@ -628,10 +628,17 @@ | |
| 628 | int optNoIndex; |
| 629 | int optIndex; |
| 630 | int optIfNeeded; |
| 631 | int compressOnlyFlag; |
| 632 | |
| 633 | omitVerify = find_option("noverify",0,0)!=0; |
| 634 | forceFlag = find_option("force","f",0)!=0; |
| 635 | randomizeFlag = find_option("randomize", 0, 0)!=0; |
| 636 | doClustering = find_option("cluster", 0, 0)!=0; |
| 637 | runVacuum = find_option("vacuum",0,0)!=0; |
| 638 |
| --- src/rebuild.c | |
| +++ src/rebuild.c | |
| @@ -628,10 +628,17 @@ | |
| 628 | int optNoIndex; |
| 629 | int optIndex; |
| 630 | int optIfNeeded; |
| 631 | int compressOnlyFlag; |
| 632 | |
| 633 | g.permissiveManifest = 1 |
| 634 | /* We always allow permissive manifest parsing when mass-dealing |
| 635 | with batches which are likely to include historical, but no |
| 636 | longer used/relevant, manifests. Though rebuild will not fail |
| 637 | for bad manifests, it will consider them to be non-manifests, |
| 638 | so would necessarily elide them from the timeline. |
| 639 | */; |
| 640 | omitVerify = find_option("noverify",0,0)!=0; |
| 641 | forceFlag = find_option("force","f",0)!=0; |
| 642 | randomizeFlag = find_option("randomize", 0, 0)!=0; |
| 643 | doClustering = find_option("cluster", 0, 0)!=0; |
| 644 | runVacuum = find_option("vacuum",0,0)!=0; |
| 645 |