Fossil SCM
Improvements to documentation and to the security_audit page.
Commit
cca43b2d7c8a08ec220447975888d7d4c34837059eea9120c3d12592b7990369
Parent
9dd7f23116dda15…
3 files changed
+7
+8
-6
+19
-6
+7
| --- src/login.c | ||
| +++ src/login.c | ||
| @@ -436,10 +436,17 @@ | ||
| 436 | 436 | |
| 437 | 437 | /* |
| 438 | 438 | ** Look at the HTTP_USER_AGENT parameter and try to determine if the user agent |
| 439 | 439 | ** is a manually operated browser or a bot. When in doubt, assume a bot. |
| 440 | 440 | ** Return true if we believe the agent is a real person. |
| 441 | +** | |
| 442 | +** UPDATE 2026-08-06: Robots have recently become much more aggressive and | |
| 443 | +** these days most of the troublesome robots do a good job of impersonating | |
| 444 | +** humans. Hence, this routine is much less effective that it used to be. | |
| 445 | +** It is still useful at blocking honest (self-reporting) robots, and so we | |
| 446 | +** keep it around for that reason. But it is not something that one should | |
| 447 | +** depend on as a robot defense. | |
| 441 | 448 | */ |
| 442 | 449 | static int isHuman(const char *zAgent){ |
| 443 | 450 | if( zAgent==0 ) return 0; /* If no UserAgent, then probably a bot */ |
| 444 | 451 | if( strstr(zAgent, "bot")!=0 ) return 0; |
| 445 | 452 | if( strstr(zAgent, "spider")!=0 ) return 0; |
| 446 | 453 |
| --- src/login.c | |
| +++ src/login.c | |
| @@ -436,10 +436,17 @@ | |
| 436 | |
| 437 | /* |
| 438 | ** Look at the HTTP_USER_AGENT parameter and try to determine if the user agent |
| 439 | ** is a manually operated browser or a bot. When in doubt, assume a bot. |
| 440 | ** Return true if we believe the agent is a real person. |
| 441 | */ |
| 442 | static int isHuman(const char *zAgent){ |
| 443 | if( zAgent==0 ) return 0; /* If no UserAgent, then probably a bot */ |
| 444 | if( strstr(zAgent, "bot")!=0 ) return 0; |
| 445 | if( strstr(zAgent, "spider")!=0 ) return 0; |
| 446 |
| --- src/login.c | |
| +++ src/login.c | |
| @@ -436,10 +436,17 @@ | |
| 436 | |
| 437 | /* |
| 438 | ** Look at the HTTP_USER_AGENT parameter and try to determine if the user agent |
| 439 | ** is a manually operated browser or a bot. When in doubt, assume a bot. |
| 440 | ** Return true if we believe the agent is a real person. |
| 441 | ** |
| 442 | ** UPDATE 2026-08-06: Robots have recently become much more aggressive and |
| 443 | ** these days most of the troublesome robots do a good job of impersonating |
| 444 | ** humans. Hence, this routine is much less effective that it used to be. |
| 445 | ** It is still useful at blocking honest (self-reporting) robots, and so we |
| 446 | ** keep it around for that reason. But it is not something that one should |
| 447 | ** depend on as a robot defense. |
| 448 | */ |
| 449 | static int isHuman(const char *zAgent){ |
| 450 | if( zAgent==0 ) return 0; /* If no UserAgent, then probably a bot */ |
| 451 | if( strstr(zAgent, "bot")!=0 ) return 0; |
| 452 | if( strstr(zAgent, "spider")!=0 ) return 0; |
| 453 |
+8
-6
| --- src/security_audit.c | ||
| +++ src/security_audit.c | ||
| @@ -712,34 +712,36 @@ | ||
| 712 | 712 | @ </p> |
| 713 | 713 | table_of_public_phantoms(); |
| 714 | 714 | @ </li> |
| 715 | 715 | } |
| 716 | 716 | |
| 717 | - @ <li><p>Robot Defenses: | |
| 717 | + @ <li><p><a href="setup_robot">Robot Defenses</a>: | |
| 718 | 718 | @ <ol type="a"> |
| 719 | 719 | switch( db_get_int("auto-hyperlink",1) ){ |
| 720 | 720 | default: |
| 721 | 721 | @ <li> No auto-enable of hyperlinks. |
| 722 | 722 | break; |
| 723 | 723 | case 1: |
| 724 | - @ <li> Hyperlinks auto-enabled based on UserAgent and Javascript. | |
| 724 | + @ <li> Hyperlinks auto-enabled based on HTTP Header and Javascript. | |
| 725 | 725 | break; |
| 726 | 726 | case 2: |
| 727 | - @ <li> Hyperlinks auto-enabled based on UserAgent only. | |
| 727 | + @ <li> Hyperlinks auto-enabled based on HTTP Header only. | |
| 728 | 728 | break; |
| 729 | 729 | } |
| 730 | 730 | z = db_get("max-loadavg",0); |
| 731 | 731 | if( z && atof(z)>0.0 ){ |
| 732 | 732 | @ <li> Maximum load average for expensive requests: %h(z); |
| 733 | 733 | }else{ |
| 734 | 734 | @ <li> No limits on the load average |
| 735 | 735 | } |
| 736 | 736 | z = db_get("robot-restrict",0); |
| 737 | - if( z==0 ){ | |
| 738 | - @ <li> No complex-request constraints on robots | |
| 737 | + if( z==0 || strcmp(z,"off")==0 ){ | |
| 738 | + @ <li> Robots are not excluded from any page. | |
| 739 | + }else if( z[0]=='*' && z[1]==0 ){ | |
| 740 | + @ <li> Robots are excluded from all pages other than /login. | |
| 739 | 741 | }else{ |
| 740 | - @ <li> Complex requests limited for pages matching: %h(z) | |
| 742 | + @ <li> Robots are excluded from pages matching: %h(z) | |
| 741 | 743 | } |
| 742 | 744 | @ </ol> |
| 743 | 745 | |
| 744 | 746 | blob_init(&cmd, 0, 0); |
| 745 | 747 | for(i=0; g.argvOrig[i]!=0; i++){ |
| 746 | 748 |
| --- src/security_audit.c | |
| +++ src/security_audit.c | |
| @@ -712,34 +712,36 @@ | |
| 712 | @ </p> |
| 713 | table_of_public_phantoms(); |
| 714 | @ </li> |
| 715 | } |
| 716 | |
| 717 | @ <li><p>Robot Defenses: |
| 718 | @ <ol type="a"> |
| 719 | switch( db_get_int("auto-hyperlink",1) ){ |
| 720 | default: |
| 721 | @ <li> No auto-enable of hyperlinks. |
| 722 | break; |
| 723 | case 1: |
| 724 | @ <li> Hyperlinks auto-enabled based on UserAgent and Javascript. |
| 725 | break; |
| 726 | case 2: |
| 727 | @ <li> Hyperlinks auto-enabled based on UserAgent only. |
| 728 | break; |
| 729 | } |
| 730 | z = db_get("max-loadavg",0); |
| 731 | if( z && atof(z)>0.0 ){ |
| 732 | @ <li> Maximum load average for expensive requests: %h(z); |
| 733 | }else{ |
| 734 | @ <li> No limits on the load average |
| 735 | } |
| 736 | z = db_get("robot-restrict",0); |
| 737 | if( z==0 ){ |
| 738 | @ <li> No complex-request constraints on robots |
| 739 | }else{ |
| 740 | @ <li> Complex requests limited for pages matching: %h(z) |
| 741 | } |
| 742 | @ </ol> |
| 743 | |
| 744 | blob_init(&cmd, 0, 0); |
| 745 | for(i=0; g.argvOrig[i]!=0; i++){ |
| 746 |
| --- src/security_audit.c | |
| +++ src/security_audit.c | |
| @@ -712,34 +712,36 @@ | |
| 712 | @ </p> |
| 713 | table_of_public_phantoms(); |
| 714 | @ </li> |
| 715 | } |
| 716 | |
| 717 | @ <li><p><a href="setup_robot">Robot Defenses</a>: |
| 718 | @ <ol type="a"> |
| 719 | switch( db_get_int("auto-hyperlink",1) ){ |
| 720 | default: |
| 721 | @ <li> No auto-enable of hyperlinks. |
| 722 | break; |
| 723 | case 1: |
| 724 | @ <li> Hyperlinks auto-enabled based on HTTP Header and Javascript. |
| 725 | break; |
| 726 | case 2: |
| 727 | @ <li> Hyperlinks auto-enabled based on HTTP Header only. |
| 728 | break; |
| 729 | } |
| 730 | z = db_get("max-loadavg",0); |
| 731 | if( z && atof(z)>0.0 ){ |
| 732 | @ <li> Maximum load average for expensive requests: %h(z); |
| 733 | }else{ |
| 734 | @ <li> No limits on the load average |
| 735 | } |
| 736 | z = db_get("robot-restrict",0); |
| 737 | if( z==0 || strcmp(z,"off")==0 ){ |
| 738 | @ <li> Robots are not excluded from any page. |
| 739 | }else if( z[0]=='*' && z[1]==0 ){ |
| 740 | @ <li> Robots are excluded from all pages other than /login. |
| 741 | }else{ |
| 742 | @ <li> Robots are excluded from pages matching: %h(z) |
| 743 | } |
| 744 | @ </ol> |
| 745 | |
| 746 | blob_init(&cmd, 0, 0); |
| 747 | for(i=0; g.argvOrig[i]!=0; i++){ |
| 748 |
+19
-6
| --- www/antibot.wiki | ||
| +++ www/antibot.wiki | ||
| @@ -88,19 +88,19 @@ | ||
| 88 | 88 | from a human or a robot. |
| 89 | 89 | |
| 90 | 90 | The [/help/auto-hyperlink|auto-hyperlink] setting, shown as |
| 91 | 91 | "<b>Enable hyperlinks based on User-Agent and/or Javascript</b>" on |
| 92 | 92 | the Robot Defense Settings page, |
| 93 | -can be set to "UserAgent only" or "UserAgent and Javascript" or "off". | |
| 93 | +can be set to "HTTP Header Only" or "HTTP Header and Javascript" or "off". | |
| 94 | 94 | If the UserAgent string looks like a human and not a robot, then |
| 95 | 95 | Fossil will enable hyperlinks even if the <b>Hyperlink</b> capability |
| 96 | 96 | is omitted from the user permissions. This setting gives humans easy |
| 97 | 97 | access to the hyperlinks while preventing robots |
| 98 | 98 | from walking the billions of pages on a typical Fossil site. |
| 99 | 99 | |
| 100 | -If the setting is "UserAgent only" (2), then the hyperlinks are simply | |
| 101 | -enabled and that is all. But if the setting is "UserAgent and Javascript" (1), | |
| 100 | +If the setting is "HTTP Header Only" (2), then the hyperlinks are simply | |
| 101 | +enabled and that is all. But if the setting is "HTTP Header and Javascript" (1), | |
| 102 | 102 | then the hyperlinks are not enabled directly. |
| 103 | 103 | Instead, the HTML code that is generated contains anchor tags ("<a>") |
| 104 | 104 | with "href=" attributes that point to [/honeypot] rather than the correct |
| 105 | 105 | link. JavaScript code is added to the end of the page that goes back and |
| 106 | 106 | fills in the correct "href=" attributes of |
| @@ -109,13 +109,17 @@ | ||
| 109 | 109 | is a security measure against robots that forge a human-looking |
| 110 | 110 | UserAgent string. Most robots do not bother to run JavaScript and |
| 111 | 111 | so to the robot the empty anchor tag will be useless. But all modern |
| 112 | 112 | web browsers implement JavaScript, so hyperlinks will show up |
| 113 | 113 | normally for human users. |
| 114 | +(UPDATE 2026-08-06: The previous two sentences | |
| 115 | +were true when this document was first written in 2013, but they are | |
| 116 | +not true any more. Most robots these days have normal-looking UserAgent | |
| 117 | +strings and do run Javascript.) | |
| 114 | 118 | |
| 115 | 119 | If the [/help/auto-hyperlink|"auto-hyperlink"] setting is (2) |
| 116 | -"<b>Enable hyperlinks using User-Agent and/or Javascript</b>", | |
| 120 | +"<b>Enable hyperlinks using HTTP-Header and/or Javascript</b>", | |
| 117 | 121 | then there are now two additional sub-settings that control when |
| 118 | 122 | hyperlinks are enabled. |
| 119 | 123 | |
| 120 | 124 | The first new sub-setting is a delay (in milliseconds) before setting |
| 121 | 125 | the "href=" attributes on anchor tags. The default value for this |
| @@ -149,10 +153,19 @@ | ||
| 149 | 153 | HTTP request is coming from a human, then any attempt to access one of |
| 150 | 154 | these pages brings up a javascript-powered captcha. The user has to |
| 151 | 155 | click the accept button the captcha once, and that sets a cookie allowing |
| 152 | 156 | the user to continue surfing without interruption for 15 minutes or so |
| 153 | 157 | before being presented with another captcha. |
| 158 | + | |
| 159 | +To disable this feature (not recommended for public-facing websites) | |
| 160 | +change this setting to "off". If robots become particularly troublesome, | |
| 161 | +change this setting to "*" (the GLOB wildcard) which restricts robots | |
| 162 | +from accessing any page of the Fossil webpage other than "/login". | |
| 163 | +Robots have gotten so bad lately (as of 2026) that the use of "*" is | |
| 164 | +advisable for any public-facing website that is running on a low-power | |
| 165 | +platform such as a RaspberryPI or a $6/month VPS, or any platform with | |
| 166 | +limited bandwidth. | |
| 154 | 167 | |
| 155 | 168 | Some path elements have special meanings: |
| 156 | 169 | |
| 157 | 170 | * <b>timelineX →</b> |
| 158 | 171 | This means a subset of /timeline/ pages that are considered |
| @@ -192,11 +205,11 @@ | ||
| 192 | 205 | This also matches /blame/ and /praise/. |
| 193 | 206 | |
| 194 | 207 | Other special keywords may be added in the future. |
| 195 | 208 | |
| 196 | 209 | The default [/help/robot-restrict|robot-restrict] |
| 197 | -setting has been shown in practice to do a good job of keeping | |
| 210 | +setting has been shown (circa 2026) to do a reasonable job of keeping | |
| 198 | 211 | robots from consuming all available CPU and bandwidth while will |
| 199 | 212 | still allowing humans access to the full power of the site without |
| 200 | 213 | having to be logged in. |
| 201 | 214 | |
| 202 | 215 | One possible enhancement is to add "zipX" to the |
| @@ -251,13 +264,13 @@ | ||
| 251 | 264 | <h2>The Ongoing Struggle</h2> |
| 252 | 265 | |
| 253 | 266 | Fossil currently does a good job of providing easy access to humans |
| 254 | 267 | while keeping out troublesome robots. However, robots |
| 255 | 268 | continue to grow more sophisticated, requiring ever more advanced |
| 256 | -defenses. This "arms race" is unlikely to ever end. The developers of | |
| 269 | +defenses. This "arms race" is unlikely to end. The developers of | |
| 257 | 270 | Fossil will continue to try improve the robot defenses of Fossil so |
| 258 | 271 | check back from time to time for the latest releases and updates. |
| 259 | 272 | |
| 260 | 273 | Readers of this page who have suggestions on how to improve the robot |
| 261 | 274 | defenses in Fossil are invited to submit your ideas to the Fossil Users |
| 262 | 275 | forum: |
| 263 | 276 | [https://fossil-scm.org/forum]. |
| 264 | 277 |
| --- www/antibot.wiki | |
| +++ www/antibot.wiki | |
| @@ -88,19 +88,19 @@ | |
| 88 | from a human or a robot. |
| 89 | |
| 90 | The [/help/auto-hyperlink|auto-hyperlink] setting, shown as |
| 91 | "<b>Enable hyperlinks based on User-Agent and/or Javascript</b>" on |
| 92 | the Robot Defense Settings page, |
| 93 | can be set to "UserAgent only" or "UserAgent and Javascript" or "off". |
| 94 | If the UserAgent string looks like a human and not a robot, then |
| 95 | Fossil will enable hyperlinks even if the <b>Hyperlink</b> capability |
| 96 | is omitted from the user permissions. This setting gives humans easy |
| 97 | access to the hyperlinks while preventing robots |
| 98 | from walking the billions of pages on a typical Fossil site. |
| 99 | |
| 100 | If the setting is "UserAgent only" (2), then the hyperlinks are simply |
| 101 | enabled and that is all. But if the setting is "UserAgent and Javascript" (1), |
| 102 | then the hyperlinks are not enabled directly. |
| 103 | Instead, the HTML code that is generated contains anchor tags ("<a>") |
| 104 | with "href=" attributes that point to [/honeypot] rather than the correct |
| 105 | link. JavaScript code is added to the end of the page that goes back and |
| 106 | fills in the correct "href=" attributes of |
| @@ -109,13 +109,17 @@ | |
| 109 | is a security measure against robots that forge a human-looking |
| 110 | UserAgent string. Most robots do not bother to run JavaScript and |
| 111 | so to the robot the empty anchor tag will be useless. But all modern |
| 112 | web browsers implement JavaScript, so hyperlinks will show up |
| 113 | normally for human users. |
| 114 | |
| 115 | If the [/help/auto-hyperlink|"auto-hyperlink"] setting is (2) |
| 116 | "<b>Enable hyperlinks using User-Agent and/or Javascript</b>", |
| 117 | then there are now two additional sub-settings that control when |
| 118 | hyperlinks are enabled. |
| 119 | |
| 120 | The first new sub-setting is a delay (in milliseconds) before setting |
| 121 | the "href=" attributes on anchor tags. The default value for this |
| @@ -149,10 +153,19 @@ | |
| 149 | HTTP request is coming from a human, then any attempt to access one of |
| 150 | these pages brings up a javascript-powered captcha. The user has to |
| 151 | click the accept button the captcha once, and that sets a cookie allowing |
| 152 | the user to continue surfing without interruption for 15 minutes or so |
| 153 | before being presented with another captcha. |
| 154 | |
| 155 | Some path elements have special meanings: |
| 156 | |
| 157 | * <b>timelineX →</b> |
| 158 | This means a subset of /timeline/ pages that are considered |
| @@ -192,11 +205,11 @@ | |
| 192 | This also matches /blame/ and /praise/. |
| 193 | |
| 194 | Other special keywords may be added in the future. |
| 195 | |
| 196 | The default [/help/robot-restrict|robot-restrict] |
| 197 | setting has been shown in practice to do a good job of keeping |
| 198 | robots from consuming all available CPU and bandwidth while will |
| 199 | still allowing humans access to the full power of the site without |
| 200 | having to be logged in. |
| 201 | |
| 202 | One possible enhancement is to add "zipX" to the |
| @@ -251,13 +264,13 @@ | |
| 251 | <h2>The Ongoing Struggle</h2> |
| 252 | |
| 253 | Fossil currently does a good job of providing easy access to humans |
| 254 | while keeping out troublesome robots. However, robots |
| 255 | continue to grow more sophisticated, requiring ever more advanced |
| 256 | defenses. This "arms race" is unlikely to ever end. The developers of |
| 257 | Fossil will continue to try improve the robot defenses of Fossil so |
| 258 | check back from time to time for the latest releases and updates. |
| 259 | |
| 260 | Readers of this page who have suggestions on how to improve the robot |
| 261 | defenses in Fossil are invited to submit your ideas to the Fossil Users |
| 262 | forum: |
| 263 | [https://fossil-scm.org/forum]. |
| 264 |
| --- www/antibot.wiki | |
| +++ www/antibot.wiki | |
| @@ -88,19 +88,19 @@ | |
| 88 | from a human or a robot. |
| 89 | |
| 90 | The [/help/auto-hyperlink|auto-hyperlink] setting, shown as |
| 91 | "<b>Enable hyperlinks based on User-Agent and/or Javascript</b>" on |
| 92 | the Robot Defense Settings page, |
| 93 | can be set to "HTTP Header Only" or "HTTP Header and Javascript" or "off". |
| 94 | If the UserAgent string looks like a human and not a robot, then |
| 95 | Fossil will enable hyperlinks even if the <b>Hyperlink</b> capability |
| 96 | is omitted from the user permissions. This setting gives humans easy |
| 97 | access to the hyperlinks while preventing robots |
| 98 | from walking the billions of pages on a typical Fossil site. |
| 99 | |
| 100 | If the setting is "HTTP Header Only" (2), then the hyperlinks are simply |
| 101 | enabled and that is all. But if the setting is "HTTP Header and Javascript" (1), |
| 102 | then the hyperlinks are not enabled directly. |
| 103 | Instead, the HTML code that is generated contains anchor tags ("<a>") |
| 104 | with "href=" attributes that point to [/honeypot] rather than the correct |
| 105 | link. JavaScript code is added to the end of the page that goes back and |
| 106 | fills in the correct "href=" attributes of |
| @@ -109,13 +109,17 @@ | |
| 109 | is a security measure against robots that forge a human-looking |
| 110 | UserAgent string. Most robots do not bother to run JavaScript and |
| 111 | so to the robot the empty anchor tag will be useless. But all modern |
| 112 | web browsers implement JavaScript, so hyperlinks will show up |
| 113 | normally for human users. |
| 114 | (UPDATE 2026-08-06: The previous two sentences |
| 115 | were true when this document was first written in 2013, but they are |
| 116 | not true any more. Most robots these days have normal-looking UserAgent |
| 117 | strings and do run Javascript.) |
| 118 | |
| 119 | If the [/help/auto-hyperlink|"auto-hyperlink"] setting is (2) |
| 120 | "<b>Enable hyperlinks using HTTP-Header and/or Javascript</b>", |
| 121 | then there are now two additional sub-settings that control when |
| 122 | hyperlinks are enabled. |
| 123 | |
| 124 | The first new sub-setting is a delay (in milliseconds) before setting |
| 125 | the "href=" attributes on anchor tags. The default value for this |
| @@ -149,10 +153,19 @@ | |
| 153 | HTTP request is coming from a human, then any attempt to access one of |
| 154 | these pages brings up a javascript-powered captcha. The user has to |
| 155 | click the accept button the captcha once, and that sets a cookie allowing |
| 156 | the user to continue surfing without interruption for 15 minutes or so |
| 157 | before being presented with another captcha. |
| 158 | |
| 159 | To disable this feature (not recommended for public-facing websites) |
| 160 | change this setting to "off". If robots become particularly troublesome, |
| 161 | change this setting to "*" (the GLOB wildcard) which restricts robots |
| 162 | from accessing any page of the Fossil webpage other than "/login". |
| 163 | Robots have gotten so bad lately (as of 2026) that the use of "*" is |
| 164 | advisable for any public-facing website that is running on a low-power |
| 165 | platform such as a RaspberryPI or a $6/month VPS, or any platform with |
| 166 | limited bandwidth. |
| 167 | |
| 168 | Some path elements have special meanings: |
| 169 | |
| 170 | * <b>timelineX →</b> |
| 171 | This means a subset of /timeline/ pages that are considered |
| @@ -192,11 +205,11 @@ | |
| 205 | This also matches /blame/ and /praise/. |
| 206 | |
| 207 | Other special keywords may be added in the future. |
| 208 | |
| 209 | The default [/help/robot-restrict|robot-restrict] |
| 210 | setting has been shown (circa 2026) to do a reasonable job of keeping |
| 211 | robots from consuming all available CPU and bandwidth while will |
| 212 | still allowing humans access to the full power of the site without |
| 213 | having to be logged in. |
| 214 | |
| 215 | One possible enhancement is to add "zipX" to the |
| @@ -251,13 +264,13 @@ | |
| 264 | <h2>The Ongoing Struggle</h2> |
| 265 | |
| 266 | Fossil currently does a good job of providing easy access to humans |
| 267 | while keeping out troublesome robots. However, robots |
| 268 | continue to grow more sophisticated, requiring ever more advanced |
| 269 | defenses. This "arms race" is unlikely to end. The developers of |
| 270 | Fossil will continue to try improve the robot defenses of Fossil so |
| 271 | check back from time to time for the latest releases and updates. |
| 272 | |
| 273 | Readers of this page who have suggestions on how to improve the robot |
| 274 | defenses in Fossil are invited to submit your ideas to the Fossil Users |
| 275 | forum: |
| 276 | [https://fossil-scm.org/forum]. |
| 277 |