Fossil SCM

Add script to rewrite test output for easier comparison of output from different runs. Invoked with tclsh, the script reads either a file argument or stdin and writes to stdout.

preben 2023-11-17 17:47 testing-improvements
Commit b3f89945a51be601a881945ca61f588501f39aa9bfe17f1698ecb11becf2608a
Parent 7fc504d100eaccb…
1 file changed +159
+ test/rewrite-test-output.tcl
A test/rewrite-test-output.tcl
+159
--- a/test/rewrite-test-output.tcl
+++ b/test/rewrite-test-output.tcl
@@ -0,0 +1,159 @@
1
+#!/usr/bin/env tclsh
2
+
3
+# Script to anonymise test results for comparison.
4
+# - Replaces hashes, pids and similar with fixed strings
5
+# - Rewrites temporary paths to standardise them in output
6
+
7
+# With no arguments or "-", use$argcme "-"
8
+if { [llength $argv] > 0 } {
9
+ set fname [lindex $argv 0]
10
+}
11
+
12
+# Any -o$argc > 1 || [regexp {^-.+} usage: [file tail $argv0] ?FILE"
13
+ puts stderr " Rewrite test output to easeexit 1
14
+} elseif { $fname ne "-" && ! [file exists $fname] } {
15
+ puts stderr "File does not else {
16
+ if { $fname eq "-" } {
17
+ set fd stdin
18
+ } else {
19
+ set fd [open $fname r\2}
20
+ {^(added by )[^ ]*( on)$}
21
+ {\1USER\2}
22
+ {^(<script nonce=['\"])[0-9a-f]{48}(['\"]>/\* [a-z]+\.c:)\d+}
23
+ {\1NONCE\2LINENO}
24
+ {^(<script nonce=['\"])[0-9a-f]{48}(['\"]>)$}
25
+ {\1NONCE\2}
26
+ {^(projectCode: ")[0-9a-f]{40}(",)$}
27
+ {\1HASH\2}
28
+ {^\d+\.\d+(s by)$}
29
+ {N.SUB\1}
30
+ {^(window\.fossil.version = ")\d+\.\d+ \[[0-9a-f]{10}\] (YYYY-mm-dd HH:MM:SS(?: UTC";)?)$}
31
+ {\1N.M [HASH] \2}
32
+ {^(Fossil) \d+\.\d+ \[[0-9a-f]{10}\]( YYYY-mm-dd HH:MM:SS)$}
33
+ {\1 N.M [HASH]\2}
34
+ {^(type: Wiki-edit by ).+?( on YYYY-mm-dd HH:MM:SS)$$}
35
+ {\1USER\2}
36
+ {^(size: )\d+( bytes)$}
37
+ {\1N\2}
38
+ {^U [^ ]+$}
39
+ {U USER}
40
+ {^Z [0-9a-f]{32}$}
41
+ {Z CHECKSUM}
42
+}
43
+
44
+#
45
+# Some pattersn are used in multiple groups
46
+#
47
+
48
+set testnames {"th1" "th1-docs" "th1-hooks"}
49
+set pat {^((?:ERROR \(1\): )?/[*]{5} Subprocess) \d+ (exit)}
50
+set sub {\1 PID \2}
51
+foreach testname $testnames {
52
+ dict lappend tests $testname $pat $sub
53
+}
54
+
55
+set testnames {"th1-docs" "th1-hooks"}
56
+set pat {(?:[A-Z]:)?/.*?/(test-http-(?:in|out))-\d+-\d+-\d+(\.txt)}
57
+set sub {/TMP/\1-PID-SEQ-SEC\2}
58
+foreach testname $testnames {
59
+ dict lappend tests $testname $pat $sub
60
+}
61
+
62
+set testnames {"json" "th1" "wiki"}
63
+set pat {^(Content-Length:) \d+$}
64
+set sub {\1 LENGTH}
65
+foreach testname $testnames {
66
+ dict lappend tests $testname $pat $sub
67
+}
68
+
69
+set testnames {"th1" "wiki"}
70
+set pat {^\d+\.\d+(s by)$}
71
+set sub {N.SUB\1}
72
+foreach testname $testnames {
73
+ dict lappend tests $testname $pat $sub
74
+}
75
+
76
+#
77
+# Main
78
+#
79
+
80
+if { $fname eq "-" } {
81
+ set fd stdin
82
+} else {
83
+ set fd [open $fname r]
84
+}
85
+
86
+# Platforms we detect
87
+set UNKOWN_PLATFORM 0
88
+set UNIX 1
89
+set WINDOWS 2
90
+set CYGWIN 3
91
+
92
+# One specific wiki test creates repetitive output of varying length
93
+set wiki_f13_cmd1 "fossil wiki create {timestamp of 2399999} f13 --technote 2399999"
94
+set wiki_f13_cmd2 "fossil wiki list --technote --show-technote-ids"
95
+set wiki_f13_cmd3 "fossil wiki export a13 --technote ID"
96
+set collecting_f3 0
97
+set collecting_f3_verbose 0
98
+
99
+# Collected lines for summaries in --extra mode
100
+set amend_ed_lines [list]
101
+set amend_ed_failed 0
102
+set symlinks_lines [list]
103
+set symlinks_failed 0
104
+set test_simplify_name_lines [list]
105
+set test_simplify_name_failed 0
106
+
107
+# State information s we progress
108
+set check_json_empty_line 0
109
+set lineno 0
110
+set platform $UNKOWN_PLATFORM
111
+set prev_line ""
112
+set testname ""
113
+
114
+while { [gets $fd line] >= 0 } {
115
+ incr lineno
116
+
117
+ if { $lineno == 1 } {
118
+ if { [string index $line 0] in {"\UFFEF" "\UFEFF"} } {
119
+ set line [string range $line 1 end]
120
+ }
121
+ }
122
+
123
+ # Remove RESULT status while matching (inserted again in output).
124
+ # If collecting lines of output, include $result_prefix as needed.
125
+ regexp {^(RESULT \([01]\): )?(.*)} $line match result_prefix line
126
+
127
+ if { [regsub {^\*{5} ([^ ]+) \*{6}$} $line {\1} new_testname] } {
128
+ # Pick up test name for special handling below
129
+ set testname "$new_testname"
130
+ } elseif { [regexp {^\*{5} End of } $line] } {
131
+ # Test done. Handle --extra before resetting.
132
+ if { $EXTRA } {
133
+ if { $testname eq "symlinks" } {
134
+ if { $symlinks_failed } {
135
+ foreach l $symlinks_lines {
136
+ puts "$l"
137
+ }
138
+ } else {
139
+ puts "All symlinks tests OK (not run on Windows)"
140
+ }
141
+ }
142
+ regsub {(: )\d+( errors so far)} $line {\1N\2} line
143
+ }
144
+ set testname ""
145
+ } elseif { $testname ne "" } {
146
+ if { $platform == $UNKOWN_PLATFORM } {
147
+ if { [regexp {^[A-Z]:/.*?/fossil\.exe } $line] } {
148
+ set platform $WINDOWS
149
+ } elseif { [regexp {^/.*?/fossil\.exe } $line] } {
150
+ # No drive, but still .exe - must be CYGWIN
151
+ set platform $CYGWIN
152
+ } elseif { [regexp {^/.*?/fossil } $line] } {
153
+ set platform $UNIX
154
+ }
155
+ }
156
+
157
+ # Do common and per testname rewrites
158
+ set line [common_rewrites $line $testname]
159
+ if { [dict exists $tests $testna
--- a/test/rewrite-test-output.tcl
+++ b/test/rewrite-test-output.tcl
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
--- a/test/rewrite-test-output.tcl
+++ b/test/rewrite-test-output.tcl
@@ -0,0 +1,159 @@
1 #!/usr/bin/env tclsh
2
3 # Script to anonymise test results for comparison.
4 # - Replaces hashes, pids and similar with fixed strings
5 # - Rewrites temporary paths to standardise them in output
6
7 # With no arguments or "-", use$argcme "-"
8 if { [llength $argv] > 0 } {
9 set fname [lindex $argv 0]
10 }
11
12 # Any -o$argc > 1 || [regexp {^-.+} usage: [file tail $argv0] ?FILE"
13 puts stderr " Rewrite test output to easeexit 1
14 } elseif { $fname ne "-" && ! [file exists $fname] } {
15 puts stderr "File does not else {
16 if { $fname eq "-" } {
17 set fd stdin
18 } else {
19 set fd [open $fname r\2}
20 {^(added by )[^ ]*( on)$}
21 {\1USER\2}
22 {^(<script nonce=['\"])[0-9a-f]{48}(['\"]>/\* [a-z]+\.c:)\d+}
23 {\1NONCE\2LINENO}
24 {^(<script nonce=['\"])[0-9a-f]{48}(['\"]>)$}
25 {\1NONCE\2}
26 {^(projectCode: ")[0-9a-f]{40}(",)$}
27 {\1HASH\2}
28 {^\d+\.\d+(s by)$}
29 {N.SUB\1}
30 {^(window\.fossil.version = ")\d+\.\d+ \[[0-9a-f]{10}\] (YYYY-mm-dd HH:MM:SS(?: UTC";)?)$}
31 {\1N.M [HASH] \2}
32 {^(Fossil) \d+\.\d+ \[[0-9a-f]{10}\]( YYYY-mm-dd HH:MM:SS)$}
33 {\1 N.M [HASH]\2}
34 {^(type: Wiki-edit by ).+?( on YYYY-mm-dd HH:MM:SS)$$}
35 {\1USER\2}
36 {^(size: )\d+( bytes)$}
37 {\1N\2}
38 {^U [^ ]+$}
39 {U USER}
40 {^Z [0-9a-f]{32}$}
41 {Z CHECKSUM}
42 }
43
44 #
45 # Some pattersn are used in multiple groups
46 #
47
48 set testnames {"th1" "th1-docs" "th1-hooks"}
49 set pat {^((?:ERROR \(1\): )?/[*]{5} Subprocess) \d+ (exit)}
50 set sub {\1 PID \2}
51 foreach testname $testnames {
52 dict lappend tests $testname $pat $sub
53 }
54
55 set testnames {"th1-docs" "th1-hooks"}
56 set pat {(?:[A-Z]:)?/.*?/(test-http-(?:in|out))-\d+-\d+-\d+(\.txt)}
57 set sub {/TMP/\1-PID-SEQ-SEC\2}
58 foreach testname $testnames {
59 dict lappend tests $testname $pat $sub
60 }
61
62 set testnames {"json" "th1" "wiki"}
63 set pat {^(Content-Length:) \d+$}
64 set sub {\1 LENGTH}
65 foreach testname $testnames {
66 dict lappend tests $testname $pat $sub
67 }
68
69 set testnames {"th1" "wiki"}
70 set pat {^\d+\.\d+(s by)$}
71 set sub {N.SUB\1}
72 foreach testname $testnames {
73 dict lappend tests $testname $pat $sub
74 }
75
76 #
77 # Main
78 #
79
80 if { $fname eq "-" } {
81 set fd stdin
82 } else {
83 set fd [open $fname r]
84 }
85
86 # Platforms we detect
87 set UNKOWN_PLATFORM 0
88 set UNIX 1
89 set WINDOWS 2
90 set CYGWIN 3
91
92 # One specific wiki test creates repetitive output of varying length
93 set wiki_f13_cmd1 "fossil wiki create {timestamp of 2399999} f13 --technote 2399999"
94 set wiki_f13_cmd2 "fossil wiki list --technote --show-technote-ids"
95 set wiki_f13_cmd3 "fossil wiki export a13 --technote ID"
96 set collecting_f3 0
97 set collecting_f3_verbose 0
98
99 # Collected lines for summaries in --extra mode
100 set amend_ed_lines [list]
101 set amend_ed_failed 0
102 set symlinks_lines [list]
103 set symlinks_failed 0
104 set test_simplify_name_lines [list]
105 set test_simplify_name_failed 0
106
107 # State information s we progress
108 set check_json_empty_line 0
109 set lineno 0
110 set platform $UNKOWN_PLATFORM
111 set prev_line ""
112 set testname ""
113
114 while { [gets $fd line] >= 0 } {
115 incr lineno
116
117 if { $lineno == 1 } {
118 if { [string index $line 0] in {"\UFFEF" "\UFEFF"} } {
119 set line [string range $line 1 end]
120 }
121 }
122
123 # Remove RESULT status while matching (inserted again in output).
124 # If collecting lines of output, include $result_prefix as needed.
125 regexp {^(RESULT \([01]\): )?(.*)} $line match result_prefix line
126
127 if { [regsub {^\*{5} ([^ ]+) \*{6}$} $line {\1} new_testname] } {
128 # Pick up test name for special handling below
129 set testname "$new_testname"
130 } elseif { [regexp {^\*{5} End of } $line] } {
131 # Test done. Handle --extra before resetting.
132 if { $EXTRA } {
133 if { $testname eq "symlinks" } {
134 if { $symlinks_failed } {
135 foreach l $symlinks_lines {
136 puts "$l"
137 }
138 } else {
139 puts "All symlinks tests OK (not run on Windows)"
140 }
141 }
142 regsub {(: )\d+( errors so far)} $line {\1N\2} line
143 }
144 set testname ""
145 } elseif { $testname ne "" } {
146 if { $platform == $UNKOWN_PLATFORM } {
147 if { [regexp {^[A-Z]:/.*?/fossil\.exe } $line] } {
148 set platform $WINDOWS
149 } elseif { [regexp {^/.*?/fossil\.exe } $line] } {
150 # No drive, but still .exe - must be CYGWIN
151 set platform $CYGWIN
152 } elseif { [regexp {^/.*?/fossil } $line] } {
153 set platform $UNIX
154 }
155 }
156
157 # Do common and per testname rewrites
158 set line [common_rewrites $line $testname]
159 if { [dict exists $tests $testna

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button