| | @@ -58,10 +58,18 @@ |
| 58 | 58 | set QUIET 1 |
| 59 | 59 | set argv [lreplace $argv $i $i] |
| 60 | 60 | } else { |
| 61 | 61 | set QUIET 0 |
| 62 | 62 | } |
| 63 | + |
| 64 | +set i [lsearch $argv -strict] |
| 65 | +if {$i>=0} { |
| 66 | + set STRICT 1 |
| 67 | + set argv [lreplace $argv $i $i] |
| 68 | +} else { |
| 69 | + set STRICT 0 |
| 70 | +} |
| 63 | 71 | |
| 64 | 72 | if {[llength $argv]==0} { |
| 65 | 73 | foreach f [lsort [glob $testdir/*.test]] { |
| 66 | 74 | set base [file root [file tail $f]] |
| 67 | 75 | lappend argv $base |
| | @@ -332,24 +340,35 @@ |
| 332 | 340 | } |
| 333 | 341 | |
| 334 | 342 | # Perform a test |
| 335 | 343 | # |
| 336 | 344 | set test_count 0 |
| 337 | | -proc test {name expr} { |
| 338 | | - global bad_test test_count RESULT |
| 345 | +proc test {name expr {constraints ""}} { |
| 346 | + global bad_test ignored_test test_count RESULT |
| 339 | 347 | incr test_count |
| 348 | + set knownBug [expr {"knownBug" in $constraints}] |
| 340 | 349 | set r [uplevel 1 [list expr $expr]] |
| 341 | 350 | if {$r} { |
| 342 | | - protOut "test $name OK" |
| 351 | + if {$knownBug && !$::STRICT} { |
| 352 | + protOut "test $name OK (knownBug)?" |
| 353 | + } else { |
| 354 | + protOut "test $name OK" |
| 355 | + } |
| 343 | 356 | } else { |
| 344 | | - protOut "test $name FAILED!" 1 |
| 345 | | - if {$::QUIET} {protOut "RESULT: $RESULT" 1} |
| 346 | | - lappend bad_test $name |
| 347 | | - if {$::HALT} exit |
| 357 | + if {$knownBug && !$::STRICT} { |
| 358 | + protOut "test $name FAILED (knownBug)!" 1 |
| 359 | + lappend ignored_test $name |
| 360 | + } else { |
| 361 | + protOut "test $name FAILED!" 1 |
| 362 | + if {$::QUIET} {protOut "RESULT: $RESULT" 1} |
| 363 | + lappend bad_test $name |
| 364 | + if {$::HALT} exit |
| 365 | + } |
| 348 | 366 | } |
| 349 | 367 | } |
| 350 | 368 | set bad_test {} |
| 369 | +set ignored_test {} |
| 351 | 370 | |
| 352 | 371 | # Return a random string N characters long. |
| 353 | 372 | # |
| 354 | 373 | set vocabulary 01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| 355 | 374 | append vocabulary " ()*^!.eeeeeeeeaaaaattiioo " |
| | @@ -499,10 +518,17 @@ |
| 499 | 518 | protOut "***** End of $testfile: [llength $bad_test] errors so far ******" |
| 500 | 519 | cd $origwd |
| 501 | 520 | } |
| 502 | 521 | set nErr [llength $bad_test] |
| 503 | 522 | if {$nErr>0 || !$::QUIET} { |
| 504 | | - protOut "***** Final result: $nErr errors out of $test_count tests" 1 |
| 523 | + protOut "***** Final results: $nErr errors out of $test_count tests" 1 |
| 524 | +} |
| 525 | +if {$nErr>0} { |
| 526 | + protOut "***** Considered failures: $bad_test" 1 |
| 527 | +} |
| 528 | +set nErr [llength $ignored_test] |
| 529 | +if {$nErr>0 || !$::QUIET} { |
| 530 | + protOut "***** Ignored results: $nErr ignored errors out of $test_count tests" 1 |
| 505 | 531 | } |
| 506 | 532 | if {$nErr>0} { |
| 507 | | - protOut "***** Failures: $bad_test" 1 |
| 533 | + protOut "***** Ignored failures: $ignored_test" 1 |
| 508 | 534 | } |
| 509 | 535 | |