1 #! /bin/sh 2 # Copyright (C) 2011-2014 Free Software Foundation, Inc. 3 # 4 # This program is free software; you can redistribute it and/or modify 5 # it under the terms of the GNU General Public License as published by 6 # the Free Software Foundation; either version 2, or (at your option) 7 # any later version. 8 # 9 # This program is distributed in the hope that it will be useful, 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # GNU General Public License for more details. 13 # 14 # You should have received a copy of the GNU General Public License 15 # along with this program. If not, see <http://www.gnu.org/licenses/>. 16 17 # As a special exception to the GNU General Public License, if you 18 # distribute this file as part of a program that contains a 19 # configuration script generated by Autoconf, you may include it under 20 # the same distribution terms that you use for the rest of that program. 21 22 # This file is maintained in Automake, please report 23 # bugs to <bug-automake (at] gnu.org> or send patches to 24 # <automake-patches (at] gnu.org>. 25 26 scriptversion=2013-12-23.17; # UTC 27 28 # Make unconditional expansion of undefined variables an error. This 29 # helps a lot in preventing typo-related bugs. 30 set -u 31 32 me=tap-driver.sh 33 34 fatal () 35 { 36 echo "$me: fatal: $*" >&2 37 exit 1 38 } 39 40 usage_error () 41 { 42 echo "$me: $*" >&2 43 print_usage >&2 44 exit 2 45 } 46 47 print_usage () 48 { 49 cat <<END 50 Usage: 51 tap-driver.sh --test-name=NAME --log-file=PATH --trs-file=PATH 52 [--expect-failure={yes|no}] [--color-tests={yes|no}] 53 [--enable-hard-errors={yes|no}] [--ignore-exit] 54 [--diagnostic-string=STRING] [--merge|--no-merge] 55 [--comments|--no-comments] [--] TEST-COMMAND 56 The '--test-name', '-log-file' and '--trs-file' options are mandatory. 57 END 58 } 59 60 # TODO: better error handling in option parsing (in particular, ensure 61 # TODO: $log_file, $trs_file and $test_name are defined). 62 test_name= # Used for reporting. 63 log_file= # Where to save the result and output of the test script. 64 trs_file= # Where to save the metadata of the test run. 65 expect_failure=0 66 color_tests=0 67 merge=0 68 ignore_exit=0 69 comments=0 70 diag_string='#' 71 while test $# -gt 0; do 72 case $1 in 73 --help) print_usage; exit $?;; 74 --version) echo "$me $scriptversion"; exit $?;; 75 --test-name) test_name=$2; shift;; 76 --log-file) log_file=$2; shift;; 77 --trs-file) trs_file=$2; shift;; 78 --color-tests) color_tests=$2; shift;; 79 --expect-failure) expect_failure=$2; shift;; 80 --enable-hard-errors) shift;; # No-op. 81 --merge) merge=1;; 82 --no-merge) merge=0;; 83 --ignore-exit) ignore_exit=1;; 84 --comments) comments=1;; 85 --no-comments) comments=0;; 86 --diagnostic-string) diag_string=$2; shift;; 87 --) shift; break;; 88 -*) usage_error "invalid option: '$1'";; 89 esac 90 shift 91 done 92 93 test $# -gt 0 || usage_error "missing test command" 94 95 case $expect_failure in 96 yes) expect_failure=1;; 97 *) expect_failure=0;; 98 esac 99 100 if test $color_tests = yes; then 101 init_colors=' 102 color_map["red"]="[0;31m" # Red. 103 color_map["grn"]="[0;32m" # Green. 104 color_map["lgn"]="[1;32m" # Light green. 105 color_map["blu"]="[1;34m" # Blue. 106 color_map["mgn"]="[0;35m" # Magenta. 107 color_map["std"]="[m" # No color. 108 color_for_result["ERROR"] = "mgn" 109 color_for_result["PASS"] = "grn" 110 color_for_result["XPASS"] = "red" 111 color_for_result["FAIL"] = "red" 112 color_for_result["XFAIL"] = "lgn" 113 color_for_result["SKIP"] = "blu"' 114 else 115 init_colors='' 116 fi 117 118 # :; is there to work around a bug in bash 3.2 (and earlier) which 119 # does not always set '$?' properly on redirection failure. 120 # See the Autoconf manual for more details. 121 :;{ 122 ( 123 # Ignore common signals (in this subshell only!), to avoid potential 124 # problems with Korn shells. Some Korn shells are known to propagate 125 # to themselves signals that have killed a child process they were 126 # waiting for; this is done at least for SIGINT (and usually only for 127 # it, in truth). Without the `trap' below, such a behaviour could 128 # cause a premature exit in the current subshell, e.g., in case the 129 # test command it runs gets terminated by a SIGINT. Thus, the awk 130 # script we are piping into would never seen the exit status it 131 # expects on its last input line (which is displayed below by the 132 # last `echo $?' statement), and would thus die reporting an internal 133 # error. 134 # For more information, see the Autoconf manual and the threads: 135 # <http://lists.gnu.org/archive/html/bug-autoconf/2011-09/msg00004.html> 136 # <http://mail.opensolaris.org/pipermail/ksh93-integration-discuss/2009-February/004121.html> 137 trap : 1 3 2 13 15 138 if test $merge -gt 0; then 139 exec 2>&1 140 else 141 exec 2>&3 142 fi 143 "$@" 144 echo $? 145 ) | LC_ALL=C ${AM_TAP_AWK-awk} \ 146 -v me="$me" \ 147 -v test_script_name="$test_name" \ 148 -v log_file="$log_file" \ 149 -v trs_file="$trs_file" \ 150 -v expect_failure="$expect_failure" \ 151 -v merge="$merge" \ 152 -v ignore_exit="$ignore_exit" \ 153 -v comments="$comments" \ 154 -v diag_string="$diag_string" \ 155 ' 156 # TODO: the usages of "cat >&3" below could be optimized when using 157 # GNU awk, and/on on systems that supports /dev/fd/. 158 159 # Implementation note: in what follows, `result_obj` will be an 160 # associative array that (partly) simulates a TAP result object 161 # from the `TAP::Parser` perl module. 162 163 ## ----------- ## 164 ## FUNCTIONS ## 165 ## ----------- ## 166 167 function fatal(msg) 168 { 169 print me ": " msg | "cat >&2" 170 exit 1 171 } 172 173 function abort(where) 174 { 175 fatal("internal error " where) 176 } 177 178 # Convert a boolean to a "yes"/"no" string. 179 function yn(bool) 180 { 181 return bool ? "yes" : "no"; 182 } 183 184 function add_test_result(result) 185 { 186 if (!test_results_index) 187 test_results_index = 0 188 test_results_list[test_results_index] = result 189 test_results_index += 1 190 test_results_seen[result] = 1; 191 } 192 193 # Whether the test script should be re-run by "make recheck". 194 function must_recheck() 195 { 196 for (k in test_results_seen) 197 if (k != "XFAIL" && k != "PASS" && k != "SKIP") 198 return 1 199 return 0 200 } 201 202 # Whether the content of the log file associated to this test should 203 # be copied into the "global" test-suite.log. 204 function copy_in_global_log() 205 { 206 for (k in test_results_seen) 207 if (k != "PASS") 208 return 1 209 return 0 210 } 211 212 function get_global_test_result() 213 { 214 if ("ERROR" in test_results_seen) 215 return "ERROR" 216 if ("FAIL" in test_results_seen || "XPASS" in test_results_seen) 217 return "FAIL" 218 all_skipped = 1 219 for (k in test_results_seen) 220 if (k != "SKIP") 221 all_skipped = 0 222 if (all_skipped) 223 return "SKIP" 224 return "PASS"; 225 } 226 227 function stringify_result_obj(result_obj) 228 { 229 if (result_obj["is_unplanned"] || result_obj["number"] != testno) 230 return "ERROR" 231 232 if (plan_seen == LATE_PLAN) 233 return "ERROR" 234 235 if (result_obj["directive"] == "TODO") 236 return result_obj["is_ok"] ? "XPASS" : "XFAIL" 237 238 if (result_obj["directive"] == "SKIP") 239 return result_obj["is_ok"] ? "SKIP" : COOKED_FAIL; 240 241 if (length(result_obj["directive"])) 242 abort("in function stringify_result_obj()") 243 244 return result_obj["is_ok"] ? COOKED_PASS : COOKED_FAIL 245 } 246 247 function decorate_result(result) 248 { 249 color_name = color_for_result[result] 250 if (color_name) 251 return color_map[color_name] "" result "" color_map["std"] 252 # If we are not using colorized output, or if we do not know how 253 # to colorize the given result, we should return it unchanged. 254 return result 255 } 256 257 function report(result, details) 258 { 259 if (result ~ /^(X?(PASS|FAIL)|SKIP|ERROR)/) 260 { 261 msg = ": " test_script_name 262 add_test_result(result) 263 } 264 else if (result == "#") 265 { 266 msg = " " test_script_name ":" 267 } 268 else 269 { 270 abort("in function report()") 271 } 272 if (length(details)) 273 msg = msg " " details 274 # Output on console might be colorized. 275 print decorate_result(result) msg 276 # Log the result in the log file too, to help debugging (this is 277 # especially true when said result is a TAP error or "Bail out!"). 278 print result msg | "cat >&3"; 279 } 280 281 function testsuite_error(error_message) 282 { 283 report("ERROR", "- " error_message) 284 } 285 286 function handle_tap_result() 287 { 288 details = result_obj["number"]; 289 if (length(result_obj["description"])) 290 details = details " " result_obj["description"] 291 292 if (plan_seen == LATE_PLAN) 293 { 294 details = details " # AFTER LATE PLAN"; 295 } 296 else if (result_obj["is_unplanned"]) 297 { 298 details = details " # UNPLANNED"; 299 } 300 else if (result_obj["number"] != testno) 301 { 302 details = sprintf("%s # OUT-OF-ORDER (expecting %d)", 303 details, testno); 304 } 305 else if (result_obj["directive"]) 306 { 307 details = details " # " result_obj["directive"]; 308 if (length(result_obj["explanation"])) 309 details = details " " result_obj["explanation"] 310 } 311 312 report(stringify_result_obj(result_obj), details) 313 } 314 315 # `skip_reason` should be empty whenever planned > 0. 316 function handle_tap_plan(planned, skip_reason) 317 { 318 planned += 0 # Avoid getting confused if, say, `planned` is "00" 319 if (length(skip_reason) && planned > 0) 320 abort("in function handle_tap_plan()") 321 if (plan_seen) 322 { 323 # Error, only one plan per stream is acceptable. 324 testsuite_error("multiple test plans") 325 return; 326 } 327 planned_tests = planned 328 # The TAP plan can come before or after *all* the TAP results; we speak 329 # respectively of an "early" or a "late" plan. If we see the plan line 330 # after at least one TAP result has been seen, assume we have a late 331 # plan; in this case, any further test result seen after the plan will 332 # be flagged as an error. 333 plan_seen = (testno >= 1 ? LATE_PLAN : EARLY_PLAN) 334 # If testno > 0, we have an error ("too many tests run") that will be 335 # automatically dealt with later, so do not worry about it here. If 336 # $plan_seen is true, we have an error due to a repeated plan, and that 337 # has already been dealt with above. Otherwise, we have a valid "plan 338 # with SKIP" specification, and should report it as a particular kind 339 # of SKIP result. 340 if (planned == 0 && testno == 0) 341 { 342 if (length(skip_reason)) 343 skip_reason = "- " skip_reason; 344 report("SKIP", skip_reason); 345 } 346 } 347 348 function extract_tap_comment(line) 349 { 350 if (index(line, diag_string) == 1) 351 { 352 # Strip leading `diag_string` from `line`. 353 line = substr(line, length(diag_string) + 1) 354 # And strip any leading and trailing whitespace left. 355 sub("^[ \t]*", "", line) 356 sub("[ \t]*$", "", line) 357 # Return what is left (if any). 358 return line; 359 } 360 return ""; 361 } 362 363 # When this function is called, we know that line is a TAP result line, 364 # so that it matches the (perl) RE "^(not )?ok\b". 365 function setup_result_obj(line) 366 { 367 # Get the result, and remove it from the line. 368 result_obj["is_ok"] = (substr(line, 1, 2) == "ok" ? 1 : 0) 369 sub("^(not )?ok[ \t]*", "", line) 370 371 # If the result has an explicit number, get it and strip it; otherwise, 372 # automatically assing the next progresive number to it. 373 if (line ~ /^[0-9]+$/ || line ~ /^[0-9]+[^a-zA-Z0-9_]/) 374 { 375 match(line, "^[0-9]+") 376 # The final `+ 0` is to normalize numbers with leading zeros. 377 result_obj["number"] = substr(line, 1, RLENGTH) + 0 378 line = substr(line, RLENGTH + 1) 379 } 380 else 381 { 382 result_obj["number"] = testno 383 } 384 385 if (plan_seen == LATE_PLAN) 386 # No further test results are acceptable after a "late" TAP plan 387 # has been seen. 388 result_obj["is_unplanned"] = 1 389 else if (plan_seen && testno > planned_tests) 390 result_obj["is_unplanned"] = 1 391 else 392 result_obj["is_unplanned"] = 0 393 394 # Strip trailing and leading whitespace. 395 sub("^[ \t]*", "", line) 396 sub("[ \t]*$", "", line) 397 398 # This will have to be corrected if we have a "TODO"/"SKIP" directive. 399 result_obj["description"] = line 400 result_obj["directive"] = "" 401 result_obj["explanation"] = "" 402 403 if (index(line, "#") == 0) 404 return # No possible directive, nothing more to do. 405 406 # Directives are case-insensitive. 407 rx = "[ \t]*#[ \t]*([tT][oO][dD][oO]|[sS][kK][iI][pP])[ \t]*" 408 409 # See whether we have the directive, and if yes, where. 410 pos = match(line, rx "$") 411 if (!pos) 412 pos = match(line, rx "[^a-zA-Z0-9_]") 413 414 # If there was no TAP directive, we have nothing more to do. 415 if (!pos) 416 return 417 418 # Let`s now see if the TAP directive has been escaped. For example: 419 # escaped: ok \# SKIP 420 # not escaped: ok \\# SKIP 421 # escaped: ok \\\\\# SKIP 422 # not escaped: ok \ # SKIP 423 if (substr(line, pos, 1) == "#") 424 { 425 bslash_count = 0 426 for (i = pos; i > 1 && substr(line, i - 1, 1) == "\\"; i--) 427 bslash_count += 1 428 if (bslash_count % 2) 429 return # Directive was escaped. 430 } 431 432 # Strip the directive and its explanation (if any) from the test 433 # description. 434 result_obj["description"] = substr(line, 1, pos - 1) 435 # Now remove the test description from the line, that has been dealt 436 # with already. 437 line = substr(line, pos) 438 # Strip the directive, and save its value (normalized to upper case). 439 sub("^[ \t]*#[ \t]*", "", line) 440 result_obj["directive"] = toupper(substr(line, 1, 4)) 441 line = substr(line, 5) 442 # Now get the explanation for the directive (if any), with leading 443 # and trailing whitespace removed. 444 sub("^[ \t]*", "", line) 445 sub("[ \t]*$", "", line) 446 result_obj["explanation"] = line 447 } 448 449 function get_test_exit_message(status) 450 { 451 if (status == 0) 452 return "" 453 if (status !~ /^[1-9][0-9]*$/) 454 abort("getting exit status") 455 if (status < 127) 456 exit_details = "" 457 else if (status == 127) 458 exit_details = " (command not found?)" 459 else if (status >= 128 && status <= 255) 460 exit_details = sprintf(" (terminated by signal %d?)", status - 128) 461 else if (status > 256 && status <= 384) 462 # We used to report an "abnormal termination" here, but some Korn 463 # shells, when a child process die due to signal number n, can leave 464 # in $? an exit status of 256+n instead of the more standard 128+n. 465 # Apparently, both behaviours are allowed by POSIX (2008), so be 466 # prepared to handle them both. See also Austing Group report ID 467 # 0000051 <http://www.austingroupbugs.net/view.php?id=51> 468 exit_details = sprintf(" (terminated by signal %d?)", status - 256) 469 else 470 # Never seen in practice. 471 exit_details = " (abnormal termination)" 472 return sprintf("exited with status %d%s", status, exit_details) 473 } 474 475 function write_test_results() 476 { 477 print ":global-test-result: " get_global_test_result() > trs_file 478 print ":recheck: " yn(must_recheck()) > trs_file 479 print ":copy-in-global-log: " yn(copy_in_global_log()) > trs_file 480 for (i = 0; i < test_results_index; i += 1) 481 print ":test-result: " test_results_list[i] > trs_file 482 close(trs_file); 483 } 484 485 BEGIN { 486 487 ## ------- ## 488 ## SETUP ## 489 ## ------- ## 490 491 '"$init_colors"' 492 493 # Properly initialized once the TAP plan is seen. 494 planned_tests = 0 495 496 COOKED_PASS = expect_failure ? "XPASS": "PASS"; 497 COOKED_FAIL = expect_failure ? "XFAIL": "FAIL"; 498 499 # Enumeration-like constants to remember which kind of plan (if any) 500 # has been seen. It is important that NO_PLAN evaluates "false" as 501 # a boolean. 502 NO_PLAN = 0 503 EARLY_PLAN = 1 504 LATE_PLAN = 2 505 506 testno = 0 # Number of test results seen so far. 507 bailed_out = 0 # Whether a "Bail out!" directive has been seen. 508 509 # Whether the TAP plan has been seen or not, and if yes, which kind 510 # it is ("early" is seen before any test result, "late" otherwise). 511 plan_seen = NO_PLAN 512 513 ## --------- ## 514 ## PARSING ## 515 ## --------- ## 516 517 is_first_read = 1 518 519 while (1) 520 { 521 # Involutions required so that we are able to read the exit status 522 # from the last input line. 523 st = getline 524 if (st < 0) # I/O error. 525 fatal("I/O error while reading from input stream") 526 else if (st == 0) # End-of-input 527 { 528 if (is_first_read) 529 abort("in input loop: only one input line") 530 break 531 } 532 if (is_first_read) 533 { 534 is_first_read = 0 535 nextline = $0 536 continue 537 } 538 else 539 { 540 curline = nextline 541 nextline = $0 542 $0 = curline 543 } 544 # Copy any input line verbatim into the log file. 545 print | "cat >&3" 546 # Parsing of TAP input should stop after a "Bail out!" directive. 547 if (bailed_out) 548 continue 549 550 # TAP test result. 551 if ($0 ~ /^(not )?ok$/ || $0 ~ /^(not )?ok[^a-zA-Z0-9_]/) 552 { 553 testno += 1 554 setup_result_obj($0) 555 handle_tap_result() 556 } 557 # TAP plan (normal or "SKIP" without explanation). 558 else if ($0 ~ /^1\.\.[0-9]+[ \t]*$/) 559 { 560 # The next two lines will put the number of planned tests in $0. 561 sub("^1\\.\\.", "") 562 sub("[^0-9]*$", "") 563 handle_tap_plan($0, "") 564 continue 565 } 566 # TAP "SKIP" plan, with an explanation. 567 else if ($0 ~ /^1\.\.0+[ \t]*#/) 568 { 569 # The next lines will put the skip explanation in $0, stripping 570 # any leading and trailing whitespace. This is a little more 571 # tricky in truth, since we want to also strip a potential leading 572 # "SKIP" string from the message. 573 sub("^[^#]*#[ \t]*(SKIP[: \t][ \t]*)?", "") 574 sub("[ \t]*$", ""); 575 handle_tap_plan(0, $0) 576 } 577 # "Bail out!" magic. 578 # Older versions of prove and TAP::Harness (e.g., 3.17) did not 579 # recognize a "Bail out!" directive when preceded by leading 580 # whitespace, but more modern versions (e.g., 3.23) do. So we 581 # emulate the latter, "more modern" behaviour. 582 else if ($0 ~ /^[ \t]*Bail out!/) 583 { 584 bailed_out = 1 585 # Get the bailout message (if any), with leading and trailing 586 # whitespace stripped. The message remains stored in `$0`. 587 sub("^[ \t]*Bail out![ \t]*", ""); 588 sub("[ \t]*$", ""); 589 # Format the error message for the 590 bailout_message = "Bail out!" 591 if (length($0)) 592 bailout_message = bailout_message " " $0 593 testsuite_error(bailout_message) 594 } 595 # Maybe we have too look for dianogtic comments too. 596 else if (comments != 0) 597 { 598 comment = extract_tap_comment($0); 599 if (length(comment)) 600 report("#", comment); 601 } 602 } 603 604 ## -------- ## 605 ## FINISH ## 606 ## -------- ## 607 608 # A "Bail out!" directive should cause us to ignore any following TAP 609 # error, as well as a non-zero exit status from the TAP producer. 610 if (!bailed_out) 611 { 612 if (!plan_seen) 613 { 614 testsuite_error("missing test plan") 615 } 616 else if (planned_tests != testno) 617 { 618 bad_amount = testno > planned_tests ? "many" : "few" 619 testsuite_error(sprintf("too %s tests run (expected %d, got %d)", 620 bad_amount, planned_tests, testno)) 621 } 622 if (!ignore_exit) 623 { 624 # Fetch exit status from the last line. 625 exit_message = get_test_exit_message(nextline) 626 if (exit_message) 627 testsuite_error(exit_message) 628 } 629 } 630 631 write_test_results() 632 633 exit 0 634 635 } # End of "BEGIN" block. 636 ' 637 638 # TODO: document that we consume the file descriptor 3 :-( 639 } 3>"$log_file" 640 641 test $? -eq 0 || fatal "I/O or internal error" 642 643 # Local Variables: 644 # mode: shell-script 645 # sh-indentation: 2 646 # eval: (add-hook 'write-file-hooks 'time-stamp) 647 # time-stamp-start: "scriptversion=" 648 # time-stamp-format: "%:y-%02m-%02d.%02H" 649 # time-stamp-time-zone: "UTC" 650 # time-stamp-end: "; # UTC" 651 # End: 652
