1 """ 2 TestCommon.py: a testing framework for commands and scripts 3 with commonly useful error handling 4 5 The TestCommon module provides a simple, high-level interface for writing 6 tests of executable commands and scripts, especially commands and scripts 7 that interact with the file system. All methods throw exceptions and 8 exit on failure, with useful error messages. This makes a number of 9 explicit checks unnecessary, making the test scripts themselves simpler 10 to write and easier to read. 11 12 The TestCommon class is a subclass of the TestCmd class. In essence, 13 TestCommon is a wrapper that handles common TestCmd error conditions in 14 useful ways. You can use TestCommon directly, or subclass it for your 15 program and add additional (or override) methods to tailor it to your 16 program's specific needs. Alternatively, the TestCommon class serves 17 as a useful example of how to define your own TestCmd subclass. 18 19 As a subclass of TestCmd, TestCommon provides access to all of the 20 variables and methods from the TestCmd module. Consequently, you can 21 use any variable or method documented in the TestCmd module without 22 having to explicitly import TestCmd. 23 24 A TestCommon environment object is created via the usual invocation: 25 26 import TestCommon 27 test = TestCommon.TestCommon() 28 29 You can use all of the TestCmd keyword arguments when instantiating a 30 TestCommon object; see the TestCmd documentation for details. 31 32 Here is an overview of the methods and keyword arguments that are 33 provided by the TestCommon class: 34 35 test.must_be_writable('file1', ['file2', ...]) 36 37 test.must_contain('file', 'required text\n') 38 39 test.must_contain_all_lines(output, lines, ['title', find]) 40 41 test.must_contain_any_line(output, lines, ['title', find]) 42 43 test.must_exist('file1', ['file2', ...]) 44 45 test.must_match('file', "expected contents\n") 46 47 test.must_not_be_writable('file1', ['file2', ...]) 48 49 test.must_not_contain('file', 'banned text\n') 50 51 test.must_not_contain_any_line(output, lines, ['title', find]) 52 53 test.must_not_exist('file1', ['file2', ...]) 54 55 test.run(options = "options to be prepended to arguments", 56 stdout = "expected standard output from the program", 57 stderr = "expected error output from the program", 58 status = expected_status, 59 match = match_function) 60 61 The TestCommon module also provides the following variables 62 63 TestCommon.python_executable 64 TestCommon.exe_suffix 65 TestCommon.obj_suffix 66 TestCommon.shobj_prefix 67 TestCommon.shobj_suffix 68 TestCommon.lib_prefix 69 TestCommon.lib_suffix 70 TestCommon.dll_prefix 71 TestCommon.dll_suffix 72 73 """ 74 75 # Copyright 2000-2010 Steven Knight 76 # This module is free software, and you may redistribute it and/or modify 77 # it under the same terms as Python itself, so long as this copyright message 78 # and disclaimer are retained in their original form. 79 # 80 # IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, 81 # SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF 82 # THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 83 # DAMAGE. 84 # 85 # THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT 86 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 87 # PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, 88 # AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, 89 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 90 91 __author__ = "Steven Knight <knight at baldmt dot com>" 92 __revision__ = "TestCommon.py 0.37.D001 2010/01/11 16:55:50 knight" 93 __version__ = "0.37" 94 95 import copy 96 import os 97 import os.path 98 import stat 99 import string 100 import sys 101 import types 102 import UserList 103 104 from TestCmd import * 105 from TestCmd import __all__ 106 107 __all__.extend([ 'TestCommon', 108 'exe_suffix', 109 'obj_suffix', 110 'shobj_prefix', 111 'shobj_suffix', 112 'lib_prefix', 113 'lib_suffix', 114 'dll_prefix', 115 'dll_suffix', 116 ]) 117 118 # Variables that describe the prefixes and suffixes on this system. 119 if sys.platform == 'win32': 120 exe_suffix = '.exe' 121 obj_suffix = '.obj' 122 shobj_suffix = '.obj' 123 shobj_prefix = '' 124 lib_prefix = '' 125 lib_suffix = '.lib' 126 dll_prefix = '' 127 dll_suffix = '.dll' 128 elif sys.platform == 'cygwin': 129 exe_suffix = '.exe' 130 obj_suffix = '.o' 131 shobj_suffix = '.os' 132 shobj_prefix = '' 133 lib_prefix = 'lib' 134 lib_suffix = '.a' 135 dll_prefix = '' 136 dll_suffix = '.dll' 137 elif string.find(sys.platform, 'irix') != -1: 138 exe_suffix = '' 139 obj_suffix = '.o' 140 shobj_suffix = '.o' 141 shobj_prefix = '' 142 lib_prefix = 'lib' 143 lib_suffix = '.a' 144 dll_prefix = 'lib' 145 dll_suffix = '.so' 146 elif string.find(sys.platform, 'darwin') != -1: 147 exe_suffix = '' 148 obj_suffix = '.o' 149 shobj_suffix = '.os' 150 shobj_prefix = '' 151 lib_prefix = 'lib' 152 lib_suffix = '.a' 153 dll_prefix = 'lib' 154 dll_suffix = '.dylib' 155 elif string.find(sys.platform, 'sunos') != -1: 156 exe_suffix = '' 157 obj_suffix = '.o' 158 shobj_suffix = '.os' 159 shobj_prefix = 'so_' 160 lib_prefix = 'lib' 161 lib_suffix = '.a' 162 dll_prefix = 'lib' 163 dll_suffix = '.dylib' 164 else: 165 exe_suffix = '' 166 obj_suffix = '.o' 167 shobj_suffix = '.os' 168 shobj_prefix = '' 169 lib_prefix = 'lib' 170 lib_suffix = '.a' 171 dll_prefix = 'lib' 172 dll_suffix = '.so' 173 174 def is_List(e): 175 return type(e) is types.ListType \ 176 or isinstance(e, UserList.UserList) 177 178 def is_writable(f): 179 mode = os.stat(f)[stat.ST_MODE] 180 return mode & stat.S_IWUSR 181 182 def separate_files(flist): 183 existing = [] 184 missing = [] 185 for f in flist: 186 if os.path.exists(f): 187 existing.append(f) 188 else: 189 missing.append(f) 190 return existing, missing 191 192 def _failed(self, status = 0): 193 if self.status is None or status is None: 194 return None 195 try: 196 return _status(self) not in status 197 except TypeError: 198 # status wasn't an iterable 199 return _status(self) != status 200 201 def _status(self): 202 return self.status 203 204 class TestCommon(TestCmd): 205 206 # Additional methods from the Perl Test::Cmd::Common module 207 # that we may wish to add in the future: 208 # 209 # $test->subdir('subdir', ...); 210 # 211 # $test->copy('src_file', 'dst_file'); 212 213 def __init__(self, **kw): 214 """Initialize a new TestCommon instance. This involves just 215 calling the base class initialization, and then changing directory 216 to the workdir. 217 """ 218 apply(TestCmd.__init__, [self], kw) 219 os.chdir(self.workdir) 220 221 def must_be_writable(self, *files): 222 """Ensures that the specified file(s) exist and are writable. 223 An individual file can be specified as a list of directory names, 224 in which case the pathname will be constructed by concatenating 225 them. Exits FAILED if any of the files does not exist or is 226 not writable. 227 """ 228 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 229 existing, missing = separate_files(files) 230 unwritable = filter(lambda x, iw=is_writable: not iw(x), existing) 231 if missing: 232 print "Missing files: `%s'" % string.join(missing, "', `") 233 if unwritable: 234 print "Unwritable files: `%s'" % string.join(unwritable, "', `") 235 self.fail_test(missing + unwritable) 236 237 def must_contain(self, file, required, mode = 'rb'): 238 """Ensures that the specified file contains the required text. 239 """ 240 file_contents = self.read(file, mode) 241 contains = (string.find(file_contents, required) != -1) 242 if not contains: 243 print "File `%s' does not contain required string." % file 244 print self.banner('Required string ') 245 print required 246 print self.banner('%s contents ' % file) 247 print file_contents 248 self.fail_test(not contains) 249 250 def must_contain_all_lines(self, output, lines, title=None, find=None): 251 """Ensures that the specified output string (first argument) 252 contains all of the specified lines (second argument). 253 254 An optional third argument can be used to describe the type 255 of output being searched, and only shows up in failure output. 256 257 An optional fourth argument can be used to supply a different 258 function, of the form "find(line, output), to use when searching 259 for lines in the output. 260 """ 261 if find is None: 262 find = lambda o, l: string.find(o, l) != -1 263 missing = [] 264 for line in lines: 265 if not find(output, line): 266 missing.append(line) 267 268 if missing: 269 if title is None: 270 title = 'output' 271 sys.stdout.write("Missing expected lines from %s:\n" % title) 272 for line in missing: 273 sys.stdout.write(' ' + repr(line) + '\n') 274 sys.stdout.write(self.banner(title + ' ')) 275 sys.stdout.write(output) 276 self.fail_test() 277 278 def must_contain_any_line(self, output, lines, title=None, find=None): 279 """Ensures that the specified output string (first argument) 280 contains at least one of the specified lines (second argument). 281 282 An optional third argument can be used to describe the type 283 of output being searched, and only shows up in failure output. 284 285 An optional fourth argument can be used to supply a different 286 function, of the form "find(line, output), to use when searching 287 for lines in the output. 288 """ 289 if find is None: 290 find = lambda o, l: string.find(o, l) != -1 291 for line in lines: 292 if find(output, line): 293 return 294 295 if title is None: 296 title = 'output' 297 sys.stdout.write("Missing any expected line from %s:\n" % title) 298 for line in lines: 299 sys.stdout.write(' ' + repr(line) + '\n') 300 sys.stdout.write(self.banner(title + ' ')) 301 sys.stdout.write(output) 302 self.fail_test() 303 304 def must_contain_lines(self, lines, output, title=None): 305 # Deprecated; retain for backwards compatibility. 306 return self.must_contain_all_lines(output, lines, title) 307 308 def must_exist(self, *files): 309 """Ensures that the specified file(s) must exist. An individual 310 file be specified as a list of directory names, in which case the 311 pathname will be constructed by concatenating them. Exits FAILED 312 if any of the files does not exist. 313 """ 314 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 315 missing = filter(lambda x: not os.path.exists(x), files) 316 if missing: 317 print "Missing files: `%s'" % string.join(missing, "', `") 318 self.fail_test(missing) 319 320 def must_match(self, file, expect, mode = 'rb'): 321 """Matches the contents of the specified file (first argument) 322 against the expected contents (second argument). The expected 323 contents are a list of lines or a string which will be split 324 on newlines. 325 """ 326 file_contents = self.read(file, mode) 327 try: 328 self.fail_test(not self.match(file_contents, expect)) 329 except KeyboardInterrupt: 330 raise 331 except: 332 print "Unexpected contents of `%s'" % file 333 self.diff(expect, file_contents, 'contents ') 334 raise 335 336 def must_not_contain(self, file, banned, mode = 'rb'): 337 """Ensures that the specified file doesn't contain the banned text. 338 """ 339 file_contents = self.read(file, mode) 340 contains = (string.find(file_contents, banned) != -1) 341 if contains: 342 print "File `%s' contains banned string." % file 343 print self.banner('Banned string ') 344 print banned 345 print self.banner('%s contents ' % file) 346 print file_contents 347 self.fail_test(contains) 348 349 def must_not_contain_any_line(self, output, lines, title=None, find=None): 350 """Ensures that the specified output string (first argument) 351 does not contain any of the specified lines (second argument). 352 353 An optional third argument can be used to describe the type 354 of output being searched, and only shows up in failure output. 355 356 An optional fourth argument can be used to supply a different 357 function, of the form "find(line, output), to use when searching 358 for lines in the output. 359 """ 360 if find is None: 361 find = lambda o, l: string.find(o, l) != -1 362 unexpected = [] 363 for line in lines: 364 if find(output, line): 365 unexpected.append(line) 366 367 if unexpected: 368 if title is None: 369 title = 'output' 370 sys.stdout.write("Unexpected lines in %s:\n" % title) 371 for line in unexpected: 372 sys.stdout.write(' ' + repr(line) + '\n') 373 sys.stdout.write(self.banner(title + ' ')) 374 sys.stdout.write(output) 375 self.fail_test() 376 377 def must_not_contain_lines(self, lines, output, title=None): 378 return self.must_not_contain_any_line(output, lines, title) 379 380 def must_not_exist(self, *files): 381 """Ensures that the specified file(s) must not exist. 382 An individual file be specified as a list of directory names, in 383 which case the pathname will be constructed by concatenating them. 384 Exits FAILED if any of the files exists. 385 """ 386 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 387 existing = filter(os.path.exists, files) 388 if existing: 389 print "Unexpected files exist: `%s'" % string.join(existing, "', `") 390 self.fail_test(existing) 391 392 def must_not_be_writable(self, *files): 393 """Ensures that the specified file(s) exist and are not writable. 394 An individual file can be specified as a list of directory names, 395 in which case the pathname will be constructed by concatenating 396 them. Exits FAILED if any of the files does not exist or is 397 writable. 398 """ 399 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 400 existing, missing = separate_files(files) 401 writable = filter(is_writable, existing) 402 if missing: 403 print "Missing files: `%s'" % string.join(missing, "', `") 404 if writable: 405 print "Writable files: `%s'" % string.join(writable, "', `") 406 self.fail_test(missing + writable) 407 408 def _complete(self, actual_stdout, expected_stdout, 409 actual_stderr, expected_stderr, status, match): 410 """ 411 Post-processes running a subcommand, checking for failure 412 status and displaying output appropriately. 413 """ 414 if _failed(self, status): 415 expect = '' 416 if status != 0: 417 expect = " (expected %s)" % str(status) 418 print "%s returned %s%s" % (self.program, str(_status(self)), expect) 419 print self.banner('STDOUT ') 420 print actual_stdout 421 print self.banner('STDERR ') 422 print actual_stderr 423 self.fail_test() 424 if not expected_stdout is None and not match(actual_stdout, expected_stdout): 425 self.diff(expected_stdout, actual_stdout, 'STDOUT ') 426 if actual_stderr: 427 print self.banner('STDERR ') 428 print actual_stderr 429 self.fail_test() 430 if not expected_stderr is None and not match(actual_stderr, expected_stderr): 431 print self.banner('STDOUT ') 432 print actual_stdout 433 self.diff(expected_stderr, actual_stderr, 'STDERR ') 434 self.fail_test() 435 436 def start(self, program = None, 437 interpreter = None, 438 arguments = None, 439 universal_newlines = None, 440 **kw): 441 """ 442 Starts a program or script for the test environment. 443 444 This handles the "options" keyword argument and exceptions. 445 """ 446 options = kw.pop('options', None) 447 if options: 448 if arguments is None: 449 arguments = options 450 else: 451 arguments = options + " " + arguments 452 453 try: 454 return apply(TestCmd.start, 455 (self, program, interpreter, arguments, universal_newlines), 456 kw) 457 except KeyboardInterrupt: 458 raise 459 except Exception, e: 460 print self.banner('STDOUT ') 461 try: 462 print self.stdout() 463 except IndexError: 464 pass 465 print self.banner('STDERR ') 466 try: 467 print self.stderr() 468 except IndexError: 469 pass 470 cmd_args = self.command_args(program, interpreter, arguments) 471 sys.stderr.write('Exception trying to execute: %s\n' % cmd_args) 472 raise e 473 474 def finish(self, popen, stdout = None, stderr = '', status = 0, **kw): 475 """ 476 Finishes and waits for the process being run under control of 477 the specified popen argument. Additional arguments are similar 478 to those of the run() method: 479 480 stdout The expected standard output from 481 the command. A value of None means 482 don't test standard output. 483 484 stderr The expected error output from 485 the command. A value of None means 486 don't test error output. 487 488 status The expected exit status from the 489 command. A value of None means don't 490 test exit status. 491 """ 492 apply(TestCmd.finish, (self, popen,), kw) 493 match = kw.get('match', self.match) 494 self._complete(self.stdout(), stdout, 495 self.stderr(), stderr, status, match) 496 497 def run(self, options = None, arguments = None, 498 stdout = None, stderr = '', status = 0, **kw): 499 """Runs the program under test, checking that the test succeeded. 500 501 The arguments are the same as the base TestCmd.run() method, 502 with the addition of: 503 504 options Extra options that get appended to the beginning 505 of the arguments. 506 507 stdout The expected standard output from 508 the command. A value of None means 509 don't test standard output. 510 511 stderr The expected error output from 512 the command. A value of None means 513 don't test error output. 514 515 status The expected exit status from the 516 command. A value of None means don't 517 test exit status. 518 519 By default, this expects a successful exit (status = 0), does 520 not test standard output (stdout = None), and expects that error 521 output is empty (stderr = ""). 522 """ 523 if options: 524 if arguments is None: 525 arguments = options 526 else: 527 arguments = options + " " + arguments 528 kw['arguments'] = arguments 529 match = kw.pop('match', self.match) 530 apply(TestCmd.run, [self], kw) 531 self._complete(self.stdout(), stdout, 532 self.stderr(), stderr, status, match) 533 534 def skip_test(self, message="Skipping test.\n"): 535 """Skips a test. 536 537 Proper test-skipping behavior is dependent on the external 538 TESTCOMMON_PASS_SKIPS environment variable. If set, we treat 539 the skip as a PASS (exit 0), and otherwise treat it as NO RESULT. 540 In either case, we print the specified message as an indication 541 that the substance of the test was skipped. 542 543 (This was originally added to support development under Aegis. 544 Technically, skipping a test is a NO RESULT, but Aegis would 545 treat that as a test failure and prevent the change from going to 546 the next step. Since we ddn't want to force anyone using Aegis 547 to have to install absolutely every tool used by the tests, we 548 would actually report to Aegis that a skipped test has PASSED 549 so that the workflow isn't held up.) 550 """ 551 if message: 552 sys.stdout.write(message) 553 sys.stdout.flush() 554 pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS') 555 if pass_skips in [None, 0, '0']: 556 # skip=1 means skip this function when showing where this 557 # result came from. They only care about the line where the 558 # script called test.skip_test(), not the line number where 559 # we call test.no_result(). 560 self.no_result(skip=1) 561 else: 562 # We're under the development directory for this change, 563 # so this is an Aegis invocation; pass the test (exit 0). 564 self.pass_test() 565 566 # Local Variables: 567 # tab-width:4 568 # indent-tabs-mode:nil 569 # End: 570 # vim: set expandtab tabstop=4 shiftwidth=4: 571
