Home | History | Annotate | Download | only in lib
      1 """
      2 TestCommon.py:  a testing framework for commands and scripts
      3                 with commonly useful error handling
      4 
      5 The TestCommon module provides a simple, high-level interface for writing
      6 tests of executable commands and scripts, especially commands and scripts
      7 that interact with the file system.  All methods throw exceptions and
      8 exit on failure, with useful error messages.  This makes a number of
      9 explicit checks unnecessary, making the test scripts themselves simpler
     10 to write and easier to read.
     11 
     12 The TestCommon class is a subclass of the TestCmd class.  In essence,
     13 TestCommon is a wrapper that handles common TestCmd error conditions in
     14 useful ways.  You can use TestCommon directly, or subclass it for your
     15 program and add additional (or override) methods to tailor it to your
     16 program's specific needs.  Alternatively, the TestCommon class serves
     17 as a useful example of how to define your own TestCmd subclass.
     18 
     19 As a subclass of TestCmd, TestCommon provides access to all of the
     20 variables and methods from the TestCmd module.  Consequently, you can
     21 use any variable or method documented in the TestCmd module without
     22 having to explicitly import TestCmd.
     23 
     24 A TestCommon environment object is created via the usual invocation:
     25 
     26     import TestCommon
     27     test = TestCommon.TestCommon()
     28 
     29 You can use all of the TestCmd keyword arguments when instantiating a
     30 TestCommon object; see the TestCmd documentation for details.
     31 
     32 Here is an overview of the methods and keyword arguments that are
     33 provided by the TestCommon class:
     34 
     35     test.must_be_writable('file1', ['file2', ...])
     36 
     37     test.must_contain('file', 'required text\n')
     38 
     39     test.must_contain_all_lines(output, lines, ['title', find])
     40 
     41     test.must_contain_any_line(output, lines, ['title', find])
     42 
     43     test.must_exist('file1', ['file2', ...])
     44 
     45     test.must_match('file', "expected contents\n")
     46 
     47     test.must_not_be_writable('file1', ['file2', ...])
     48 
     49     test.must_not_contain('file', 'banned text\n')
     50 
     51     test.must_not_contain_any_line(output, lines, ['title', find])
     52 
     53     test.must_not_exist('file1', ['file2', ...])
     54 
     55     test.run(options = "options to be prepended to arguments",
     56              stdout = "expected standard output from the program",
     57              stderr = "expected error output from the program",
     58              status = expected_status,
     59              match = match_function)
     60 
     61 The TestCommon module also provides the following variables
     62 
     63     TestCommon.python_executable
     64     TestCommon.exe_suffix
     65     TestCommon.obj_suffix
     66     TestCommon.shobj_prefix
     67     TestCommon.shobj_suffix
     68     TestCommon.lib_prefix
     69     TestCommon.lib_suffix
     70     TestCommon.dll_prefix
     71     TestCommon.dll_suffix
     72 
     73 """
     74 
     75 # Copyright 2000-2010 Steven Knight
     76 # This module is free software, and you may redistribute it and/or modify
     77 # it under the same terms as Python itself, so long as this copyright message
     78 # and disclaimer are retained in their original form.
     79 #
     80 # IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
     81 # SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF
     82 # THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
     83 # DAMAGE.
     84 #
     85 # THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
     86 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
     87 # PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
     88 # AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
     89 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
     90 
     91 __author__ = "Steven Knight <knight at baldmt dot com>"
     92 __revision__ = "TestCommon.py 0.37.D001 2010/01/11 16:55:50 knight"
     93 __version__ = "0.37"
     94 
     95 import copy
     96 import os
     97 import os.path
     98 import stat
     99 import string
    100 import sys
    101 import types
    102 import UserList
    103 
    104 from TestCmd import *
    105 from TestCmd import __all__
    106 
    107 __all__.extend([ 'TestCommon',
    108                  'exe_suffix',
    109                  'obj_suffix',
    110                  'shobj_prefix',
    111                  'shobj_suffix',
    112                  'lib_prefix',
    113                  'lib_suffix',
    114                  'dll_prefix',
    115                  'dll_suffix',
    116                ])
    117 
    118 # Variables that describe the prefixes and suffixes on this system.
    119 if sys.platform == 'win32':
    120     exe_suffix    = '.exe'
    121     obj_suffix    = '.obj'
    122     shobj_suffix  = '.obj'
    123     shobj_prefix  = ''
    124     lib_prefix    = ''
    125     lib_suffix    = '.lib'
    126     dll_prefix    = ''
    127     dll_suffix    = '.dll'
    128     module_prefix = ''
    129     module_suffix = '.dll'
    130 elif sys.platform == 'cygwin':
    131     exe_suffix    = '.exe'
    132     obj_suffix    = '.o'
    133     shobj_suffix  = '.os'
    134     shobj_prefix  = ''
    135     lib_prefix    = 'lib'
    136     lib_suffix    = '.a'
    137     dll_prefix    = ''
    138     dll_suffix    = '.dll'
    139     module_prefix = ''
    140     module_suffix = '.dll'
    141 elif string.find(sys.platform, 'irix') != -1:
    142     exe_suffix    = ''
    143     obj_suffix    = '.o'
    144     shobj_suffix  = '.o'
    145     shobj_prefix  = ''
    146     lib_prefix    = 'lib'
    147     lib_suffix    = '.a'
    148     dll_prefix    = 'lib'
    149     dll_suffix    = '.so'
    150     module_prefix = 'lib'
    151     module_prefix = '.so'
    152 elif string.find(sys.platform, 'darwin') != -1:
    153     exe_suffix    = ''
    154     obj_suffix    = '.o'
    155     shobj_suffix  = '.os'
    156     shobj_prefix  = ''
    157     lib_prefix    = 'lib'
    158     lib_suffix    = '.a'
    159     dll_prefix    = 'lib'
    160     dll_suffix    = '.dylib'
    161     module_prefix = ''
    162     module_suffix = '.so'
    163 elif string.find(sys.platform, 'sunos') != -1:
    164     exe_suffix    = ''
    165     obj_suffix    = '.o'
    166     shobj_suffix  = '.os'
    167     shobj_prefix  = 'so_'
    168     lib_prefix    = 'lib'
    169     lib_suffix    = '.a'
    170     dll_prefix    = 'lib'
    171     dll_suffix    = '.dylib'
    172     module_prefix = ''
    173     module_suffix = '.so'
    174 else:
    175     exe_suffix    = ''
    176     obj_suffix    = '.o'
    177     shobj_suffix  = '.os'
    178     shobj_prefix  = ''
    179     lib_prefix    = 'lib'
    180     lib_suffix    = '.a'
    181     dll_prefix    = 'lib'
    182     dll_suffix    = '.so'
    183     module_prefix = 'lib'
    184     module_suffix = '.so'
    185 
    186 def is_List(e):
    187     return type(e) is types.ListType \
    188         or isinstance(e, UserList.UserList)
    189 
    190 def is_writable(f):
    191     mode = os.stat(f)[stat.ST_MODE]
    192     return mode & stat.S_IWUSR
    193 
    194 def separate_files(flist):
    195     existing = []
    196     missing = []
    197     for f in flist:
    198         if os.path.exists(f):
    199             existing.append(f)
    200         else:
    201             missing.append(f)
    202     return existing, missing
    203 
    204 def _failed(self, status = 0):
    205     if self.status is None or status is None:
    206         return None
    207     try:
    208         return _status(self) not in status
    209     except TypeError:
    210         # status wasn't an iterable
    211         return _status(self) != status
    212 
    213 def _status(self):
    214     return self.status
    215 
    216 class TestCommon(TestCmd):
    217 
    218     # Additional methods from the Perl Test::Cmd::Common module
    219     # that we may wish to add in the future:
    220     #
    221     #  $test->subdir('subdir', ...);
    222     #
    223     #  $test->copy('src_file', 'dst_file');
    224 
    225     def __init__(self, **kw):
    226         """Initialize a new TestCommon instance.  This involves just
    227         calling the base class initialization, and then changing directory
    228         to the workdir.
    229         """
    230         apply(TestCmd.__init__, [self], kw)
    231         os.chdir(self.workdir)
    232 
    233     def must_be_writable(self, *files):
    234         """Ensures that the specified file(s) exist and are writable.
    235         An individual file can be specified as a list of directory names,
    236         in which case the pathname will be constructed by concatenating
    237         them.  Exits FAILED if any of the files does not exist or is
    238         not writable.
    239         """
    240         files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
    241         existing, missing = separate_files(files)
    242         unwritable = filter(lambda x, iw=is_writable: not iw(x), existing)
    243         if missing:
    244             print "Missing files: `%s'" % string.join(missing, "', `")
    245         if unwritable:
    246             print "Unwritable files: `%s'" % string.join(unwritable, "', `")
    247         self.fail_test(missing + unwritable)
    248 
    249     def must_contain(self, file, required, mode = 'rb'):
    250         """Ensures that the specified file contains the required text.
    251         """
    252         file_contents = self.read(file, mode)
    253         contains = (string.find(file_contents, required) != -1)
    254         if not contains:
    255             print "File `%s' does not contain required string." % file
    256             print self.banner('Required string ')
    257             print required
    258             print self.banner('%s contents ' % file)
    259             print file_contents
    260             self.fail_test(not contains)
    261 
    262     def must_contain_all_lines(self, output, lines, title=None, find=None):
    263         """Ensures that the specified output string (first argument)
    264         contains all of the specified lines (second argument).
    265 
    266         An optional third argument can be used to describe the type
    267         of output being searched, and only shows up in failure output.
    268 
    269         An optional fourth argument can be used to supply a different
    270         function, of the form "find(line, output), to use when searching
    271         for lines in the output.
    272         """
    273         if find is None:
    274             find = lambda o, l: string.find(o, l) != -1
    275         missing = []
    276         for line in lines:
    277             if not find(output, line):
    278                 missing.append(line)
    279 
    280         if missing:
    281             if title is None:
    282                 title = 'output'
    283             sys.stdout.write("Missing expected lines from %s:\n" % title)
    284             for line in missing:
    285                 sys.stdout.write('    ' + repr(line) + '\n')
    286             sys.stdout.write(self.banner(title + ' '))
    287             sys.stdout.write(output)
    288             self.fail_test()
    289 
    290     def must_contain_any_line(self, output, lines, title=None, find=None):
    291         """Ensures that the specified output string (first argument)
    292         contains at least one of the specified lines (second argument).
    293 
    294         An optional third argument can be used to describe the type
    295         of output being searched, and only shows up in failure output.
    296 
    297         An optional fourth argument can be used to supply a different
    298         function, of the form "find(line, output), to use when searching
    299         for lines in the output.
    300         """
    301         if find is None:
    302             find = lambda o, l: string.find(o, l) != -1
    303         for line in lines:
    304             if find(output, line):
    305                 return
    306 
    307         if title is None:
    308             title = 'output'
    309         sys.stdout.write("Missing any expected line from %s:\n" % title)
    310         for line in lines:
    311             sys.stdout.write('    ' + repr(line) + '\n')
    312         sys.stdout.write(self.banner(title + ' '))
    313         sys.stdout.write(output)
    314         self.fail_test()
    315 
    316     def must_contain_lines(self, lines, output, title=None):
    317         # Deprecated; retain for backwards compatibility.
    318         return self.must_contain_all_lines(output, lines, title)
    319 
    320     def must_exist(self, *files):
    321         """Ensures that the specified file(s) must exist.  An individual
    322         file be specified as a list of directory names, in which case the
    323         pathname will be constructed by concatenating them.  Exits FAILED
    324         if any of the files does not exist.
    325         """
    326         files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
    327         missing = filter(lambda x: not os.path.exists(x), files)
    328         if missing:
    329             print "Missing files: `%s'" % string.join(missing, "', `")
    330             self.fail_test(missing)
    331 
    332     def must_match(self, file, expect, mode = 'rb'):
    333         """Matches the contents of the specified file (first argument)
    334         against the expected contents (second argument).  The expected
    335         contents are a list of lines or a string which will be split
    336         on newlines.
    337         """
    338         file_contents = self.read(file, mode)
    339         try:
    340             self.fail_test(not self.match(file_contents, expect))
    341         except KeyboardInterrupt:
    342             raise
    343         except:
    344             print "Unexpected contents of `%s'" % file
    345             self.diff(expect, file_contents, 'contents ')
    346             raise
    347 
    348     def must_not_contain(self, file, banned, mode = 'rb'):
    349         """Ensures that the specified file doesn't contain the banned text.
    350         """
    351         file_contents = self.read(file, mode)
    352         contains = (string.find(file_contents, banned) != -1)
    353         if contains:
    354             print "File `%s' contains banned string." % file
    355             print self.banner('Banned string ')
    356             print banned
    357             print self.banner('%s contents ' % file)
    358             print file_contents
    359             self.fail_test(contains)
    360 
    361     def must_not_contain_any_line(self, output, lines, title=None, find=None):
    362         """Ensures that the specified output string (first argument)
    363         does not contain any of the specified lines (second argument).
    364 
    365         An optional third argument can be used to describe the type
    366         of output being searched, and only shows up in failure output.
    367 
    368         An optional fourth argument can be used to supply a different
    369         function, of the form "find(line, output), to use when searching
    370         for lines in the output.
    371         """
    372         if find is None:
    373             find = lambda o, l: string.find(o, l) != -1
    374         unexpected = []
    375         for line in lines:
    376             if find(output, line):
    377                 unexpected.append(line)
    378 
    379         if unexpected:
    380             if title is None:
    381                 title = 'output'
    382             sys.stdout.write("Unexpected lines in %s:\n" % title)
    383             for line in unexpected:
    384                 sys.stdout.write('    ' + repr(line) + '\n')
    385             sys.stdout.write(self.banner(title + ' '))
    386             sys.stdout.write(output)
    387             self.fail_test()
    388 
    389     def must_not_contain_lines(self, lines, output, title=None):
    390         return self.must_not_contain_any_line(output, lines, title)
    391 
    392     def must_not_exist(self, *files):
    393         """Ensures that the specified file(s) must not exist.
    394         An individual file be specified as a list of directory names, in
    395         which case the pathname will be constructed by concatenating them.
    396         Exits FAILED if any of the files exists.
    397         """
    398         files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
    399         existing = filter(os.path.exists, files)
    400         if existing:
    401             print "Unexpected files exist: `%s'" % string.join(existing, "', `")
    402             self.fail_test(existing)
    403 
    404     def must_not_be_writable(self, *files):
    405         """Ensures that the specified file(s) exist and are not writable.
    406         An individual file can be specified as a list of directory names,
    407         in which case the pathname will be constructed by concatenating
    408         them.  Exits FAILED if any of the files does not exist or is
    409         writable.
    410         """
    411         files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
    412         existing, missing = separate_files(files)
    413         writable = filter(is_writable, existing)
    414         if missing:
    415             print "Missing files: `%s'" % string.join(missing, "', `")
    416         if writable:
    417             print "Writable files: `%s'" % string.join(writable, "', `")
    418         self.fail_test(missing + writable)
    419 
    420     def _complete(self, actual_stdout, expected_stdout,
    421                         actual_stderr, expected_stderr, status, match):
    422         """
    423         Post-processes running a subcommand, checking for failure
    424         status and displaying output appropriately.
    425         """
    426         if _failed(self, status):
    427             expect = ''
    428             if status != 0:
    429                 expect = " (expected %s)" % str(status)
    430             print "%s returned %s%s" % (self.program, str(_status(self)), expect)
    431             print self.banner('STDOUT ')
    432             print actual_stdout
    433             print self.banner('STDERR ')
    434             print actual_stderr
    435             self.fail_test()
    436         if not expected_stdout is None and not match(actual_stdout, expected_stdout):
    437             self.diff(expected_stdout, actual_stdout, 'STDOUT ')
    438             if actual_stderr:
    439                 print self.banner('STDERR ')
    440                 print actual_stderr
    441             self.fail_test()
    442         if not expected_stderr is None and not match(actual_stderr, expected_stderr):
    443             print self.banner('STDOUT ')
    444             print actual_stdout
    445             self.diff(expected_stderr, actual_stderr, 'STDERR ')
    446             self.fail_test()
    447 
    448     def start(self, program = None,
    449                     interpreter = None,
    450                     arguments = None,
    451                     universal_newlines = None,
    452                     **kw):
    453         """
    454         Starts a program or script for the test environment.
    455 
    456         This handles the "options" keyword argument and exceptions.
    457         """
    458         options = kw.pop('options', None)
    459         if options:
    460             if arguments is None:
    461                 arguments = options
    462             else:
    463                 arguments = options + " " + arguments
    464 
    465         try:
    466             return apply(TestCmd.start,
    467                          (self, program, interpreter, arguments, universal_newlines),
    468                          kw)
    469         except KeyboardInterrupt:
    470             raise
    471         except Exception, e:
    472             print self.banner('STDOUT ')
    473             try:
    474                 print self.stdout()
    475             except IndexError:
    476                 pass
    477             print self.banner('STDERR ')
    478             try:
    479                 print self.stderr()
    480             except IndexError:
    481                 pass
    482             cmd_args = self.command_args(program, interpreter, arguments)
    483             sys.stderr.write('Exception trying to execute: %s\n' % cmd_args)
    484             raise e
    485 
    486     def finish(self, popen, stdout = None, stderr = '', status = 0, **kw):
    487         """
    488         Finishes and waits for the process being run under control of
    489         the specified popen argument.  Additional arguments are similar
    490         to those of the run() method:
    491 
    492                 stdout  The expected standard output from
    493                         the command.  A value of None means
    494                         don't test standard output.
    495 
    496                 stderr  The expected error output from
    497                         the command.  A value of None means
    498                         don't test error output.
    499 
    500                 status  The expected exit status from the
    501                         command.  A value of None means don't
    502                         test exit status.
    503         """
    504         apply(TestCmd.finish, (self, popen,), kw)
    505         match = kw.get('match', self.match)
    506         self._complete(self.stdout(), stdout,
    507                        self.stderr(), stderr, status, match)
    508 
    509     def run(self, options = None, arguments = None,
    510                   stdout = None, stderr = '', status = 0, **kw):
    511         """Runs the program under test, checking that the test succeeded.
    512 
    513         The arguments are the same as the base TestCmd.run() method,
    514         with the addition of:
    515 
    516                 options Extra options that get appended to the beginning
    517                         of the arguments.
    518 
    519                 stdout  The expected standard output from
    520                         the command.  A value of None means
    521                         don't test standard output.
    522 
    523                 stderr  The expected error output from
    524                         the command.  A value of None means
    525                         don't test error output.
    526 
    527                 status  The expected exit status from the
    528                         command.  A value of None means don't
    529                         test exit status.
    530 
    531         By default, this expects a successful exit (status = 0), does
    532         not test standard output (stdout = None), and expects that error
    533         output is empty (stderr = "").
    534         """
    535         if options:
    536             if arguments is None:
    537                 arguments = options
    538             else:
    539                 arguments = options + " " + arguments
    540         kw['arguments'] = arguments
    541         match = kw.pop('match', self.match)
    542         apply(TestCmd.run, [self], kw)
    543         self._complete(self.stdout(), stdout,
    544                        self.stderr(), stderr, status, match)
    545 
    546     def skip_test(self, message="Skipping test.\n"):
    547         """Skips a test.
    548 
    549         Proper test-skipping behavior is dependent on the external
    550         TESTCOMMON_PASS_SKIPS environment variable.  If set, we treat
    551         the skip as a PASS (exit 0), and otherwise treat it as NO RESULT.
    552         In either case, we print the specified message as an indication
    553         that the substance of the test was skipped.
    554 
    555         (This was originally added to support development under Aegis.
    556         Technically, skipping a test is a NO RESULT, but Aegis would
    557         treat that as a test failure and prevent the change from going to
    558         the next step.  Since we ddn't want to force anyone using Aegis
    559         to have to install absolutely every tool used by the tests, we
    560         would actually report to Aegis that a skipped test has PASSED
    561         so that the workflow isn't held up.)
    562         """
    563         if message:
    564             sys.stdout.write(message)
    565             sys.stdout.flush()
    566         pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS')
    567         if pass_skips in [None, 0, '0']:
    568             # skip=1 means skip this function when showing where this
    569             # result came from.  They only care about the line where the
    570             # script called test.skip_test(), not the line number where
    571             # we call test.no_result().
    572             self.no_result(skip=1)
    573         else:
    574             # We're under the development directory for this change,
    575             # so this is an Aegis invocation; pass the test (exit 0).
    576             self.pass_test()
    577 
    578 # Local Variables:
    579 # tab-width:4
    580 # indent-tabs-mode:nil
    581 # End:
    582 # vim: set expandtab tabstop=4 shiftwidth=4:
    583