Home | History | Annotate | Download | only in src
      1 #!/usr/bin/python -u
      2 import glob, os, string, sys, thread, time
      3 # import difflib
      4 import libxml2
      5 
      6 ###
      7 #
      8 # This is a "Work in Progress" attempt at a python script to run the
      9 # various regression tests.  The rationale for this is that it should be
     10 # possible to run this on most major platforms, including those (such as
     11 # Windows) which don't support gnu Make.
     12 #
     13 # The script is driven by a parameter file which defines the various tests
     14 # to be run, together with the unique settings for each of these tests.  A
     15 # script for Linux is included (regressions.xml), with comments indicating
     16 # the significance of the various parameters.  To run the tests under Windows,
     17 # edit regressions.xml and remove the comment around the default parameter
     18 # "<execpath>" (i.e. make it point to the location of the binary executables).
     19 #
     20 # Note that this current version requires the Python bindings for libxml2 to
     21 # have been previously installed and accessible
     22 #
     23 # See Copyright for the status of this software.
     24 # William Brack (wbrack (at] mmm.com.hk)
     25 #
     26 ###
     27 defaultParams = {}	# will be used as a dictionary to hold the parsed params
     28 
     29 # This routine is used for comparing the expected stdout / stdin with the results.
     30 # The expected data has already been read in; the result is a file descriptor.
     31 # Within the two sets of data, lines may begin with a path string.  If so, the
     32 # code "relativises" it by removing the path component.  The first argument is a
     33 # list already read in by a separate thread; the second is a file descriptor.
     34 # The two 'base' arguments are to let me "relativise" the results files, allowing
     35 # the script to be run from any directory.
     36 def compFiles(res, expected, base1, base2):
     37     l1 = len(base1)
     38     exp = expected.readlines()
     39     expected.close()
     40     # the "relativisation" is done here
     41     for i in range(len(res)):
     42         j = string.find(res[i],base1)
     43         if (j == 0) or ((j == 2) and (res[i][0:2] == './')):
     44             col = string.find(res[i],':')
     45             if col > 0:
     46                 start = string.rfind(res[i][:col], '/')
     47                 if start > 0:
     48                     res[i] = res[i][start+1:]
     49 
     50     for i in range(len(exp)):
     51         j = string.find(exp[i],base2)
     52         if (j == 0) or ((j == 2) and (exp[i][0:2] == './')):
     53             col = string.find(exp[i],':')
     54             if col > 0:
     55                 start = string.rfind(exp[i][:col], '/')
     56                 if start > 0:
     57                     exp[i] = exp[i][start+1:]
     58 
     59     ret = 0
     60     # ideally we would like to use difflib functions here to do a
     61     # nice comparison of the two sets.  Unfortunately, during testing
     62     # (using python 2.3.3 and 2.3.4) the following code went into
     63     # a dead loop under windows.  I'll pursue this later.
     64 #    diff = difflib.ndiff(res, exp)
     65 #    diff = list(diff)
     66 #    for line in diff:
     67 #        if line[:2] != '  ':
     68 #            print string.strip(line)
     69 #            ret = -1
     70 
     71     # the following simple compare is fine for when the two data sets
     72     # (actual result vs. expected result) are equal, which should be true for
     73     # us.  Unfortunately, if the test fails it's not nice at all.
     74     rl = len(res)
     75     el = len(exp)
     76     if el != rl:
     77         print 'Length of expected is %d, result is %d' % (el, rl)
     78 	ret = -1
     79     for i in range(min(el, rl)):
     80         if string.strip(res[i]) != string.strip(exp[i]):
     81             print '+:%s-:%s' % (res[i], exp[i])
     82             ret = -1
     83     if el > rl:
     84         for i in range(rl, el):
     85             print '-:%s' % exp[i]
     86             ret = -1
     87     elif rl > el:
     88         for i in range (el, rl):
     89             print '+:%s' % res[i]
     90             ret = -1
     91     return ret
     92 
     93 # Separate threads to handle stdout and stderr are created to run this function
     94 def readPfile(file, list, flag):
     95     data = file.readlines()	# no call by reference, so I cheat
     96     for l in data:
     97         list.append(l)
     98     file.close()
     99     flag.append('ok')
    100 
    101 # This routine runs the test program (e.g. xmllint)
    102 def runOneTest(testDescription, filename, inbase, errbase):
    103     if 'execpath' in testDescription:
    104         dir = testDescription['execpath'] + '/'
    105     else:
    106         dir = ''
    107     cmd = os.path.abspath(dir + testDescription['testprog'])
    108     if 'flag' in testDescription:
    109         for f in string.split(testDescription['flag']):
    110             cmd += ' ' + f
    111     if 'stdin' not in testDescription:
    112         cmd += ' ' + inbase + filename
    113     if 'extarg' in testDescription:
    114         cmd += ' ' + testDescription['extarg']
    115 
    116     noResult = 0
    117     expout = None
    118     if 'resext' in testDescription:
    119         if testDescription['resext'] == 'None':
    120             noResult = 1
    121         else:
    122             ext = '.' + testDescription['resext']
    123     else:
    124         ext = ''
    125     if not noResult:
    126         try:
    127             fname = errbase + filename + ext
    128             expout = open(fname, 'rt')
    129         except:
    130             print "Can't open result file %s - bypassing test" % fname
    131             return
    132 
    133     noErrors = 0
    134     if 'reserrext' in testDescription:
    135         if testDescription['reserrext'] == 'None':
    136             noErrors = 1
    137         else:
    138             if len(testDescription['reserrext'])>0:
    139                 ext = '.' + testDescription['reserrext']
    140             else:
    141                 ext = ''
    142     else:
    143         ext = ''
    144     if not noErrors:
    145         try:
    146             fname = errbase + filename + ext
    147             experr = open(fname, 'rt')
    148         except:
    149             experr = None
    150     else:
    151         experr = None
    152 
    153     pin, pout, perr = os.popen3(cmd)
    154     if 'stdin' in testDescription:
    155         infile = open(inbase + filename, 'rt')
    156         pin.writelines(infile.readlines())
    157         infile.close()
    158         pin.close()
    159 
    160     # popen is great fun, but can lead to the old "deadly embrace", because
    161     # synchronizing the writing (by the task being run) of stdout and stderr
    162     # with respect to the reading (by this task) is basically impossible.  I
    163     # tried several ways to cheat, but the only way I have found which works
    164     # is to do a *very* elementary multi-threading approach.  We can only hope
    165     # that Python threads are implemented on the target system (it's okay for
    166     # Linux and Windows)
    167 
    168     th1Flag = []	# flags to show when threads finish
    169     th2Flag = []
    170     outfile = []	# lists to contain the pipe data
    171     errfile = []
    172     th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag))
    173     th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag))
    174     while (len(th1Flag)==0) or (len(th2Flag)==0):
    175         time.sleep(0.001)
    176     if not noResult:
    177         ret = compFiles(outfile, expout, inbase, 'test/')
    178         if ret != 0:
    179             print 'trouble with %s' % cmd
    180     else:
    181         if len(outfile) != 0:
    182             for l in outfile:
    183                 print l
    184             print 'trouble with %s' % cmd
    185     if experr != None:
    186         ret = compFiles(errfile, experr, inbase, 'test/')
    187         if ret != 0:
    188             print 'trouble with %s' % cmd
    189     else:
    190         if not noErrors:
    191             if len(errfile) != 0:
    192                 for l in errfile:
    193                     print l
    194                 print 'trouble with %s' % cmd
    195 
    196     if 'stdin' not in testDescription:
    197         pin.close()
    198 
    199 # This routine is called by the parameter decoding routine whenever the end of a
    200 # 'test' section is encountered.  Depending upon file globbing, a large number of
    201 # individual tests may be run.
    202 def runTest(description):
    203     testDescription = defaultParams.copy()		# set defaults
    204     testDescription.update(description)			# override with current ent
    205     if 'testname' in testDescription:
    206         print "## %s" % testDescription['testname']
    207     if not 'file' in testDescription:
    208         print "No file specified - can't run this test!"
    209         return
    210     # Set up the source and results directory paths from the decoded params
    211     dir = ''
    212     if 'srcdir' in testDescription:
    213         dir += testDescription['srcdir'] + '/'
    214     if 'srcsub' in testDescription:
    215         dir += testDescription['srcsub'] + '/'
    216 
    217     rdir = ''
    218     if 'resdir' in testDescription:
    219         rdir += testDescription['resdir'] + '/'
    220     if 'ressub' in testDescription:
    221         rdir += testDescription['ressub'] + '/'
    222 
    223     testFiles = glob.glob(os.path.abspath(dir + testDescription['file']))
    224     if testFiles == []:
    225         print "No files result from '%s'" % testDescription['file']
    226         return
    227 
    228     # Some test programs just don't work (yet).  For now we exclude them.
    229     count = 0
    230     excl = []
    231     if 'exclfile' in testDescription:
    232         for f in string.split(testDescription['exclfile']):
    233             glb = glob.glob(dir + f)
    234             for g in glb:
    235                 excl.append(os.path.abspath(g))
    236 
    237     # Run the specified test program
    238     for f in testFiles:
    239         if not os.path.isdir(f):
    240             if f not in excl:
    241                 count = count + 1
    242                 runOneTest(testDescription, os.path.basename(f), dir, rdir)
    243 
    244 #
    245 # The following classes are used with the xmlreader interface to interpret the
    246 # parameter file.  Once a test section has been identified, runTest is called
    247 # with a dictionary containing the parsed results of the interpretation.
    248 #
    249 
    250 class testDefaults:
    251     curText = ''	# accumulates text content of parameter
    252 
    253     def addToDict(self, key):
    254         txt = string.strip(self.curText)
    255 #        if txt == '':
    256 #            return
    257         if key not in defaultParams:
    258             defaultParams[key] = txt
    259         else:
    260             defaultParams[key] += ' ' + txt
    261         
    262     def processNode(self, reader, curClass):
    263         if reader.Depth() == 2:
    264             if reader.NodeType() == 1:
    265                 self.curText = ''	# clear the working variable
    266             elif reader.NodeType() == 15:
    267                 if (reader.Name() != '#text') and (reader.Name() != '#comment'):
    268                     self.addToDict(reader.Name())
    269         elif reader.Depth() == 3:
    270             if reader.Name() == '#text':
    271                 self.curText += reader.Value()
    272 
    273         elif reader.NodeType() == 15:	# end of element
    274             print "Defaults have been set to:"
    275             for k in defaultParams.keys():
    276                 print "   %s : '%s'" % (k, defaultParams[k])
    277             curClass = rootClass()
    278         return curClass
    279 
    280 
    281 class testClass:
    282     def __init__(self):
    283         self.testParams = {}	# start with an empty set of params
    284         self.curText = ''	# and empty text
    285 
    286     def addToDict(self, key):
    287         data = string.strip(self.curText)
    288         if key not in self.testParams:
    289             self.testParams[key] = data
    290         else:
    291             if self.testParams[key] != '':
    292                 data = ' ' + data
    293             self.testParams[key] += data
    294 
    295     def processNode(self, reader, curClass):
    296         if reader.Depth() == 2:
    297             if reader.NodeType() == 1:
    298                 self.curText = ''	# clear the working variable
    299                 if reader.Name() not in self.testParams:
    300                     self.testParams[reader.Name()] = ''
    301             elif reader.NodeType() == 15:
    302                 if (reader.Name() != '#text') and (reader.Name() != '#comment'):
    303                     self.addToDict(reader.Name())
    304         elif reader.Depth() == 3:
    305             if reader.Name() == '#text':
    306                 self.curText += reader.Value()
    307 
    308         elif reader.NodeType() == 15:	# end of element
    309             runTest(self.testParams)
    310             curClass = rootClass()
    311         return curClass
    312 
    313 
    314 class rootClass:
    315     def processNode(self, reader, curClass):
    316         if reader.Depth() == 0:
    317             return curClass
    318         if reader.Depth() != 1:
    319             print "Unexpected junk: Level %d, type %d, name %s" % (
    320                   reader.Depth(), reader.NodeType(), reader.Name())
    321             return curClass
    322         if reader.Name() == 'test':
    323             curClass = testClass()
    324             curClass.testParams = {}
    325         elif reader.Name() == 'defaults':
    326             curClass = testDefaults()
    327         return curClass
    328 
    329 def streamFile(filename):
    330     try:
    331         reader = libxml2.newTextReaderFilename(filename)
    332     except:
    333         print "unable to open %s" % (filename)
    334         return
    335 
    336     curClass = rootClass()
    337     ret = reader.Read()
    338     while ret == 1:
    339         curClass = curClass.processNode(reader, curClass)
    340         ret = reader.Read()
    341 
    342     if ret != 0:
    343         print "%s : failed to parse" % (filename)
    344 
    345 # OK, we're finished with all the routines.  Now for the main program:-
    346 if len(sys.argv) != 2:
    347     print "Usage: maketest {filename}"
    348     sys.exit(-1)
    349 
    350 streamFile(sys.argv[1])
    351