1 #!/usr/bin/python -u 2 import glob, os, string, sys, thread, time 3 # import difflib 4 import libxml2 5 6 ### 7 # 8 # This is a "Work in Progress" attempt at a python script to run the 9 # various regression tests. The rationale for this is that it should be 10 # possible to run this on most major platforms, including those (such as 11 # Windows) which don't support gnu Make. 12 # 13 # The script is driven by a parameter file which defines the various tests 14 # to be run, together with the unique settings for each of these tests. A 15 # script for Linux is included (regressions.xml), with comments indicating 16 # the significance of the various parameters. To run the tests under Windows, 17 # edit regressions.xml and remove the comment around the default parameter 18 # "<execpath>" (i.e. make it point to the location of the binary executables). 19 # 20 # Note that this current version requires the Python bindings for libxml2 to 21 # have been previously installed and accessible 22 # 23 # See Copyright for the status of this software. 24 # William Brack (wbrack (at] mmm.com.hk) 25 # 26 ### 27 defaultParams = {} # will be used as a dictionary to hold the parsed params 28 29 # This routine is used for comparing the expected stdout / stdin with the results. 30 # The expected data has already been read in; the result is a file descriptor. 31 # Within the two sets of data, lines may begin with a path string. If so, the 32 # code "relativises" it by removing the path component. The first argument is a 33 # list already read in by a separate thread; the second is a file descriptor. 34 # The two 'base' arguments are to let me "relativise" the results files, allowing 35 # the script to be run from any directory. 36 def compFiles(res, expected, base1, base2): 37 l1 = len(base1) 38 exp = expected.readlines() 39 expected.close() 40 # the "relativisation" is done here 41 for i in range(len(res)): 42 j = string.find(res[i],base1) 43 if (j == 0) or ((j == 2) and (res[i][0:2] == './')): 44 col = string.find(res[i],':') 45 if col > 0: 46 start = string.rfind(res[i][:col], '/') 47 if start > 0: 48 res[i] = res[i][start+1:] 49 50 for i in range(len(exp)): 51 j = string.find(exp[i],base2) 52 if (j == 0) or ((j == 2) and (exp[i][0:2] == './')): 53 col = string.find(exp[i],':') 54 if col > 0: 55 start = string.rfind(exp[i][:col], '/') 56 if start > 0: 57 exp[i] = exp[i][start+1:] 58 59 ret = 0 60 # ideally we would like to use difflib functions here to do a 61 # nice comparison of the two sets. Unfortunately, during testing 62 # (using python 2.3.3 and 2.3.4) the following code went into 63 # a dead loop under windows. I'll pursue this later. 64 # diff = difflib.ndiff(res, exp) 65 # diff = list(diff) 66 # for line in diff: 67 # if line[:2] != ' ': 68 # print string.strip(line) 69 # ret = -1 70 71 # the following simple compare is fine for when the two data sets 72 # (actual result vs. expected result) are equal, which should be true for 73 # us. Unfortunately, if the test fails it's not nice at all. 74 rl = len(res) 75 el = len(exp) 76 if el != rl: 77 print 'Length of expected is %d, result is %d' % (el, rl) 78 ret = -1 79 for i in range(min(el, rl)): 80 if string.strip(res[i]) != string.strip(exp[i]): 81 print '+:%s-:%s' % (res[i], exp[i]) 82 ret = -1 83 if el > rl: 84 for i in range(rl, el): 85 print '-:%s' % exp[i] 86 ret = -1 87 elif rl > el: 88 for i in range (el, rl): 89 print '+:%s' % res[i] 90 ret = -1 91 return ret 92 93 # Separate threads to handle stdout and stderr are created to run this function 94 def readPfile(file, list, flag): 95 data = file.readlines() # no call by reference, so I cheat 96 for l in data: 97 list.append(l) 98 file.close() 99 flag.append('ok') 100 101 # This routine runs the test program (e.g. xmllint) 102 def runOneTest(testDescription, filename, inbase, errbase): 103 if 'execpath' in testDescription: 104 dir = testDescription['execpath'] + '/' 105 else: 106 dir = '' 107 cmd = os.path.abspath(dir + testDescription['testprog']) 108 if 'flag' in testDescription: 109 for f in string.split(testDescription['flag']): 110 cmd += ' ' + f 111 if 'stdin' not in testDescription: 112 cmd += ' ' + inbase + filename 113 if 'extarg' in testDescription: 114 cmd += ' ' + testDescription['extarg'] 115 116 noResult = 0 117 expout = None 118 if 'resext' in testDescription: 119 if testDescription['resext'] == 'None': 120 noResult = 1 121 else: 122 ext = '.' + testDescription['resext'] 123 else: 124 ext = '' 125 if not noResult: 126 try: 127 fname = errbase + filename + ext 128 expout = open(fname, 'rt') 129 except: 130 print "Can't open result file %s - bypassing test" % fname 131 return 132 133 noErrors = 0 134 if 'reserrext' in testDescription: 135 if testDescription['reserrext'] == 'None': 136 noErrors = 1 137 else: 138 if len(testDescription['reserrext'])>0: 139 ext = '.' + testDescription['reserrext'] 140 else: 141 ext = '' 142 else: 143 ext = '' 144 if not noErrors: 145 try: 146 fname = errbase + filename + ext 147 experr = open(fname, 'rt') 148 except: 149 experr = None 150 else: 151 experr = None 152 153 pin, pout, perr = os.popen3(cmd) 154 if 'stdin' in testDescription: 155 infile = open(inbase + filename, 'rt') 156 pin.writelines(infile.readlines()) 157 infile.close() 158 pin.close() 159 160 # popen is great fun, but can lead to the old "deadly embrace", because 161 # synchronizing the writing (by the task being run) of stdout and stderr 162 # with respect to the reading (by this task) is basically impossible. I 163 # tried several ways to cheat, but the only way I have found which works 164 # is to do a *very* elementary multi-threading approach. We can only hope 165 # that Python threads are implemented on the target system (it's okay for 166 # Linux and Windows) 167 168 th1Flag = [] # flags to show when threads finish 169 th2Flag = [] 170 outfile = [] # lists to contain the pipe data 171 errfile = [] 172 th1 = thread.start_new_thread(readPfile, (pout, outfile, th1Flag)) 173 th2 = thread.start_new_thread(readPfile, (perr, errfile, th2Flag)) 174 while (len(th1Flag)==0) or (len(th2Flag)==0): 175 time.sleep(0.001) 176 if not noResult: 177 ret = compFiles(outfile, expout, inbase, 'test/') 178 if ret != 0: 179 print 'trouble with %s' % cmd 180 else: 181 if len(outfile) != 0: 182 for l in outfile: 183 print l 184 print 'trouble with %s' % cmd 185 if experr != None: 186 ret = compFiles(errfile, experr, inbase, 'test/') 187 if ret != 0: 188 print 'trouble with %s' % cmd 189 else: 190 if not noErrors: 191 if len(errfile) != 0: 192 for l in errfile: 193 print l 194 print 'trouble with %s' % cmd 195 196 if 'stdin' not in testDescription: 197 pin.close() 198 199 # This routine is called by the parameter decoding routine whenever the end of a 200 # 'test' section is encountered. Depending upon file globbing, a large number of 201 # individual tests may be run. 202 def runTest(description): 203 testDescription = defaultParams.copy() # set defaults 204 testDescription.update(description) # override with current ent 205 if 'testname' in testDescription: 206 print "## %s" % testDescription['testname'] 207 if not 'file' in testDescription: 208 print "No file specified - can't run this test!" 209 return 210 # Set up the source and results directory paths from the decoded params 211 dir = '' 212 if 'srcdir' in testDescription: 213 dir += testDescription['srcdir'] + '/' 214 if 'srcsub' in testDescription: 215 dir += testDescription['srcsub'] + '/' 216 217 rdir = '' 218 if 'resdir' in testDescription: 219 rdir += testDescription['resdir'] + '/' 220 if 'ressub' in testDescription: 221 rdir += testDescription['ressub'] + '/' 222 223 testFiles = glob.glob(os.path.abspath(dir + testDescription['file'])) 224 if testFiles == []: 225 print "No files result from '%s'" % testDescription['file'] 226 return 227 228 # Some test programs just don't work (yet). For now we exclude them. 229 count = 0 230 excl = [] 231 if 'exclfile' in testDescription: 232 for f in string.split(testDescription['exclfile']): 233 glb = glob.glob(dir + f) 234 for g in glb: 235 excl.append(os.path.abspath(g)) 236 237 # Run the specified test program 238 for f in testFiles: 239 if not os.path.isdir(f): 240 if f not in excl: 241 count = count + 1 242 runOneTest(testDescription, os.path.basename(f), dir, rdir) 243 244 # 245 # The following classes are used with the xmlreader interface to interpret the 246 # parameter file. Once a test section has been identified, runTest is called 247 # with a dictionary containing the parsed results of the interpretation. 248 # 249 250 class testDefaults: 251 curText = '' # accumulates text content of parameter 252 253 def addToDict(self, key): 254 txt = string.strip(self.curText) 255 # if txt == '': 256 # return 257 if key not in defaultParams: 258 defaultParams[key] = txt 259 else: 260 defaultParams[key] += ' ' + txt 261 262 def processNode(self, reader, curClass): 263 if reader.Depth() == 2: 264 if reader.NodeType() == 1: 265 self.curText = '' # clear the working variable 266 elif reader.NodeType() == 15: 267 if (reader.Name() != '#text') and (reader.Name() != '#comment'): 268 self.addToDict(reader.Name()) 269 elif reader.Depth() == 3: 270 if reader.Name() == '#text': 271 self.curText += reader.Value() 272 273 elif reader.NodeType() == 15: # end of element 274 print "Defaults have been set to:" 275 for k in defaultParams.keys(): 276 print " %s : '%s'" % (k, defaultParams[k]) 277 curClass = rootClass() 278 return curClass 279 280 281 class testClass: 282 def __init__(self): 283 self.testParams = {} # start with an empty set of params 284 self.curText = '' # and empty text 285 286 def addToDict(self, key): 287 data = string.strip(self.curText) 288 if key not in self.testParams: 289 self.testParams[key] = data 290 else: 291 if self.testParams[key] != '': 292 data = ' ' + data 293 self.testParams[key] += data 294 295 def processNode(self, reader, curClass): 296 if reader.Depth() == 2: 297 if reader.NodeType() == 1: 298 self.curText = '' # clear the working variable 299 if reader.Name() not in self.testParams: 300 self.testParams[reader.Name()] = '' 301 elif reader.NodeType() == 15: 302 if (reader.Name() != '#text') and (reader.Name() != '#comment'): 303 self.addToDict(reader.Name()) 304 elif reader.Depth() == 3: 305 if reader.Name() == '#text': 306 self.curText += reader.Value() 307 308 elif reader.NodeType() == 15: # end of element 309 runTest(self.testParams) 310 curClass = rootClass() 311 return curClass 312 313 314 class rootClass: 315 def processNode(self, reader, curClass): 316 if reader.Depth() == 0: 317 return curClass 318 if reader.Depth() != 1: 319 print "Unexpected junk: Level %d, type %d, name %s" % ( 320 reader.Depth(), reader.NodeType(), reader.Name()) 321 return curClass 322 if reader.Name() == 'test': 323 curClass = testClass() 324 curClass.testParams = {} 325 elif reader.Name() == 'defaults': 326 curClass = testDefaults() 327 return curClass 328 329 def streamFile(filename): 330 try: 331 reader = libxml2.newTextReaderFilename(filename) 332 except: 333 print "unable to open %s" % (filename) 334 return 335 336 curClass = rootClass() 337 ret = reader.Read() 338 while ret == 1: 339 curClass = curClass.processNode(reader, curClass) 340 ret = reader.Read() 341 342 if ret != 0: 343 print "%s : failed to parse" % (filename) 344 345 # OK, we're finished with all the routines. Now for the main program:- 346 if len(sys.argv) != 2: 347 print "Usage: maketest {filename}" 348 sys.exit(-1) 349 350 streamFile(sys.argv[1]) 351
