Home | History | Annotate | Download | only in release 
      1 #!/usr/bin/env python
      2 import re, string, sys, os, time
      3 
      4 DEBUG = 0
      5 testDirName = 'llvm-test'
      6 test      = ['compile', 'llc', 'jit', 'cbe']
      7 exectime     = ['llc-time', 'jit-time', 'cbe-time',]
      8 comptime     = ['llc', 'jit-comptime', 'compile']
      9 
     10 (tp, exp) = ('compileTime_', 'executeTime_')
     11 
     12 def parse(file):
     13   f=open(file, 'r')
     14   d = f.read()
     15   
     16   #Cleanup weird stuff
     17   d = re.sub(r',\d+:\d','', d)
     18    
     19   r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
     20    
     21   test = {}
     22   fname = ''
     23   for t in r:
     24     if DEBUG:
     25       print t
     26     if t[0] == 'PASS' or t[0] == 'FAIL' :
     27       tmp = t[2].split(testDirName)
     28       
     29       if DEBUG:
     30         print tmp
     31       
     32       if len(tmp) == 2:
     33         fname = tmp[1].strip('\r\n')
     34       else:
     35         fname = tmp[0].strip('\r\n')
     36       
     37       if not test.has_key(fname) :
     38         test[fname] = {}
     39       
     40       for k in test:
     41         test[fname][k] = 'NA'
     42         test[fname][t[1]] = t[0]
     43         if DEBUG:
     44           print test[fname][t[1]]
     45     else :
     46       try:
     47         n = t[0].split('RESULT-')[1]
     48         
     49         if DEBUG:
     50           print n;
     51         
     52         if n == 'llc' or n == 'jit-comptime' or n == 'compile':
     53           test[fname][tp + n] = float(t[2].split(' ')[2])
     54           if DEBUG:
     55             print test[fname][tp + n]
     56         
     57         elif n.endswith('-time') :
     58             test[fname][exp + n] = float(t[2].strip('\r\n'))
     59             if DEBUG:
     60               print test[fname][exp + n]
     61         
     62         else :
     63           print "ERROR!"
     64           sys.exit(1)
     65       
     66       except:
     67           continue
     68 
     69   return test
     70 
     71 # Diff results and look for regressions.
     72 def diffResults(d_old, d_new):
     73 
     74   for t in sorted(d_old.keys()) :
     75     if DEBUG:
     76       print t
     77         
     78     if d_new.has_key(t) :
     79     
     80       # Check if the test passed or failed.
     81       for x in test:
     82         if d_old[t].has_key(x):
     83           if d_new[t].has_key(x):
     84             if d_old[t][x] == 'PASS':
     85               if d_new[t][x] != 'PASS':
     86                 print t + " *** REGRESSION (" + x + ")\n"
     87             else:
     88               if d_new[t][x] == 'PASS':
     89                 print t + " * NEW PASS (" + x + ")\n"
     90                 
     91           else :
     92             print t + "*** REGRESSION (" + x + ")\n"
     93         
     94         # For execution time, if there is no result, its a fail.
     95         for x in exectime:
     96           if d_old[t].has_key(tp + x):
     97             if not d_new[t].has_key(tp + x):
     98               print t + " *** REGRESSION (" + tp + x + ")\n"
     99                 
    100           else :
    101             if d_new[t].has_key(tp + x):
    102               print t + " * NEW PASS (" + tp + x + ")\n"
    103 
    104        
    105         for x in comptime:
    106           if d_old[t].has_key(exp + x):
    107             if not d_new[t].has_key(exp + x):
    108               print t + " *** REGRESSION (" + exp + x + ")\n"
    109                 
    110           else :
    111             if d_new[t].has_key(exp + x):
    112               print t + " * NEW PASS (" + exp + x + ")\n"
    113               
    114     else :
    115       print t + ": Removed from test-suite.\n"
    116     
    117 
    118 #Main
    119 if len(sys.argv) < 3 :
    120     print 'Usage:', sys.argv[0], \
    121           '<old log> <new log>'
    122     sys.exit(-1)
    123 
    124 d_old = parse(sys.argv[1])
    125 d_new = parse(sys.argv[2])
    126 
    127 
    128 diffResults(d_old, d_new)
    129 
    130 
    131