1 #!/usr/bin/env python 2 import re, string, sys, os, time, math 3 4 DEBUG = 0 5 6 (tp, exp) = ('compile', 'exec') 7 8 def parse(file): 9 f = open(file, 'r') 10 d = f.read() 11 12 # Cleanup weird stuff 13 d = re.sub(r',\d+:\d', '', d) 14 15 r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) 16 17 test = {} 18 fname = '' 19 for t in r: 20 if DEBUG: 21 print t 22 23 if t[0] == 'PASS' or t[0] == 'FAIL' : 24 tmp = t[2].split('llvm-test/') 25 26 if DEBUG: 27 print tmp 28 29 if len(tmp) == 2: 30 fname = tmp[1].strip('\r\n') 31 else: 32 fname = tmp[0].strip('\r\n') 33 34 if not test.has_key(fname): 35 test[fname] = {} 36 37 test[fname][t[1] + ' state'] = t[0] 38 test[fname][t[1] + ' time'] = float('nan') 39 else : 40 try: 41 n = t[0].split('RESULT-')[1] 42 43 if DEBUG: 44 print "n == ", n; 45 46 if n == 'compile-success': 47 test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) 48 49 elif n == 'exec-success': 50 test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) 51 if DEBUG: 52 print test[fname][string.replace(n, '-success', '')] 53 54 else : 55 # print "ERROR!" 56 sys.exit(1) 57 58 except: 59 continue 60 61 return test 62 63 # Diff results and look for regressions. 64 def diffResults(d_old, d_new): 65 regressions = {} 66 passes = {} 67 removed = '' 68 69 for x in ['compile state', 'compile time', 'exec state', 'exec time']: 70 regressions[x] = '' 71 passes[x] = '' 72 73 for t in sorted(d_old.keys()) : 74 if d_new.has_key(t): 75 76 # Check if the test passed or failed. 77 for x in ['compile state', 'compile time', 'exec state', 'exec time']: 78 79 if not d_old[t].has_key(x) and not d_new[t].has_key(x): 80 continue 81 82 if d_old[t].has_key(x): 83 if d_new[t].has_key(x): 84 85 if d_old[t][x] == 'PASS': 86 if d_new[t][x] != 'PASS': 87 regressions[x] += t + "\n" 88 else: 89 if d_new[t][x] == 'PASS': 90 passes[x] += t + "\n" 91 92 else : 93 regressions[x] += t + "\n" 94 95 if x == 'compile state' or x == 'exec state': 96 continue 97 98 # For execution time, if there is no result it's a fail. 99 if not d_old[t].has_key(x) and not d_new[t].has_key(x): 100 continue 101 elif not d_new[t].has_key(x): 102 regressions[x] += t + "\n" 103 elif not d_old[t].has_key(x): 104 passes[x] += t + "\n" 105 106 if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 107 continue 108 109 elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): 110 passes[x] += t + "\n" 111 112 elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 113 regressions[x] += t + ": NaN%\n" 114 115 if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \ 116 (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05: 117 regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n" 118 119 else : 120 removed += t + "\n" 121 122 if len(regressions['compile state']) != 0: 123 print 'REGRESSION: Compilation Failed' 124 print regressions['compile state'] 125 126 if len(regressions['exec state']) != 0: 127 print 'REGRESSION: Execution Failed' 128 print regressions['exec state'] 129 130 if len(regressions['compile time']) != 0: 131 print 'REGRESSION: Compilation Time' 132 print regressions['compile time'] 133 134 if len(regressions['exec time']) != 0: 135 print 'REGRESSION: Execution Time' 136 print regressions['exec time'] 137 138 if len(passes['compile state']) != 0: 139 print 'NEW PASSES: Compilation' 140 print passes['compile state'] 141 142 if len(passes['exec state']) != 0: 143 print 'NEW PASSES: Execution' 144 print passes['exec state'] 145 146 if len(removed) != 0: 147 print 'REMOVED TESTS' 148 print removed 149 150 # Main 151 if len(sys.argv) < 3 : 152 print 'Usage:', sys.argv[0], '<old log> <new log>' 153 sys.exit(-1) 154 155 d_old = parse(sys.argv[1]) 156 d_new = parse(sys.argv[2]) 157 158 diffResults(d_old, d_new) 159
