1 #!/usr/bin/env python 2 # 3 # Given a previous good compile narrow down miscompiles. 4 # Expects two directories named "before" and "after" each containing a set of 5 # assembly or object files where the "after" version is assumed to be broken. 6 # You also have to provide a script called "link_test". It is called with a list 7 # of files which should be linked together and result tested. "link_test" should 8 # returns with exitcode 0 if the linking and testing succeeded. 9 # 10 # abtest.py operates by taking all files from the "before" directory and 11 # in each step replacing one of them with a file from the "bad" directory. 12 # 13 # Additionally you can perform the same steps with a single .s file. In this 14 # mode functions are identified by "# -- Begin FunctionName" and 15 # "# -- End FunctionName" markers. The abtest.py then takes all functions from 16 # the file in the "before" directory and replaces one function with the 17 # corresponding function from the "bad" file in each step. 18 # 19 # Example usage to identify miscompiled files: 20 # 1. Create a link_test script, make it executable. Simple Example: 21 # clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM" 22 # 2. Run the script to figure out which files are miscompiled: 23 # > ./abtest.py 24 # somefile.s: ok 25 # someotherfile.s: skipped: same content 26 # anotherfile.s: failed: './link_test' exitcode != 0 27 # ... 28 # Example usage to identify miscompiled functions inside a file: 29 # 3. First you have to mark begin and end of the functions. 30 # The script comes with some examples called mark_xxx.py. 31 # Unfortunately this is very specific to your environment and it is likely 32 # that you have to write a custom version for your environment. 33 # > for i in before/*.s after/*.s; do mark_xxx.py $i; done 34 # 4. Run the tests on a single file (assuming before/file.s and 35 # after/file.s exist) 36 # > ./abtest.py file.s 37 # funcname1 [0/XX]: ok 38 # funcname2 [1/XX]: ok 39 # funcname3 [2/XX]: skipped: same content 40 # funcname4 [3/XX]: failed: './link_test' exitcode != 0 41 # ... 42 from fnmatch import filter 43 from sys import stderr 44 import argparse 45 import filecmp 46 import os 47 import subprocess 48 import sys 49 50 LINKTEST="./link_test" 51 ESCAPE="\033[%sm" 52 BOLD=ESCAPE % "1" 53 RED=ESCAPE % "31" 54 NORMAL=ESCAPE % "0" 55 FAILED=RED+"failed"+NORMAL 56 57 def find(dir, file_filter=None): 58 files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]] 59 if file_filter != None: 60 files = filter(files, file_filter) 61 return files 62 63 def error(message): 64 stderr.write("Error: %s\n" % (message,)) 65 66 def warn(message): 67 stderr.write("Warning: %s\n" % (message,)) 68 69 def extract_functions(file): 70 functions = [] 71 in_function = None 72 for line in open(file): 73 if line.startswith("# -- Begin "): 74 if in_function != None: 75 warn("Missing end of function %s" % (in_function,)) 76 funcname = line[12:-1] 77 in_function = funcname 78 text = line 79 elif line.startswith("# -- End "): 80 function_name = line[10:-1] 81 if in_function != function_name: 82 warn("End %s does not match begin %s" % (function_name, in_function)) 83 else: 84 text += line 85 functions.append( (in_function, text) ) 86 in_function = None 87 elif in_function != None: 88 text += line 89 return functions 90 91 def replace_function(file, function, replacement, dest): 92 out = open(dest, "w") 93 skip = False 94 found = False 95 in_function = None 96 for line in open(file): 97 if line.startswith("# -- Begin "): 98 if in_function != None: 99 warn("Missing end of function %s" % (in_function,)) 100 funcname = line[12:-1] 101 in_function = funcname 102 if in_function == function: 103 out.write(replacement) 104 skip = True 105 elif line.startswith("# -- End "): 106 function_name = line[10:-1] 107 if in_function != function_name: 108 warn("End %s does not match begin %s" % (function_name, in_function)) 109 in_function = None 110 if skip: 111 skip = False 112 continue 113 if not skip: 114 out.write(line) 115 116 def announce_test(name): 117 stderr.write("%s%s%s: " % (BOLD, name, NORMAL)) 118 stderr.flush() 119 120 def announce_result(result, info): 121 stderr.write(result) 122 if info != "": 123 stderr.write(": %s" % info) 124 stderr.write("\n") 125 stderr.flush() 126 127 def testrun(files): 128 linkline="%s %s" % (LINKTEST, " ".join(files),) 129 res = subprocess.call(linkline, shell=True) 130 if res != 0: 131 announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST) 132 return False 133 else: 134 announce_result("ok", "") 135 return True 136 137 def check_files(): 138 """Check files mode""" 139 for i in range(0, len(NO_PREFIX)): 140 f = NO_PREFIX[i] 141 b=baddir+"/"+f 142 if b not in BAD_FILES: 143 warn("There is no corresponding file to '%s' in %s" \ 144 % (gooddir+"/"+f, baddir)) 145 continue 146 147 announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX))) 148 149 # combine files (everything from good except f) 150 testfiles=[] 151 skip=False 152 for c in NO_PREFIX: 153 badfile = baddir+"/"+c 154 goodfile = gooddir+"/"+c 155 if c == f: 156 testfiles.append(badfile) 157 if filecmp.cmp(goodfile, badfile): 158 announce_result("skipped", "same content") 159 skip = True 160 break 161 else: 162 testfiles.append(goodfile) 163 if skip: 164 continue 165 testrun(testfiles) 166 167 def check_functions_in_file(base, goodfile, badfile): 168 functions = extract_functions(goodfile) 169 if len(functions) == 0: 170 warn("Couldn't find any function in %s, missing annotations?" % (goodfile,)) 171 return 172 badfunctions = dict(extract_functions(badfile)) 173 if len(functions) == 0: 174 warn("Couldn't find any function in %s, missing annotations?" % (badfile,)) 175 return 176 177 COMBINED="/tmp/combined.s" 178 i = 0 179 for (func,func_text) in functions: 180 announce_test(func + " [%s/%s]" % (i+1, len(functions))) 181 i+=1 182 if func not in badfunctions: 183 warn("Function '%s' missing from bad file" % func) 184 continue 185 if badfunctions[func] == func_text: 186 announce_result("skipped", "same content") 187 continue 188 replace_function(goodfile, func, badfunctions[func], COMBINED) 189 testfiles=[] 190 for c in NO_PREFIX: 191 if c == base: 192 testfiles.append(COMBINED) 193 continue 194 testfiles.append(gooddir + "/" + c) 195 196 testrun(testfiles) 197 198 parser = argparse.ArgumentParser() 199 parser.add_argument('--a', dest='dir_a', default='before') 200 parser.add_argument('--b', dest='dir_b', default='after') 201 parser.add_argument('--insane', help='Skip sanity check', action='store_true') 202 parser.add_argument('file', metavar='file', nargs='?') 203 config = parser.parse_args() 204 205 gooddir=config.dir_a 206 baddir=config.dir_b 207 208 BAD_FILES=find(baddir, "*") 209 GOOD_FILES=find(gooddir, "*") 210 NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES]) 211 212 # "Checking whether build environment is sane ..." 213 if not config.insane: 214 announce_test("sanity check") 215 if not os.access(LINKTEST, os.X_OK): 216 error("Expect '%s' to be present and executable" % (LINKTEST,)) 217 exit(1) 218 219 res = testrun(GOOD_FILES) 220 if not res: 221 # "build environment is grinning and holding a spatula. Guess not." 222 linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),) 223 stderr.write("\n%s\n\n" % linkline) 224 stderr.write("Returned with exitcode != 0\n") 225 sys.exit(1) 226 227 if config.file is not None: 228 # File exchange mode 229 goodfile = gooddir+"/"+config.file 230 badfile = baddir+"/"+config.file 231 check_functions_in_file(config.file, goodfile, badfile) 232 else: 233 # Function exchange mode 234 check_files() 235
