1 #!/usr/bin/env python 2 3 """ 4 This is a generic fuzz testing tool, see --help for more information. 5 """ 6 7 import os 8 import sys 9 import random 10 import subprocess 11 import itertools 12 13 class TestGenerator: 14 def __init__(self, inputs, delete, insert, replace, 15 insert_strings, pick_input): 16 self.inputs = [(s, open(s).read()) for s in inputs] 17 18 self.delete = bool(delete) 19 self.insert = bool(insert) 20 self.replace = bool(replace) 21 self.pick_input = bool(pick_input) 22 self.insert_strings = list(insert_strings) 23 24 self.num_positions = sum([len(d) for _,d in self.inputs]) 25 self.num_insert_strings = len(insert_strings) 26 self.num_tests = ((delete + (insert + replace)*self.num_insert_strings) 27 * self.num_positions) 28 self.num_tests += 1 29 30 if self.pick_input: 31 self.num_tests *= self.num_positions 32 33 def position_to_source_index(self, position): 34 for i,(s,d) in enumerate(self.inputs): 35 n = len(d) 36 if position < n: 37 return (i,position) 38 position -= n 39 raise ValueError,'Invalid position.' 40 41 def get_test(self, index): 42 assert 0 <= index < self.num_tests 43 44 picked_position = None 45 if self.pick_input: 46 index,picked_position = divmod(index, self.num_positions) 47 picked_position = self.position_to_source_index(picked_position) 48 49 if index == 0: 50 return ('nothing', None, None, picked_position) 51 52 index -= 1 53 index,position = divmod(index, self.num_positions) 54 position = self.position_to_source_index(position) 55 if self.delete: 56 if index == 0: 57 return ('delete', position, None, picked_position) 58 index -= 1 59 60 index,insert_index = divmod(index, self.num_insert_strings) 61 insert_str = self.insert_strings[insert_index] 62 if self.insert: 63 if index == 0: 64 return ('insert', position, insert_str, picked_position) 65 index -= 1 66 67 assert self.replace 68 assert index == 0 69 return ('replace', position, insert_str, picked_position) 70 71 class TestApplication: 72 def __init__(self, tg, test): 73 self.tg = tg 74 self.test = test 75 76 def apply(self): 77 if self.test[0] == 'nothing': 78 pass 79 else: 80 i,j = self.test[1] 81 name,data = self.tg.inputs[i] 82 if self.test[0] == 'delete': 83 data = data[:j] + data[j+1:] 84 elif self.test[0] == 'insert': 85 data = data[:j] + self.test[2] + data[j:] 86 elif self.test[0] == 'replace': 87 data = data[:j] + self.test[2] + data[j+1:] 88 else: 89 raise ValueError,'Invalid test %r' % self.test 90 open(name,'wb').write(data) 91 92 def revert(self): 93 if self.test[0] != 'nothing': 94 i,j = self.test[1] 95 name,data = self.tg.inputs[i] 96 open(name,'wb').write(data) 97 98 def quote(str): 99 return '"' + str + '"' 100 101 def run_one_test(test_application, index, input_files, args): 102 test = test_application.test 103 104 # Interpolate arguments. 105 options = { 'index' : index, 106 'inputs' : ' '.join(quote(f) for f in input_files) } 107 108 # Add picked input interpolation arguments, if used. 109 if test[3] is not None: 110 pos = test[3][1] 111 options['picked_input'] = input_files[test[3][0]] 112 options['picked_input_pos'] = pos 113 # Compute the line and column. 114 file_data = test_application.tg.inputs[test[3][0]][1] 115 line = column = 1 116 for i in range(pos): 117 c = file_data[i] 118 if c == '\n': 119 line += 1 120 column = 1 121 else: 122 column += 1 123 options['picked_input_line'] = line 124 options['picked_input_col'] = column 125 126 test_args = [a % options for a in args] 127 if opts.verbose: 128 print '%s: note: executing %r' % (sys.argv[0], test_args) 129 130 stdout = None 131 stderr = None 132 if opts.log_dir: 133 stdout_log_path = os.path.join(opts.log_dir, '%s.out' % index) 134 stderr_log_path = os.path.join(opts.log_dir, '%s.err' % index) 135 stdout = open(stdout_log_path, 'wb') 136 stderr = open(stderr_log_path, 'wb') 137 else: 138 sys.stdout.flush() 139 p = subprocess.Popen(test_args, stdout=stdout, stderr=stderr) 140 p.communicate() 141 exit_code = p.wait() 142 143 test_result = (exit_code == opts.expected_exit_code or 144 exit_code in opts.extra_exit_codes) 145 146 if stdout is not None: 147 stdout.close() 148 stderr.close() 149 150 # Remove the logs for passes, unless logging all results. 151 if not opts.log_all and test_result: 152 os.remove(stdout_log_path) 153 os.remove(stderr_log_path) 154 155 if not test_result: 156 print 'FAIL: %d' % index 157 elif not opts.succinct: 158 print 'PASS: %d' % index 159 160 def main(): 161 global opts 162 from optparse import OptionParser, OptionGroup 163 parser = OptionParser("""%prog [options] ... test command args ... 164 165 %prog is a tool for fuzzing inputs and testing them. 166 167 The most basic usage is something like: 168 169 $ %prog --file foo.txt ./test.sh 170 171 which will run a default list of fuzzing strategies on the input. For each 172 fuzzed input, it will overwrite the input files (in place), run the test script, 173 then restore the files back to their original contents. 174 175 NOTE: You should make sure you have a backup copy of your inputs, in case 176 something goes wrong!!! 177 178 You can cause the fuzzing to not restore the original files with 179 '--no-revert'. Generally this is used with '--test <index>' to run one failing 180 test and then leave the fuzzed inputs in place to examine the failure. 181 182 For each fuzzed input, %prog will run the test command given on the command 183 line. Each argument in the command is subject to string interpolation before 184 being executed. The syntax is "%(VARIABLE)FORMAT" where FORMAT is a standard 185 printf format, and VARIBLE is one of: 186 187 'index' - the test index being run 188 'inputs' - the full list of test inputs 189 'picked_input' - (with --pick-input) the selected input file 190 'picked_input_pos' - (with --pick-input) the selected input position 191 'picked_input_line' - (with --pick-input) the selected input line 192 'picked_input_col' - (with --pick-input) the selected input column 193 194 By default, the script will run forever continually picking new tests to 195 run. You can limit the number of tests that are run with '--max-tests <number>', 196 and you can run a particular test with '--test <index>'. 197 """) 198 parser.add_option("-v", "--verbose", help="Show more output", 199 action='store_true', dest="verbose", default=False) 200 parser.add_option("-s", "--succinct", help="Reduce amount of output", 201 action="store_true", dest="succinct", default=False) 202 203 group = OptionGroup(parser, "Test Execution") 204 group.add_option("", "--expected-exit-code", help="Set expected exit code", 205 type=int, dest="expected_exit_code", 206 default=0) 207 group.add_option("", "--extra-exit-code", 208 help="Set additional expected exit code", 209 type=int, action="append", dest="extra_exit_codes", 210 default=[]) 211 group.add_option("", "--log-dir", 212 help="Capture test logs to an output directory", 213 type=str, dest="log_dir", 214 default=None) 215 group.add_option("", "--log-all", 216 help="Log all outputs (not just failures)", 217 action="store_true", dest="log_all", default=False) 218 parser.add_option_group(group) 219 220 group = OptionGroup(parser, "Input Files") 221 group.add_option("", "--file", metavar="PATH", 222 help="Add an input file to fuzz", 223 type=str, action="append", dest="input_files", default=[]) 224 group.add_option("", "--filelist", metavar="LIST", 225 help="Add a list of inputs files to fuzz (one per line)", 226 type=int, action="append", dest="filelists", default=[]) 227 parser.add_option_group(group) 228 229 group = OptionGroup(parser, "Fuzz Options") 230 group.add_option("", "--replacement-chars", dest="replacement_chars", 231 help="Characters to insert/replace", 232 default="0{}[]<>\;@#$^%& ") 233 group.add_option("", "--replacement-string", dest="replacement_strings", 234 action="append", help="Add a replacement string to use", 235 default=[]) 236 group.add_option("", "--replacement-list", dest="replacement_lists", 237 help="Add a list of replacement strings (one per line)", 238 action="append", default=[]) 239 group.add_option("", "--no-delete", help="Don't delete characters", 240 action='store_false', dest="enable_delete", default=True) 241 group.add_option("", "--no-insert", help="Don't insert strings", 242 action='store_false', dest="enable_insert", default=True) 243 group.add_option("", "--no-replace", help="Don't replace strings", 244 action='store_false', dest="enable_replace", default=True) 245 group.add_option("", "--no-revert", help="Don't revert changes", 246 action='store_false', dest="revert", default=True) 247 parser.add_option_group(group) 248 249 group = OptionGroup(parser, "Test Selection") 250 group.add_option("", "--test", help="Run a particular test", 251 type=int, dest="test", default=None, metavar="INDEX") 252 group.add_option("", "--max-tests", help="Maximum number of tests", 253 type=int, dest="max_tests", default=10, metavar="COUNT") 254 group.add_option("", "--pick-input", 255 help="Randomly select an input byte as well as fuzzing", 256 action='store_true', dest="pick_input", default=False) 257 parser.add_option_group(group) 258 259 parser.disable_interspersed_args() 260 261 (opts, args) = parser.parse_args() 262 263 if not args: 264 parser.error("Invalid number of arguments") 265 266 # Collect the list of inputs. 267 input_files = list(opts.input_files) 268 for filelist in opts.filelists: 269 f = open(filelist) 270 try: 271 for ln in f: 272 ln = ln.strip() 273 if ln: 274 input_files.append(ln) 275 finally: 276 f.close() 277 input_files.sort() 278 279 if not input_files: 280 parser.error("No input files!") 281 282 print '%s: note: fuzzing %d files.' % (sys.argv[0], len(input_files)) 283 284 # Make sure the log directory exists if used. 285 if opts.log_dir: 286 if not os.path.exists(opts.log_dir): 287 try: 288 os.mkdir(opts.log_dir) 289 except OSError: 290 print "%s: error: log directory couldn't be created!" % ( 291 sys.argv[0],) 292 raise SystemExit,1 293 294 # Get the list if insert/replacement strings. 295 replacements = list(opts.replacement_chars) 296 replacements.extend(opts.replacement_strings) 297 for replacement_list in opts.replacement_lists: 298 f = open(replacement_list) 299 try: 300 for ln in f: 301 ln = ln[:-1] 302 if ln: 303 replacements.append(ln) 304 finally: 305 f.close() 306 307 # Unique and order the replacement list. 308 replacements = list(set(replacements)) 309 replacements.sort() 310 311 # Create the test generator. 312 tg = TestGenerator(input_files, opts.enable_delete, opts.enable_insert, 313 opts.enable_replace, replacements, opts.pick_input) 314 315 print '%s: note: %d input bytes.' % (sys.argv[0], tg.num_positions) 316 print '%s: note: %d total tests.' % (sys.argv[0], tg.num_tests) 317 if opts.test is not None: 318 it = [opts.test] 319 elif opts.max_tests is not None: 320 it = itertools.imap(random.randrange, 321 itertools.repeat(tg.num_tests, opts.max_tests)) 322 else: 323 it = itertools.imap(random.randrange, itertools.repeat(tg.num_tests)) 324 for test in it: 325 t = tg.get_test(test) 326 327 if opts.verbose: 328 print '%s: note: running test %d: %r' % (sys.argv[0], test, t) 329 ta = TestApplication(tg, t) 330 try: 331 ta.apply() 332 run_one_test(ta, test, input_files, args) 333 finally: 334 if opts.revert: 335 ta.revert() 336 337 sys.stdout.flush() 338 339 if __name__ == '__main__': 340 main() 341
