1 #!/usr/bin/env python3.4 2 # 3 # Copyright (C) 2016 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import abc 18 import argparse 19 import filecmp 20 import os 21 import shlex 22 import shutil 23 import subprocess 24 import sys 25 26 from glob import glob 27 from subprocess import DEVNULL 28 from tempfile import mkdtemp 29 30 sys.path.append(os.path.dirname(os.path.dirname( 31 os.path.realpath(__file__)))) 32 33 from common.common import RetCode 34 from common.common import CommandListToCommandString 35 from common.common import FatalError 36 from common.common import GetJackClassPath 37 from common.common import GetEnvVariableOrError 38 from common.common import RunCommand 39 from common.common import RunCommandForOutput 40 from common.common import DeviceTestEnv 41 42 # Return codes supported by bisection bug search. 43 BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) 44 45 46 def GetExecutionModeRunner(use_dx, device, mode): 47 """Returns a runner for the given execution mode. 48 49 Args: 50 use_dx: boolean, if True use dx rather than jack 51 device: string, target device serial number (or None) 52 mode: string, execution mode 53 Returns: 54 TestRunner with given execution mode 55 Raises: 56 FatalError: error for unknown execution mode 57 """ 58 if mode == 'ri': 59 return TestRunnerRIOnHost() 60 if mode == 'hint': 61 return TestRunnerArtIntOnHost(use_dx) 62 if mode == 'hopt': 63 return TestRunnerArtOptOnHost(use_dx) 64 if mode == 'tint': 65 return TestRunnerArtIntOnTarget(use_dx, device) 66 if mode == 'topt': 67 return TestRunnerArtOptOnTarget(use_dx, device) 68 raise FatalError('Unknown execution mode') 69 70 71 # 72 # Execution mode classes. 73 # 74 75 76 class TestRunner(object): 77 """Abstraction for running a test in a particular execution mode.""" 78 __meta_class__ = abc.ABCMeta 79 80 @abc.abstractproperty 81 def description(self): 82 """Returns a description string of the execution mode.""" 83 84 @abc.abstractproperty 85 def id(self): 86 """Returns a short string that uniquely identifies the execution mode.""" 87 88 @property 89 def output_file(self): 90 return self.id + '_out.txt' 91 92 @abc.abstractmethod 93 def GetBisectionSearchArgs(self): 94 """Get arguments to pass to bisection search tool. 95 96 Returns: 97 list of strings - arguments for bisection search tool, or None if 98 runner is not bisectable 99 """ 100 101 @abc.abstractmethod 102 def CompileAndRunTest(self): 103 """Compile and run the generated test. 104 105 Ensures that the current Test.java in the temporary directory is compiled 106 and executed under the current execution mode. On success, transfers the 107 generated output to the file self.output_file in the temporary directory. 108 109 Most nonzero return codes are assumed non-divergent, since systems may 110 exit in different ways. This is enforced by normalizing return codes. 111 112 Returns: 113 normalized return code 114 """ 115 116 117 class TestRunnerWithHostCompilation(TestRunner): 118 """Abstract test runner that supports compilation on host.""" 119 120 def __init__(self, use_dx): 121 """Constructor for the runner with host compilation. 122 123 Args: 124 use_dx: boolean, if True use dx rather than jack 125 """ 126 self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', 127 'Test.java'] 128 self._use_dx = use_dx 129 130 def CompileOnHost(self): 131 if self._use_dx: 132 if RunCommand(['javac', 'Test.java'], 133 out=None, err=None, timeout=30) == RetCode.SUCCESS: 134 retc = RunCommand(['dx', '--dex', '--output=classes.dex'] + glob('*.class'), 135 out=None, err='dxerr.txt', timeout=30) 136 else: 137 retc = RetCode.NOTCOMPILED 138 else: 139 retc = RunCommand(['jack'] + self._jack_args, 140 out=None, err='jackerr.txt', timeout=30) 141 return retc 142 143 144 class TestRunnerRIOnHost(TestRunner): 145 """Concrete test runner of the reference implementation on host.""" 146 147 @property 148 def description(self): 149 return 'RI on host' 150 151 @property 152 def id(self): 153 return 'RI' 154 155 def CompileAndRunTest(self): 156 if RunCommand(['javac', 'Test.java'], 157 out=None, err=None, timeout=30) == RetCode.SUCCESS: 158 retc = RunCommand(['java', 'Test'], self.output_file, err=None) 159 else: 160 retc = RetCode.NOTCOMPILED 161 return retc 162 163 def GetBisectionSearchArgs(self): 164 return None 165 166 167 class TestRunnerArtOnHost(TestRunnerWithHostCompilation): 168 """Abstract test runner of Art on host.""" 169 170 def __init__(self, use_dx, extra_args=None): 171 """Constructor for the Art on host tester. 172 173 Args: 174 use_dx: boolean, if True use dx rather than jack 175 extra_args: list of strings, extra arguments for dalvikvm 176 """ 177 super().__init__(use_dx) 178 self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex'] 179 if extra_args is not None: 180 self._art_cmd += extra_args 181 self._art_cmd.append('Test') 182 183 def CompileAndRunTest(self): 184 if self.CompileOnHost() == RetCode.SUCCESS: 185 retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt') 186 else: 187 retc = RetCode.NOTCOMPILED 188 return retc 189 190 191 class TestRunnerArtIntOnHost(TestRunnerArtOnHost): 192 """Concrete test runner of interpreter mode Art on host.""" 193 194 def __init__(self, use_dx): 195 """Constructor for the Art on host tester (interpreter). 196 197 Args: 198 use_dx: boolean, if True use dx rather than jack 199 """ 200 super().__init__(use_dx, ['-Xint']) 201 202 @property 203 def description(self): 204 return 'Art interpreter on host' 205 206 @property 207 def id(self): 208 return 'HInt' 209 210 def GetBisectionSearchArgs(self): 211 return None 212 213 214 class TestRunnerArtOptOnHost(TestRunnerArtOnHost): 215 """Concrete test runner of optimizing compiler mode Art on host.""" 216 217 def __init__(self, use_dx): 218 """Constructor for the Art on host tester (optimizing). 219 220 Args: 221 use_dx: boolean, if True use dx rather than jack 222 """ 223 super().__init__(use_dx, None) 224 225 @property 226 def description(self): 227 return 'Art optimizing on host' 228 229 @property 230 def id(self): 231 return 'HOpt' 232 233 def GetBisectionSearchArgs(self): 234 cmd_str = CommandListToCommandString( 235 self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:]) 236 return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 237 238 239 class TestRunnerArtOnTarget(TestRunnerWithHostCompilation): 240 """Abstract test runner of Art on target.""" 241 242 def __init__(self, use_dx, device, extra_args=None): 243 """Constructor for the Art on target tester. 244 245 Args: 246 use_dx: boolean, if True use dx rather than jack 247 device: string, target device serial number (or None) 248 extra_args: list of strings, extra arguments for dalvikvm 249 """ 250 super().__init__(use_dx) 251 self._test_env = DeviceTestEnv('jfuzz_', specific_device=device) 252 self._dalvik_cmd = ['dalvikvm'] 253 if extra_args is not None: 254 self._dalvik_cmd += extra_args 255 self._device = device 256 self._device_classpath = None 257 258 def CompileAndRunTest(self): 259 if self.CompileOnHost() == RetCode.SUCCESS: 260 self._device_classpath = self._test_env.PushClasspath('classes.dex') 261 cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'] 262 (output, retc) = self._test_env.RunCommand( 263 cmd, {'ANDROID_LOG_TAGS': '*:s'}) 264 with open(self.output_file, 'w') as run_out: 265 run_out.write(output) 266 else: 267 retc = RetCode.NOTCOMPILED 268 return retc 269 270 def GetBisectionSearchArgs(self): 271 cmd_str = CommandListToCommandString( 272 self._dalvik_cmd + ['-cp',self._device_classpath, 'Test']) 273 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 274 if self._device: 275 cmd += ['--device-serial', self._device] 276 else: 277 cmd.append('--device') 278 return cmd 279 280 281 class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget): 282 """Concrete test runner of interpreter mode Art on target.""" 283 284 def __init__(self, use_dx, device): 285 """Constructor for the Art on target tester (interpreter). 286 287 Args: 288 use_dx: boolean, if True use dx rather than jack 289 device: string, target device serial number (or None) 290 """ 291 super().__init__(use_dx, device, ['-Xint']) 292 293 @property 294 def description(self): 295 return 'Art interpreter on target' 296 297 @property 298 def id(self): 299 return 'TInt' 300 301 def GetBisectionSearchArgs(self): 302 return None 303 304 305 class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): 306 """Concrete test runner of optimizing compiler mode Art on target.""" 307 308 def __init__(self, use_dx, device): 309 """Constructor for the Art on target tester (optimizing). 310 311 Args: 312 use_dx: boolean, if True use dx rather than jack 313 device: string, target device serial number (or None) 314 """ 315 super().__init__(use_dx, device, None) 316 317 @property 318 def description(self): 319 return 'Art optimizing on target' 320 321 @property 322 def id(self): 323 return 'TOpt' 324 325 def GetBisectionSearchArgs(self): 326 cmd_str = CommandListToCommandString( 327 self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']) 328 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 329 if self._device: 330 cmd += ['--device-serial', self._device] 331 else: 332 cmd.append('--device') 333 return cmd 334 335 336 # 337 # Tester class. 338 # 339 340 341 class JFuzzTester(object): 342 """Tester that runs JFuzz many times and report divergences.""" 343 344 def __init__(self, num_tests, device, mode1, mode2, jfuzz_args, 345 report_script, true_divergence_only, use_dx): 346 """Constructor for the tester. 347 348 Args: 349 num_tests: int, number of tests to run 350 device: string, target device serial number (or None) 351 mode1: string, execution mode for first runner 352 mode2: string, execution mode for second runner 353 jfuzz_args: list of strings, additional arguments for jfuzz 354 report_script: string, path to script called for each divergence 355 true_divergence_only: boolean, if True don't bisect timeout divergences 356 use_dx: boolean, if True use dx rather than jack 357 """ 358 self._num_tests = num_tests 359 self._device = device 360 self._runner1 = GetExecutionModeRunner(use_dx, device, mode1) 361 self._runner2 = GetExecutionModeRunner(use_dx, device, mode2) 362 self._jfuzz_args = jfuzz_args 363 self._report_script = report_script 364 self._true_divergence_only = true_divergence_only 365 self._use_dx = use_dx 366 self._save_dir = None 367 self._results_dir = None 368 self._jfuzz_dir = None 369 # Statistics. 370 self._test = 0 371 self._num_success = 0 372 self._num_not_compiled = 0 373 self._num_not_run = 0 374 self._num_timed_out = 0 375 self._num_divergences = 0 376 377 def __enter__(self): 378 """On entry, enters new temp directory after saving current directory. 379 380 Raises: 381 FatalError: error when temp directory cannot be constructed 382 """ 383 self._save_dir = os.getcwd() 384 self._results_dir = mkdtemp(dir='/tmp/') 385 self._jfuzz_dir = mkdtemp(dir=self._results_dir) 386 if self._results_dir is None or self._jfuzz_dir is None: 387 raise FatalError('Cannot obtain temp directory') 388 os.chdir(self._jfuzz_dir) 389 return self 390 391 def __exit__(self, etype, evalue, etraceback): 392 """On exit, re-enters previously saved current directory and cleans up.""" 393 os.chdir(self._save_dir) 394 shutil.rmtree(self._jfuzz_dir) 395 if self._num_divergences == 0: 396 shutil.rmtree(self._results_dir) 397 398 def Run(self): 399 """Runs JFuzz many times and report divergences.""" 400 print() 401 print('**\n**** JFuzz Testing\n**') 402 print() 403 print('#Tests :', self._num_tests) 404 print('Device :', self._device) 405 print('Directory :', self._results_dir) 406 print('Exec-mode1:', self._runner1.description) 407 print('Exec-mode2:', self._runner2.description) 408 print('Compiler :', 'dx' if self._use_dx else 'jack') 409 print() 410 self.ShowStats() 411 for self._test in range(1, self._num_tests + 1): 412 self.RunJFuzzTest() 413 self.ShowStats() 414 if self._num_divergences == 0: 415 print('\n\nsuccess (no divergences)\n') 416 else: 417 print('\n\nfailure (divergences)\n') 418 419 def ShowStats(self): 420 """Shows current statistics (on same line) while tester is running.""" 421 print('\rTests:', self._test, 422 'Success:', self._num_success, 423 'Not-compiled:', self._num_not_compiled, 424 'Not-run:', self._num_not_run, 425 'Timed-out:', self._num_timed_out, 426 'Divergences:', self._num_divergences, 427 end='') 428 sys.stdout.flush() 429 430 def RunJFuzzTest(self): 431 """Runs a single JFuzz test, comparing two execution modes.""" 432 self.ConstructTest() 433 retc1 = self._runner1.CompileAndRunTest() 434 retc2 = self._runner2.CompileAndRunTest() 435 self.CheckForDivergence(retc1, retc2) 436 self.CleanupTest() 437 438 def ConstructTest(self): 439 """Use JFuzz to generate next Test.java test. 440 441 Raises: 442 FatalError: error when jfuzz fails 443 """ 444 if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None) 445 != RetCode.SUCCESS): 446 raise FatalError('Unexpected error while running JFuzz') 447 448 def CheckForDivergence(self, retc1, retc2): 449 """Checks for divergences and updates statistics. 450 451 Args: 452 retc1: int, normalized return code of first runner 453 retc2: int, normalized return code of second runner 454 """ 455 if retc1 == retc2: 456 # No divergence in return code. 457 if retc1 == RetCode.SUCCESS: 458 # Both compilations and runs were successful, inspect generated output. 459 runner1_out = self._runner1.output_file 460 runner2_out = self._runner2.output_file 461 if not filecmp.cmp(runner1_out, runner2_out, shallow=False): 462 # Divergence in output. 463 self.ReportDivergence(retc1, retc2, is_output_divergence=True) 464 else: 465 # No divergence in output. 466 self._num_success += 1 467 elif retc1 == RetCode.TIMEOUT: 468 self._num_timed_out += 1 469 elif retc1 == RetCode.NOTCOMPILED: 470 self._num_not_compiled += 1 471 else: 472 self._num_not_run += 1 473 elif self._true_divergence_only and RetCode.TIMEOUT in (retc1, retc2): 474 # When only true divergences are requested, any divergence in return 475 # code where one is a time out is treated as a regular time out. 476 self._num_timed_out += 1 477 else: 478 # Divergence in return code. 479 self.ReportDivergence(retc1, retc2, is_output_divergence=False) 480 481 def GetCurrentDivergenceDir(self): 482 return self._results_dir + '/divergence' + str(self._num_divergences) 483 484 def ReportDivergence(self, retc1, retc2, is_output_divergence): 485 """Reports and saves a divergence.""" 486 self._num_divergences += 1 487 print('\n' + str(self._num_divergences), end='') 488 if is_output_divergence: 489 print(' divergence in output') 490 else: 491 print(' divergence in return code: ' + retc1.name + ' vs. ' + 492 retc2.name) 493 # Save. 494 ddir = self.GetCurrentDivergenceDir() 495 os.mkdir(ddir) 496 for f in glob('*.txt') + ['Test.java']: 497 shutil.copy(f, ddir) 498 # Maybe run bisection bug search. 499 if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES: 500 self.MaybeBisectDivergence(retc1, retc2, is_output_divergence) 501 # Call reporting script. 502 if self._report_script: 503 self.RunReportScript(retc1, retc2, is_output_divergence) 504 505 def RunReportScript(self, retc1, retc2, is_output_divergence): 506 """Runs report script.""" 507 try: 508 title = "Divergence between {0} and {1} (found with fuzz testing)".format( 509 self._runner1.description, self._runner2.description) 510 # Prepare divergence comment. 511 jfuzz_cmd_and_version = subprocess.check_output( 512 ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True) 513 (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(') 514 # Strip right parenthesis and new line. 515 jfuzz_ver = jfuzz_ver[:-2] 516 jfuzz_args = ['\'-{0}\''.format(arg) 517 for arg in jfuzz_cmd_str.strip().split(' -')][1:] 518 wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args] 519 repro_cmd_str = (os.path.basename(__file__) + ' --num_tests 1 ' + 520 ' '.join(wrapped_args)) 521 comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format( 522 jfuzz_ver, jfuzz_cmd_str, repro_cmd_str) 523 if is_output_divergence: 524 (output, _, _) = RunCommandForOutput( 525 ['diff', self._runner1.output_file, self._runner2.output_file], 526 None, subprocess.PIPE, subprocess.STDOUT) 527 comment += 'Diff:\n' + output 528 else: 529 comment += '{0} vs {1}\n'.format(retc1, retc2) 530 # Prepare report script command. 531 script_cmd = [self._report_script, title, comment] 532 ddir = self.GetCurrentDivergenceDir() 533 bisection_out_files = glob(ddir + '/*_bisection_out.txt') 534 if bisection_out_files: 535 script_cmd += ['--bisection_out', bisection_out_files[0]] 536 subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL) 537 except subprocess.CalledProcessError as err: 538 print('Failed to run report script.\n', err) 539 540 def RunBisectionSearch(self, args, expected_retcode, expected_output, 541 runner_id): 542 ddir = self.GetCurrentDivergenceDir() 543 outfile_path = ddir + '/' + runner_id + '_bisection_out.txt' 544 logfile_path = ddir + '/' + runner_id + '_bisection_log.txt' 545 errfile_path = ddir + '/' + runner_id + '_bisection_err.txt' 546 args = list(args) + ['--logfile', logfile_path, '--cleanup'] 547 args += ['--expected-retcode', expected_retcode.name] 548 if expected_output: 549 args += ['--expected-output', expected_output] 550 bisection_search_path = os.path.join( 551 GetEnvVariableOrError('ANDROID_BUILD_TOP'), 552 'art/tools/bisection_search/bisection_search.py') 553 if RunCommand([bisection_search_path] + args, out=outfile_path, 554 err=errfile_path, timeout=300) == RetCode.TIMEOUT: 555 print('Bisection search TIMEOUT') 556 557 def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence): 558 bisection_args1 = self._runner1.GetBisectionSearchArgs() 559 bisection_args2 = self._runner2.GetBisectionSearchArgs() 560 if is_output_divergence: 561 maybe_output1 = self._runner1.output_file 562 maybe_output2 = self._runner2.output_file 563 else: 564 maybe_output1 = maybe_output2 = None 565 if bisection_args1 is not None: 566 self.RunBisectionSearch(bisection_args1, retc2, maybe_output2, 567 self._runner1.id) 568 if bisection_args2 is not None: 569 self.RunBisectionSearch(bisection_args2, retc1, maybe_output1, 570 self._runner2.id) 571 572 def CleanupTest(self): 573 """Cleans up after a single test run.""" 574 for file_name in os.listdir(self._jfuzz_dir): 575 file_path = os.path.join(self._jfuzz_dir, file_name) 576 if os.path.isfile(file_path): 577 os.unlink(file_path) 578 elif os.path.isdir(file_path): 579 shutil.rmtree(file_path) 580 581 582 def main(): 583 # Handle arguments. 584 parser = argparse.ArgumentParser() 585 parser.add_argument('--num_tests', default=10000, 586 type=int, help='number of tests to run') 587 parser.add_argument('--device', help='target device serial number') 588 parser.add_argument('--mode1', default='ri', 589 help='execution mode 1 (default: ri)') 590 parser.add_argument('--mode2', default='hopt', 591 help='execution mode 2 (default: hopt)') 592 parser.add_argument('--report_script', help='script called for each' 593 ' divergence') 594 parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args', 595 action='append', help='argument for jfuzz') 596 parser.add_argument('--true_divergence', default=False, action='store_true', 597 help='don\'t bisect timeout divergences') 598 parser.add_argument('--use_dx', default=False, action='store_true', 599 help='use old-style dx (rather than jack)') 600 args = parser.parse_args() 601 if args.mode1 == args.mode2: 602 raise FatalError('Identical execution modes given') 603 # Run the JFuzz tester. 604 with JFuzzTester(args.num_tests, 605 args.device, args.mode1, args.mode2, 606 args.jfuzz_args, args.report_script, 607 args.true_divergence, args.use_dx) as fuzzer: 608 fuzzer.Run() 609 610 if __name__ == '__main__': 611 main() 612
