1 #!/usr/bin/env python3.4 2 # 3 # Copyright (C) 2016 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import abc 18 import argparse 19 import filecmp 20 import os 21 import shlex 22 import shutil 23 import subprocess 24 import sys 25 26 from glob import glob 27 from subprocess import DEVNULL 28 from tempfile import mkdtemp 29 30 sys.path.append(os.path.dirname(os.path.dirname( 31 os.path.realpath(__file__)))) 32 33 from common.common import RetCode 34 from common.common import CommandListToCommandString 35 from common.common import FatalError 36 from common.common import GetJackClassPath 37 from common.common import GetEnvVariableOrError 38 from common.common import RunCommand 39 from common.common import RunCommandForOutput 40 from common.common import DeviceTestEnv 41 42 # Return codes supported by bisection bug search. 43 BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) 44 45 46 def GetExecutionModeRunner(use_dx, device, mode): 47 """Returns a runner for the given execution mode. 48 49 Args: 50 use_dx: boolean, if True use dx rather than jack 51 device: string, target device serial number (or None) 52 mode: string, execution mode 53 Returns: 54 TestRunner with given execution mode 55 Raises: 56 FatalError: error for unknown execution mode 57 """ 58 if mode == 'ri': 59 return TestRunnerRIOnHost() 60 if mode == 'hint': 61 return TestRunnerArtIntOnHost(use_dx) 62 if mode == 'hopt': 63 return TestRunnerArtOptOnHost(use_dx) 64 if mode == 'tint': 65 return TestRunnerArtIntOnTarget(use_dx, device) 66 if mode == 'topt': 67 return TestRunnerArtOptOnTarget(use_dx, device) 68 raise FatalError('Unknown execution mode') 69 70 71 # 72 # Execution mode classes. 73 # 74 75 76 class TestRunner(object): 77 """Abstraction for running a test in a particular execution mode.""" 78 __meta_class__ = abc.ABCMeta 79 80 @abc.abstractproperty 81 def description(self): 82 """Returns a description string of the execution mode.""" 83 84 @abc.abstractproperty 85 def id(self): 86 """Returns a short string that uniquely identifies the execution mode.""" 87 88 @property 89 def output_file(self): 90 return self.id + '_out.txt' 91 92 @abc.abstractmethod 93 def GetBisectionSearchArgs(self): 94 """Get arguments to pass to bisection search tool. 95 96 Returns: 97 list of strings - arguments for bisection search tool, or None if 98 runner is not bisectable 99 """ 100 101 @abc.abstractmethod 102 def CompileAndRunTest(self): 103 """Compile and run the generated test. 104 105 Ensures that the current Test.java in the temporary directory is compiled 106 and executed under the current execution mode. On success, transfers the 107 generated output to the file self.output_file in the temporary directory. 108 109 Most nonzero return codes are assumed non-divergent, since systems may 110 exit in different ways. This is enforced by normalizing return codes. 111 112 Returns: 113 normalized return code 114 """ 115 116 117 class TestRunnerWithHostCompilation(TestRunner): 118 """Abstract test runner that supports compilation on host.""" 119 120 def __init__(self, use_dx): 121 """Constructor for the runner with host compilation. 122 123 Args: 124 use_dx: boolean, if True use dx rather than jack 125 """ 126 self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', 127 'Test.java'] 128 self._use_dx = use_dx 129 130 def CompileOnHost(self): 131 if self._use_dx: 132 if RunCommand(['javac', 'Test.java'], 133 out=None, err=None, timeout=30) == RetCode.SUCCESS: 134 retc = RunCommand(['dx', '--dex', '--output=classes.dex'] + glob('*.class'), 135 out=None, err='dxerr.txt', timeout=30) 136 else: 137 retc = RetCode.NOTCOMPILED 138 else: 139 retc = RunCommand(['jack'] + self._jack_args, 140 out=None, err='jackerr.txt', timeout=30) 141 return retc 142 143 144 class TestRunnerRIOnHost(TestRunner): 145 """Concrete test runner of the reference implementation on host.""" 146 147 @property 148 def description(self): 149 return 'RI on host' 150 151 @property 152 def id(self): 153 return 'RI' 154 155 def CompileAndRunTest(self): 156 if RunCommand(['javac', 'Test.java'], 157 out=None, err=None, timeout=30) == RetCode.SUCCESS: 158 retc = RunCommand(['java', 'Test'], self.output_file, err=None) 159 else: 160 retc = RetCode.NOTCOMPILED 161 return retc 162 163 def GetBisectionSearchArgs(self): 164 return None 165 166 167 class TestRunnerArtOnHost(TestRunnerWithHostCompilation): 168 """Abstract test runner of Art on host.""" 169 170 def __init__(self, use_dx, extra_args=None): 171 """Constructor for the Art on host tester. 172 173 Args: 174 use_dx: boolean, if True use dx rather than jack 175 extra_args: list of strings, extra arguments for dalvikvm 176 """ 177 super().__init__(use_dx) 178 self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex'] 179 if extra_args is not None: 180 self._art_cmd += extra_args 181 self._art_cmd.append('Test') 182 183 def CompileAndRunTest(self): 184 if self.CompileOnHost() == RetCode.SUCCESS: 185 retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt') 186 else: 187 retc = RetCode.NOTCOMPILED 188 return retc 189 190 191 class TestRunnerArtIntOnHost(TestRunnerArtOnHost): 192 """Concrete test runner of interpreter mode Art on host.""" 193 194 def __init__(self, use_dx): 195 """Constructor for the Art on host tester (interpreter). 196 197 Args: 198 use_dx: boolean, if True use dx rather than jack 199 """ 200 super().__init__(use_dx, ['-Xint']) 201 202 @property 203 def description(self): 204 return 'Art interpreter on host' 205 206 @property 207 def id(self): 208 return 'HInt' 209 210 def GetBisectionSearchArgs(self): 211 return None 212 213 214 class TestRunnerArtOptOnHost(TestRunnerArtOnHost): 215 """Concrete test runner of optimizing compiler mode Art on host.""" 216 217 def __init__(self, use_dx): 218 """Constructor for the Art on host tester (optimizing). 219 220 Args: 221 use_dx: boolean, if True use dx rather than jack 222 """ 223 super().__init__(use_dx, None) 224 225 @property 226 def description(self): 227 return 'Art optimizing on host' 228 229 @property 230 def id(self): 231 return 'HOpt' 232 233 def GetBisectionSearchArgs(self): 234 cmd_str = CommandListToCommandString( 235 self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:]) 236 return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 237 238 239 class TestRunnerArtOnTarget(TestRunnerWithHostCompilation): 240 """Abstract test runner of Art on target.""" 241 242 def __init__(self, use_dx, device, extra_args=None): 243 """Constructor for the Art on target tester. 244 245 Args: 246 use_dx: boolean, if True use dx rather than jack 247 device: string, target device serial number (or None) 248 extra_args: list of strings, extra arguments for dalvikvm 249 """ 250 super().__init__(use_dx) 251 self._test_env = DeviceTestEnv('jfuzz_', specific_device=device) 252 self._dalvik_cmd = ['dalvikvm'] 253 if extra_args is not None: 254 self._dalvik_cmd += extra_args 255 self._device = device 256 self._device_classpath = None 257 258 def CompileAndRunTest(self): 259 if self.CompileOnHost() == RetCode.SUCCESS: 260 self._device_classpath = self._test_env.PushClasspath('classes.dex') 261 cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'] 262 (output, retc) = self._test_env.RunCommand( 263 cmd, {'ANDROID_LOG_TAGS': '*:s'}) 264 with open(self.output_file, 'w') as run_out: 265 run_out.write(output) 266 else: 267 retc = RetCode.NOTCOMPILED 268 return retc 269 270 def GetBisectionSearchArgs(self): 271 cmd_str = CommandListToCommandString( 272 self._dalvik_cmd + ['-cp',self._device_classpath, 'Test']) 273 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 274 if self._device: 275 cmd += ['--device-serial', self._device] 276 else: 277 cmd.append('--device') 278 return cmd 279 280 281 class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget): 282 """Concrete test runner of interpreter mode Art on target.""" 283 284 def __init__(self, use_dx, device): 285 """Constructor for the Art on target tester (interpreter). 286 287 Args: 288 use_dx: boolean, if True use dx rather than jack 289 device: string, target device serial number (or None) 290 """ 291 super().__init__(use_dx, device, ['-Xint']) 292 293 @property 294 def description(self): 295 return 'Art interpreter on target' 296 297 @property 298 def id(self): 299 return 'TInt' 300 301 def GetBisectionSearchArgs(self): 302 return None 303 304 305 class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): 306 """Concrete test runner of optimizing compiler mode Art on target.""" 307 308 def __init__(self, use_dx, device): 309 """Constructor for the Art on target tester (optimizing). 310 311 Args: 312 use_dx: boolean, if True use dx rather than jack 313 device: string, target device serial number (or None) 314 """ 315 super().__init__(use_dx, device, None) 316 317 @property 318 def description(self): 319 return 'Art optimizing on target' 320 321 @property 322 def id(self): 323 return 'TOpt' 324 325 def GetBisectionSearchArgs(self): 326 cmd_str = CommandListToCommandString( 327 self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']) 328 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 329 if self._device: 330 cmd += ['--device-serial', self._device] 331 else: 332 cmd.append('--device') 333 return cmd 334 335 336 # 337 # Tester class. 338 # 339 340 341 class JFuzzTester(object): 342 """Tester that runs JFuzz many times and report divergences.""" 343 344 def __init__(self, num_tests, device, mode1, mode2, jfuzz_args, 345 report_script, true_divergence_only, use_dx): 346 """Constructor for the tester. 347 348 Args: 349 num_tests: int, number of tests to run 350 device: string, target device serial number (or None) 351 mode1: string, execution mode for first runner 352 mode2: string, execution mode for second runner 353 jfuzz_args: list of strings, additional arguments for jfuzz 354 report_script: string, path to script called for each divergence 355 true_divergence_only: boolean, if True don't bisect timeout divergences 356 use_dx: boolean, if True use dx rather than jack 357 """ 358 self._num_tests = num_tests 359 self._device = device 360 self._runner1 = GetExecutionModeRunner(use_dx, device, mode1) 361 self._runner2 = GetExecutionModeRunner(use_dx, device, mode2) 362 self._jfuzz_args = jfuzz_args 363 self._report_script = report_script 364 self._true_divergence_only = true_divergence_only 365 self._use_dx = use_dx 366 self._save_dir = None 367 self._results_dir = None 368 self._jfuzz_dir = None 369 # Statistics. 370 self._test = 0 371 self._num_success = 0 372 self._num_not_compiled = 0 373 self._num_not_run = 0 374 self._num_timed_out = 0 375 self._num_divergences = 0 376 377 def __enter__(self): 378 """On entry, enters new temp directory after saving current directory. 379 380 Raises: 381 FatalError: error when temp directory cannot be constructed 382 """ 383 self._save_dir = os.getcwd() 384 self._results_dir = mkdtemp(dir='/tmp/') 385 self._jfuzz_dir = mkdtemp(dir=self._results_dir) 386 if self._results_dir is None or self._jfuzz_dir is None: 387 raise FatalError('Cannot obtain temp directory') 388 os.chdir(self._jfuzz_dir) 389 return self 390 391 def __exit__(self, etype, evalue, etraceback): 392 """On exit, re-enters previously saved current directory and cleans up.""" 393 os.chdir(self._save_dir) 394 shutil.rmtree(self._jfuzz_dir) 395 if self._num_divergences == 0: 396 shutil.rmtree(self._results_dir) 397 398 def Run(self): 399 """Runs JFuzz many times and report divergences.""" 400 print() 401 print('**\n**** JFuzz Testing\n**') 402 print() 403 print('#Tests :', self._num_tests) 404 print('Device :', self._device) 405 print('Directory :', self._results_dir) 406 print('Exec-mode1:', self._runner1.description) 407 print('Exec-mode2:', self._runner2.description) 408 print('Compiler :', 'dx' if self._use_dx else 'jack') 409 print() 410 self.ShowStats() 411 for self._test in range(1, self._num_tests + 1): 412 self.RunJFuzzTest() 413 self.ShowStats() 414 if self._num_divergences == 0: 415 print('\n\nsuccess (no divergences)\n') 416 else: 417 print('\n\nfailure (divergences)\n') 418 419 def ShowStats(self): 420 """Shows current statistics (on same line) while tester is running.""" 421 print('\rTests:', self._test, 422 'Success:', self._num_success, 423 'Not-compiled:', self._num_not_compiled, 424 'Not-run:', self._num_not_run, 425 'Timed-out:', self._num_timed_out, 426 'Divergences:', self._num_divergences, 427 end='') 428 sys.stdout.flush() 429 430 def RunJFuzzTest(self): 431 """Runs a single JFuzz test, comparing two execution modes.""" 432 self.ConstructTest() 433 retc1 = self._runner1.CompileAndRunTest() 434 retc2 = self._runner2.CompileAndRunTest() 435 self.CheckForDivergence(retc1, retc2) 436 self.CleanupTest() 437 438 def ConstructTest(self): 439 """Use JFuzz to generate next Test.java test. 440 441 Raises: 442 FatalError: error when jfuzz fails 443 """ 444 if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None) 445 != RetCode.SUCCESS): 446 raise FatalError('Unexpected error while running JFuzz') 447 448 def CheckForDivergence(self, retc1, retc2): 449 """Checks for divergences and updates statistics. 450 451 Args: 452 retc1: int, normalized return code of first runner 453 retc2: int, normalized return code of second runner 454 """ 455 if retc1 == retc2: 456 # No divergence in return code. 457 if retc1 == RetCode.SUCCESS: 458 # Both compilations and runs were successful, inspect generated output. 459 runner1_out = self._runner1.output_file 460 runner2_out = self._runner2.output_file 461 if not filecmp.cmp(runner1_out, runner2_out, shallow=False): 462 # Divergence in output. 463 self.ReportDivergence(retc1, retc2, is_output_divergence=True) 464 else: 465 # No divergence in output. 466 self._num_success += 1 467 elif retc1 == RetCode.TIMEOUT: 468 self._num_timed_out += 1 469 elif retc1 == RetCode.NOTCOMPILED: 470 self._num_not_compiled += 1 471 else: 472 self._num_not_run += 1 473 else: 474 # Divergence in return code. 475 if self._true_divergence_only: 476 # When only true divergences are requested, any divergence in return 477 # code where one is a time out is treated as a regular time out. 478 if RetCode.TIMEOUT in (retc1, retc2): 479 self._num_timed_out += 1 480 return 481 # When only true divergences are requested, a runtime crash in just 482 # the RI is treated as if not run at all. 483 if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS: 484 if self._runner1.GetBisectionSearchArgs() is None: 485 self._num_not_run += 1 486 return 487 self.ReportDivergence(retc1, retc2, is_output_divergence=False) 488 489 def GetCurrentDivergenceDir(self): 490 return self._results_dir + '/divergence' + str(self._num_divergences) 491 492 def ReportDivergence(self, retc1, retc2, is_output_divergence): 493 """Reports and saves a divergence.""" 494 self._num_divergences += 1 495 print('\n' + str(self._num_divergences), end='') 496 if is_output_divergence: 497 print(' divergence in output') 498 else: 499 print(' divergence in return code: ' + retc1.name + ' vs. ' + 500 retc2.name) 501 # Save. 502 ddir = self.GetCurrentDivergenceDir() 503 os.mkdir(ddir) 504 for f in glob('*.txt') + ['Test.java']: 505 shutil.copy(f, ddir) 506 # Maybe run bisection bug search. 507 if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES: 508 self.MaybeBisectDivergence(retc1, retc2, is_output_divergence) 509 # Call reporting script. 510 if self._report_script: 511 self.RunReportScript(retc1, retc2, is_output_divergence) 512 513 def RunReportScript(self, retc1, retc2, is_output_divergence): 514 """Runs report script.""" 515 try: 516 title = "Divergence between {0} and {1} (found with fuzz testing)".format( 517 self._runner1.description, self._runner2.description) 518 # Prepare divergence comment. 519 jfuzz_cmd_and_version = subprocess.check_output( 520 ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True) 521 (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(') 522 # Strip right parenthesis and new line. 523 jfuzz_ver = jfuzz_ver[:-2] 524 jfuzz_args = ['\'-{0}\''.format(arg) 525 for arg in jfuzz_cmd_str.strip().split(' -')][1:] 526 wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args] 527 repro_cmd_str = (os.path.basename(__file__) + ' --num_tests 1 ' + 528 ' '.join(wrapped_args)) 529 comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format( 530 jfuzz_ver, jfuzz_cmd_str, repro_cmd_str) 531 if is_output_divergence: 532 (output, _, _) = RunCommandForOutput( 533 ['diff', self._runner1.output_file, self._runner2.output_file], 534 None, subprocess.PIPE, subprocess.STDOUT) 535 comment += 'Diff:\n' + output 536 else: 537 comment += '{0} vs {1}\n'.format(retc1, retc2) 538 # Prepare report script command. 539 script_cmd = [self._report_script, title, comment] 540 ddir = self.GetCurrentDivergenceDir() 541 bisection_out_files = glob(ddir + '/*_bisection_out.txt') 542 if bisection_out_files: 543 script_cmd += ['--bisection_out', bisection_out_files[0]] 544 subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL) 545 except subprocess.CalledProcessError as err: 546 print('Failed to run report script.\n', err) 547 548 def RunBisectionSearch(self, args, expected_retcode, expected_output, 549 runner_id): 550 ddir = self.GetCurrentDivergenceDir() 551 outfile_path = ddir + '/' + runner_id + '_bisection_out.txt' 552 logfile_path = ddir + '/' + runner_id + '_bisection_log.txt' 553 errfile_path = ddir + '/' + runner_id + '_bisection_err.txt' 554 args = list(args) + ['--logfile', logfile_path, '--cleanup'] 555 args += ['--expected-retcode', expected_retcode.name] 556 if expected_output: 557 args += ['--expected-output', expected_output] 558 bisection_search_path = os.path.join( 559 GetEnvVariableOrError('ANDROID_BUILD_TOP'), 560 'art/tools/bisection_search/bisection_search.py') 561 if RunCommand([bisection_search_path] + args, out=outfile_path, 562 err=errfile_path, timeout=300) == RetCode.TIMEOUT: 563 print('Bisection search TIMEOUT') 564 565 def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence): 566 bisection_args1 = self._runner1.GetBisectionSearchArgs() 567 bisection_args2 = self._runner2.GetBisectionSearchArgs() 568 if is_output_divergence: 569 maybe_output1 = self._runner1.output_file 570 maybe_output2 = self._runner2.output_file 571 else: 572 maybe_output1 = maybe_output2 = None 573 if bisection_args1 is not None: 574 self.RunBisectionSearch(bisection_args1, retc2, maybe_output2, 575 self._runner1.id) 576 if bisection_args2 is not None: 577 self.RunBisectionSearch(bisection_args2, retc1, maybe_output1, 578 self._runner2.id) 579 580 def CleanupTest(self): 581 """Cleans up after a single test run.""" 582 for file_name in os.listdir(self._jfuzz_dir): 583 file_path = os.path.join(self._jfuzz_dir, file_name) 584 if os.path.isfile(file_path): 585 os.unlink(file_path) 586 elif os.path.isdir(file_path): 587 shutil.rmtree(file_path) 588 589 590 def main(): 591 # Handle arguments. 592 parser = argparse.ArgumentParser() 593 parser.add_argument('--num_tests', default=10000, 594 type=int, help='number of tests to run') 595 parser.add_argument('--device', help='target device serial number') 596 parser.add_argument('--mode1', default='ri', 597 help='execution mode 1 (default: ri)') 598 parser.add_argument('--mode2', default='hopt', 599 help='execution mode 2 (default: hopt)') 600 parser.add_argument('--report_script', help='script called for each' 601 ' divergence') 602 parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args', 603 action='append', help='argument for jfuzz') 604 parser.add_argument('--true_divergence', default=False, action='store_true', 605 help='don\'t bisect timeout divergences') 606 parser.add_argument('--use_dx', default=False, action='store_true', 607 help='use old-style dx (rather than jack)') 608 args = parser.parse_args() 609 if args.mode1 == args.mode2: 610 raise FatalError('Identical execution modes given') 611 # Run the JFuzz tester. 612 with JFuzzTester(args.num_tests, 613 args.device, args.mode1, args.mode2, 614 args.jfuzz_args, args.report_script, 615 args.true_divergence, args.use_dx) as fuzzer: 616 fuzzer.Run() 617 618 if __name__ == '__main__': 619 main() 620
