Home | History | Annotate | Download | only in jfuzz
      1 #!/usr/bin/env python3.4
      2 #
      3 # Copyright (C) 2016 The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #   http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 import abc
     18 import argparse
     19 import filecmp
     20 import os
     21 import shlex
     22 import shutil
     23 import subprocess
     24 import sys
     25 
     26 from glob import glob
     27 from subprocess import DEVNULL
     28 from tempfile import mkdtemp
     29 
     30 sys.path.append(os.path.dirname(os.path.dirname(
     31     os.path.realpath(__file__))))
     32 
     33 from common.common import RetCode
     34 from common.common import CommandListToCommandString
     35 from common.common import FatalError
     36 from common.common import GetJackClassPath
     37 from common.common import GetEnvVariableOrError
     38 from common.common import RunCommand
     39 from common.common import RunCommandForOutput
     40 from common.common import DeviceTestEnv
     41 
     42 # Return codes supported by bisection bug search.
     43 BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
     44 
     45 
     46 def GetExecutionModeRunner(use_dx, device, mode):
     47   """Returns a runner for the given execution mode.
     48 
     49   Args:
     50     use_dx: boolean, if True use dx rather than jack
     51     device: string, target device serial number (or None)
     52     mode: string, execution mode
     53   Returns:
     54     TestRunner with given execution mode
     55   Raises:
     56     FatalError: error for unknown execution mode
     57   """
     58   if mode == 'ri':
     59     return TestRunnerRIOnHost()
     60   if mode == 'hint':
     61     return TestRunnerArtIntOnHost(use_dx)
     62   if mode == 'hopt':
     63     return TestRunnerArtOptOnHost(use_dx)
     64   if mode == 'tint':
     65     return TestRunnerArtIntOnTarget(use_dx, device)
     66   if mode == 'topt':
     67     return TestRunnerArtOptOnTarget(use_dx, device)
     68   raise FatalError('Unknown execution mode')
     69 
     70 
     71 #
     72 # Execution mode classes.
     73 #
     74 
     75 
     76 class TestRunner(object):
     77   """Abstraction for running a test in a particular execution mode."""
     78   __meta_class__ = abc.ABCMeta
     79 
     80   @abc.abstractproperty
     81   def description(self):
     82     """Returns a description string of the execution mode."""
     83 
     84   @abc.abstractproperty
     85   def id(self):
     86     """Returns a short string that uniquely identifies the execution mode."""
     87 
     88   @property
     89   def output_file(self):
     90     return self.id + '_out.txt'
     91 
     92   @abc.abstractmethod
     93   def GetBisectionSearchArgs(self):
     94     """Get arguments to pass to bisection search tool.
     95 
     96     Returns:
     97       list of strings - arguments for bisection search tool, or None if
     98       runner is not bisectable
     99     """
    100 
    101   @abc.abstractmethod
    102   def CompileAndRunTest(self):
    103     """Compile and run the generated test.
    104 
    105     Ensures that the current Test.java in the temporary directory is compiled
    106     and executed under the current execution mode. On success, transfers the
    107     generated output to the file self.output_file in the temporary directory.
    108 
    109     Most nonzero return codes are assumed non-divergent, since systems may
    110     exit in different ways. This is enforced by normalizing return codes.
    111 
    112     Returns:
    113       normalized return code
    114     """
    115 
    116 
    117 class TestRunnerWithHostCompilation(TestRunner):
    118   """Abstract test runner that supports compilation on host."""
    119 
    120   def  __init__(self, use_dx):
    121     """Constructor for the runner with host compilation.
    122 
    123     Args:
    124       use_dx: boolean, if True use dx rather than jack
    125     """
    126     self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
    127                        'Test.java']
    128     self._use_dx = use_dx
    129 
    130   def CompileOnHost(self):
    131     if self._use_dx:
    132       if RunCommand(['javac', 'Test.java'],
    133                     out=None, err=None, timeout=30) == RetCode.SUCCESS:
    134         retc = RunCommand(['dx', '--dex', '--output=classes.dex'] + glob('*.class'),
    135                           out=None, err='dxerr.txt', timeout=30)
    136       else:
    137         retc = RetCode.NOTCOMPILED
    138     else:
    139       retc = RunCommand(['jack'] + self._jack_args,
    140                         out=None, err='jackerr.txt', timeout=30)
    141     return retc
    142 
    143 
    144 class TestRunnerRIOnHost(TestRunner):
    145   """Concrete test runner of the reference implementation on host."""
    146 
    147   @property
    148   def description(self):
    149     return 'RI on host'
    150 
    151   @property
    152   def id(self):
    153     return 'RI'
    154 
    155   def CompileAndRunTest(self):
    156     if RunCommand(['javac', 'Test.java'],
    157                   out=None, err=None, timeout=30) == RetCode.SUCCESS:
    158       retc = RunCommand(['java', 'Test'], self.output_file, err=None)
    159     else:
    160       retc = RetCode.NOTCOMPILED
    161     return retc
    162 
    163   def GetBisectionSearchArgs(self):
    164     return None
    165 
    166 
    167 class TestRunnerArtOnHost(TestRunnerWithHostCompilation):
    168   """Abstract test runner of Art on host."""
    169 
    170   def  __init__(self, use_dx, extra_args=None):
    171     """Constructor for the Art on host tester.
    172 
    173     Args:
    174       use_dx: boolean, if True use dx rather than jack
    175       extra_args: list of strings, extra arguments for dalvikvm
    176     """
    177     super().__init__(use_dx)
    178     self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
    179     if extra_args is not None:
    180       self._art_cmd += extra_args
    181     self._art_cmd.append('Test')
    182 
    183   def CompileAndRunTest(self):
    184     if self.CompileOnHost() == RetCode.SUCCESS:
    185       retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
    186     else:
    187       retc = RetCode.NOTCOMPILED
    188     return retc
    189 
    190 
    191 class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
    192   """Concrete test runner of interpreter mode Art on host."""
    193 
    194   def  __init__(self, use_dx):
    195     """Constructor for the Art on host tester (interpreter).
    196 
    197     Args:
    198       use_dx: boolean, if True use dx rather than jack
    199    """
    200     super().__init__(use_dx, ['-Xint'])
    201 
    202   @property
    203   def description(self):
    204     return 'Art interpreter on host'
    205 
    206   @property
    207   def id(self):
    208     return 'HInt'
    209 
    210   def GetBisectionSearchArgs(self):
    211     return None
    212 
    213 
    214 class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
    215   """Concrete test runner of optimizing compiler mode Art on host."""
    216 
    217   def  __init__(self, use_dx):
    218     """Constructor for the Art on host tester (optimizing).
    219 
    220     Args:
    221       use_dx: boolean, if True use dx rather than jack
    222    """
    223     super().__init__(use_dx, None)
    224 
    225   @property
    226   def description(self):
    227     return 'Art optimizing on host'
    228 
    229   @property
    230   def id(self):
    231     return 'HOpt'
    232 
    233   def GetBisectionSearchArgs(self):
    234     cmd_str = CommandListToCommandString(
    235         self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
    236     return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
    237 
    238 
    239 class TestRunnerArtOnTarget(TestRunnerWithHostCompilation):
    240   """Abstract test runner of Art on target."""
    241 
    242   def  __init__(self, use_dx, device, extra_args=None):
    243     """Constructor for the Art on target tester.
    244 
    245     Args:
    246       use_dx: boolean, if True use dx rather than jack
    247       device: string, target device serial number (or None)
    248       extra_args: list of strings, extra arguments for dalvikvm
    249     """
    250     super().__init__(use_dx)
    251     self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
    252     self._dalvik_cmd = ['dalvikvm']
    253     if extra_args is not None:
    254       self._dalvik_cmd += extra_args
    255     self._device = device
    256     self._device_classpath = None
    257 
    258   def CompileAndRunTest(self):
    259     if self.CompileOnHost() == RetCode.SUCCESS:
    260       self._device_classpath = self._test_env.PushClasspath('classes.dex')
    261       cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
    262       (output, retc) = self._test_env.RunCommand(
    263           cmd, {'ANDROID_LOG_TAGS': '*:s'})
    264       with open(self.output_file, 'w') as run_out:
    265         run_out.write(output)
    266     else:
    267       retc = RetCode.NOTCOMPILED
    268     return retc
    269 
    270   def GetBisectionSearchArgs(self):
    271     cmd_str = CommandListToCommandString(
    272         self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
    273     cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
    274     if self._device:
    275       cmd += ['--device-serial', self._device]
    276     else:
    277       cmd.append('--device')
    278     return cmd
    279 
    280 
    281 class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
    282   """Concrete test runner of interpreter mode Art on target."""
    283 
    284   def  __init__(self, use_dx, device):
    285     """Constructor for the Art on target tester (interpreter).
    286 
    287     Args:
    288       use_dx: boolean, if True use dx rather than jack
    289       device: string, target device serial number (or None)
    290     """
    291     super().__init__(use_dx, device, ['-Xint'])
    292 
    293   @property
    294   def description(self):
    295     return 'Art interpreter on target'
    296 
    297   @property
    298   def id(self):
    299     return 'TInt'
    300 
    301   def GetBisectionSearchArgs(self):
    302     return None
    303 
    304 
    305 class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
    306   """Concrete test runner of optimizing compiler mode Art on target."""
    307 
    308   def  __init__(self, use_dx, device):
    309     """Constructor for the Art on target tester (optimizing).
    310 
    311     Args:
    312       use_dx: boolean, if True use dx rather than jack
    313       device: string, target device serial number (or None)
    314     """
    315     super().__init__(use_dx, device, None)
    316 
    317   @property
    318   def description(self):
    319     return 'Art optimizing on target'
    320 
    321   @property
    322   def id(self):
    323     return 'TOpt'
    324 
    325   def GetBisectionSearchArgs(self):
    326     cmd_str = CommandListToCommandString(
    327         self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
    328     cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
    329     if self._device:
    330       cmd += ['--device-serial', self._device]
    331     else:
    332       cmd.append('--device')
    333     return cmd
    334 
    335 
    336 #
    337 # Tester class.
    338 #
    339 
    340 
    341 class JFuzzTester(object):
    342   """Tester that runs JFuzz many times and report divergences."""
    343 
    344   def  __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
    345                 report_script, true_divergence_only, use_dx):
    346     """Constructor for the tester.
    347 
    348     Args:
    349       num_tests: int, number of tests to run
    350       device: string, target device serial number (or None)
    351       mode1: string, execution mode for first runner
    352       mode2: string, execution mode for second runner
    353       jfuzz_args: list of strings, additional arguments for jfuzz
    354       report_script: string, path to script called for each divergence
    355       true_divergence_only: boolean, if True don't bisect timeout divergences
    356       use_dx: boolean, if True use dx rather than jack
    357     """
    358     self._num_tests = num_tests
    359     self._device = device
    360     self._runner1 = GetExecutionModeRunner(use_dx, device, mode1)
    361     self._runner2 = GetExecutionModeRunner(use_dx, device, mode2)
    362     self._jfuzz_args = jfuzz_args
    363     self._report_script = report_script
    364     self._true_divergence_only = true_divergence_only
    365     self._use_dx = use_dx
    366     self._save_dir = None
    367     self._results_dir = None
    368     self._jfuzz_dir = None
    369     # Statistics.
    370     self._test = 0
    371     self._num_success = 0
    372     self._num_not_compiled = 0
    373     self._num_not_run = 0
    374     self._num_timed_out = 0
    375     self._num_divergences = 0
    376 
    377   def __enter__(self):
    378     """On entry, enters new temp directory after saving current directory.
    379 
    380     Raises:
    381       FatalError: error when temp directory cannot be constructed
    382     """
    383     self._save_dir = os.getcwd()
    384     self._results_dir = mkdtemp(dir='/tmp/')
    385     self._jfuzz_dir = mkdtemp(dir=self._results_dir)
    386     if self._results_dir is None or self._jfuzz_dir is None:
    387       raise FatalError('Cannot obtain temp directory')
    388     os.chdir(self._jfuzz_dir)
    389     return self
    390 
    391   def __exit__(self, etype, evalue, etraceback):
    392     """On exit, re-enters previously saved current directory and cleans up."""
    393     os.chdir(self._save_dir)
    394     shutil.rmtree(self._jfuzz_dir)
    395     if self._num_divergences == 0:
    396       shutil.rmtree(self._results_dir)
    397 
    398   def Run(self):
    399     """Runs JFuzz many times and report divergences."""
    400     print()
    401     print('**\n**** JFuzz Testing\n**')
    402     print()
    403     print('#Tests    :', self._num_tests)
    404     print('Device    :', self._device)
    405     print('Directory :', self._results_dir)
    406     print('Exec-mode1:', self._runner1.description)
    407     print('Exec-mode2:', self._runner2.description)
    408     print('Compiler  :', 'dx' if self._use_dx else 'jack')
    409     print()
    410     self.ShowStats()
    411     for self._test in range(1, self._num_tests + 1):
    412       self.RunJFuzzTest()
    413       self.ShowStats()
    414     if self._num_divergences == 0:
    415       print('\n\nsuccess (no divergences)\n')
    416     else:
    417       print('\n\nfailure (divergences)\n')
    418 
    419   def ShowStats(self):
    420     """Shows current statistics (on same line) while tester is running."""
    421     print('\rTests:', self._test,
    422           'Success:', self._num_success,
    423           'Not-compiled:', self._num_not_compiled,
    424           'Not-run:', self._num_not_run,
    425           'Timed-out:', self._num_timed_out,
    426           'Divergences:', self._num_divergences,
    427           end='')
    428     sys.stdout.flush()
    429 
    430   def RunJFuzzTest(self):
    431     """Runs a single JFuzz test, comparing two execution modes."""
    432     self.ConstructTest()
    433     retc1 = self._runner1.CompileAndRunTest()
    434     retc2 = self._runner2.CompileAndRunTest()
    435     self.CheckForDivergence(retc1, retc2)
    436     self.CleanupTest()
    437 
    438   def ConstructTest(self):
    439     """Use JFuzz to generate next Test.java test.
    440 
    441     Raises:
    442       FatalError: error when jfuzz fails
    443     """
    444     if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
    445           != RetCode.SUCCESS):
    446       raise FatalError('Unexpected error while running JFuzz')
    447 
    448   def CheckForDivergence(self, retc1, retc2):
    449     """Checks for divergences and updates statistics.
    450 
    451     Args:
    452       retc1: int, normalized return code of first runner
    453       retc2: int, normalized return code of second runner
    454     """
    455     if retc1 == retc2:
    456       # No divergence in return code.
    457       if retc1 == RetCode.SUCCESS:
    458         # Both compilations and runs were successful, inspect generated output.
    459         runner1_out = self._runner1.output_file
    460         runner2_out = self._runner2.output_file
    461         if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
    462           # Divergence in output.
    463           self.ReportDivergence(retc1, retc2, is_output_divergence=True)
    464         else:
    465           # No divergence in output.
    466           self._num_success += 1
    467       elif retc1 == RetCode.TIMEOUT:
    468         self._num_timed_out += 1
    469       elif retc1 == RetCode.NOTCOMPILED:
    470         self._num_not_compiled += 1
    471       else:
    472         self._num_not_run += 1
    473     elif self._true_divergence_only and RetCode.TIMEOUT in (retc1, retc2):
    474       # When only true divergences are requested, any divergence in return
    475       # code where one is a time out is treated as a regular time out.
    476       self._num_timed_out += 1
    477     else:
    478       # Divergence in return code.
    479       self.ReportDivergence(retc1, retc2, is_output_divergence=False)
    480 
    481   def GetCurrentDivergenceDir(self):
    482     return self._results_dir + '/divergence' + str(self._num_divergences)
    483 
    484   def ReportDivergence(self, retc1, retc2, is_output_divergence):
    485     """Reports and saves a divergence."""
    486     self._num_divergences += 1
    487     print('\n' + str(self._num_divergences), end='')
    488     if is_output_divergence:
    489       print(' divergence in output')
    490     else:
    491       print(' divergence in return code: ' + retc1.name + ' vs. ' +
    492             retc2.name)
    493     # Save.
    494     ddir = self.GetCurrentDivergenceDir()
    495     os.mkdir(ddir)
    496     for f in glob('*.txt') + ['Test.java']:
    497       shutil.copy(f, ddir)
    498     # Maybe run bisection bug search.
    499     if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
    500       self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
    501     # Call reporting script.
    502     if self._report_script:
    503       self.RunReportScript(retc1, retc2, is_output_divergence)
    504 
    505   def RunReportScript(self, retc1, retc2, is_output_divergence):
    506     """Runs report script."""
    507     try:
    508       title = "Divergence between {0} and {1} (found with fuzz testing)".format(
    509           self._runner1.description, self._runner2.description)
    510       # Prepare divergence comment.
    511       jfuzz_cmd_and_version = subprocess.check_output(
    512           ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
    513       (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
    514       # Strip right parenthesis and new line.
    515       jfuzz_ver = jfuzz_ver[:-2]
    516       jfuzz_args = ['\'-{0}\''.format(arg)
    517                     for arg in jfuzz_cmd_str.strip().split(' -')][1:]
    518       wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
    519       repro_cmd_str = (os.path.basename(__file__) + ' --num_tests 1 ' +
    520                        ' '.join(wrapped_args))
    521       comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
    522           jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
    523       if is_output_divergence:
    524         (output, _, _) = RunCommandForOutput(
    525             ['diff', self._runner1.output_file, self._runner2.output_file],
    526             None, subprocess.PIPE, subprocess.STDOUT)
    527         comment += 'Diff:\n' + output
    528       else:
    529         comment += '{0} vs {1}\n'.format(retc1, retc2)
    530       # Prepare report script command.
    531       script_cmd = [self._report_script, title, comment]
    532       ddir = self.GetCurrentDivergenceDir()
    533       bisection_out_files = glob(ddir + '/*_bisection_out.txt')
    534       if bisection_out_files:
    535         script_cmd += ['--bisection_out', bisection_out_files[0]]
    536       subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
    537     except subprocess.CalledProcessError as err:
    538       print('Failed to run report script.\n', err)
    539 
    540   def RunBisectionSearch(self, args, expected_retcode, expected_output,
    541                          runner_id):
    542     ddir = self.GetCurrentDivergenceDir()
    543     outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
    544     logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
    545     errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
    546     args = list(args) + ['--logfile', logfile_path, '--cleanup']
    547     args += ['--expected-retcode', expected_retcode.name]
    548     if expected_output:
    549       args += ['--expected-output', expected_output]
    550     bisection_search_path = os.path.join(
    551         GetEnvVariableOrError('ANDROID_BUILD_TOP'),
    552         'art/tools/bisection_search/bisection_search.py')
    553     if RunCommand([bisection_search_path] + args, out=outfile_path,
    554                   err=errfile_path, timeout=300) == RetCode.TIMEOUT:
    555       print('Bisection search TIMEOUT')
    556 
    557   def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
    558     bisection_args1 = self._runner1.GetBisectionSearchArgs()
    559     bisection_args2 = self._runner2.GetBisectionSearchArgs()
    560     if is_output_divergence:
    561       maybe_output1 = self._runner1.output_file
    562       maybe_output2 = self._runner2.output_file
    563     else:
    564       maybe_output1 = maybe_output2 = None
    565     if bisection_args1 is not None:
    566       self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
    567                               self._runner1.id)
    568     if bisection_args2 is not None:
    569       self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
    570                               self._runner2.id)
    571 
    572   def CleanupTest(self):
    573     """Cleans up after a single test run."""
    574     for file_name in os.listdir(self._jfuzz_dir):
    575       file_path = os.path.join(self._jfuzz_dir, file_name)
    576       if os.path.isfile(file_path):
    577         os.unlink(file_path)
    578       elif os.path.isdir(file_path):
    579         shutil.rmtree(file_path)
    580 
    581 
    582 def main():
    583   # Handle arguments.
    584   parser = argparse.ArgumentParser()
    585   parser.add_argument('--num_tests', default=10000,
    586                       type=int, help='number of tests to run')
    587   parser.add_argument('--device', help='target device serial number')
    588   parser.add_argument('--mode1', default='ri',
    589                       help='execution mode 1 (default: ri)')
    590   parser.add_argument('--mode2', default='hopt',
    591                       help='execution mode 2 (default: hopt)')
    592   parser.add_argument('--report_script', help='script called for each'
    593                                               ' divergence')
    594   parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
    595                       action='append', help='argument for jfuzz')
    596   parser.add_argument('--true_divergence', default=False, action='store_true',
    597                       help='don\'t bisect timeout divergences')
    598   parser.add_argument('--use_dx', default=False, action='store_true',
    599                       help='use old-style dx (rather than jack)')
    600   args = parser.parse_args()
    601   if args.mode1 == args.mode2:
    602     raise FatalError('Identical execution modes given')
    603   # Run the JFuzz tester.
    604   with JFuzzTester(args.num_tests,
    605                    args.device, args.mode1, args.mode2,
    606                    args.jfuzz_args, args.report_script,
    607                    args.true_divergence, args.use_dx) as fuzzer:
    608     fuzzer.Run()
    609 
    610 if __name__ == '__main__':
    611   main()
    612