Home | History | Annotate | Download | only in isolate 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 
      6 # TODO(borenet): This module was copied from build.git and heavily modified to
      7 # remove dependencies on other modules in build.git.  It belongs in a different
      8 # repo. Remove this once it has been moved.
      9 
     10 
     11 import itertools
     12 from recipe_engine import recipe_api
     13 
     14 
     15 class IsolateApi(recipe_api.RecipeApi):
     16   """APIs for interacting with isolates."""
     17 
     18   def __init__(self, **kwargs):
     19     super(IsolateApi, self).__init__(**kwargs)
     20     self._isolate_server = 'https://isolateserver.appspot.com'
     21     self._isolated_tests = {}
     22 
     23   @property
     24   def isolate_server(self):
     25     """URL of Isolate server to use, default is a production one."""
     26     return self._isolate_server
     27 
     28   @isolate_server.setter
     29   def isolate_server(self, value):
     30     """Changes URL of Isolate server to use."""
     31     self._isolate_server = value
     32 
     33   def clean_isolated_files(self, build_dir):
     34     """Cleans out all *.isolated files from the build directory in
     35     preparation for the compile. Needed in order to ensure isolates
     36     are rebuilt properly because their dependencies are currently not
     37     completely described to gyp.
     38     """
     39     self.m.python(
     40       'clean isolated files',
     41       self.resource('find_isolated_tests.py'),
     42       [
     43         '--build-dir', build_dir,
     44         '--clean-isolated-files'
     45       ])
     46 
     47   def find_isolated_tests(self, build_dir, targets=None, **kwargs):
     48     """Returns a step which finds all *.isolated files in a build directory.
     49 
     50     Useful only with 'archive' isolation mode.
     51     In 'prepare' mode use 'isolate_tests' instead.
     52 
     53     Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes
     54     build property. This implies this step can currently only be run once
     55     per recipe.
     56 
     57     If |targets| is None, the step will use all *.isolated files it finds.
     58     Otherwise, it will verify that all |targets| are found and will use only
     59     them. If some expected targets are missing, will abort the build.
     60     """
     61     step_result = self.m.python(
     62       'find isolated tests',
     63       self.resource('find_isolated_tests.py'),
     64       [
     65         '--build-dir', build_dir,
     66         '--output-json', self.m.json.output(),
     67       ],
     68       step_test_data=lambda: (self.test_api.output_json(targets)),
     69       **kwargs)
     70 
     71     assert isinstance(step_result.json.output, dict)
     72     self._isolated_tests = step_result.json.output
     73     if targets is not None and (
     74             step_result.presentation.status != self.m.step.FAILURE):
     75       found = set(step_result.json.output)
     76       expected = set(targets)
     77       if found >= expected:  # pragma: no cover
     78         # Limit result only to |expected|.
     79         self._isolated_tests = {
     80           target: step_result.json.output[target] for target in expected
     81         }
     82       else:
     83         # Some expected targets are missing? Fail the step.
     84         step_result.presentation.status = self.m.step.FAILURE
     85         step_result.presentation.logs['missing.isolates'] = (
     86             ['Failed to find *.isolated files:'] + list(expected - found))
     87     step_result.presentation.properties['swarm_hashes'] = self._isolated_tests
     88     # No isolated files found? That looks suspicious, emit warning.
     89     if (not self._isolated_tests and
     90         step_result.presentation.status != self.m.step.FAILURE):
     91       step_result.presentation.status = self.m.step.WARNING
     92 
     93   def isolate_tests(self, build_dir, targets=None, verbose=False,
     94                     set_swarm_hashes=True, always_use_exparchive=False,
     95                     **kwargs):
     96     """Archives prepared tests in |build_dir| to isolate server.
     97 
     98     src/tools/isolate_driver.py is invoked by ninja during compilation
     99     to produce *.isolated.gen.json files that describe how to archive tests.
    100 
    101     This step then uses *.isolated.gen.json files to actually performs the
    102     archival. By archiving all tests at once it is able to reduce the total
    103     amount of work. Tests share many common files, and such files are processed
    104     only once.
    105 
    106     Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes
    107     build property (also accessible as 'isolated_tests' property). This implies
    108     this step can currently only be run once per recipe.
    109     """
    110     # TODO(tansell): Make all steps in this function nested under one overall
    111     # 'isolate tests' master step.
    112 
    113     # TODO(vadimsh): Always require |targets| to be passed explicitly. Currently
    114     # chromium_trybot, blink_trybot and swarming/canary recipes rely on targets
    115     # autodiscovery. The code path in chromium_trybot that needs it is being
    116     # deprecated in favor of to *_ng builders, that pass targets explicitly.
    117     if targets is None:
    118       # Ninja builds <target>.isolated.gen.json files via isolate_driver.py.
    119       paths = self.m.file.glob_paths(
    120           'find isolated targets',
    121           build_dir,
    122           '*.isolated.gen.json',
    123           test_data=[
    124               'dummy_target_%d.isolated.gen.json' % i
    125               for i in (1, 2)
    126           ])
    127       targets = []
    128       for p in paths:
    129         name = self.m.path.basename(p)
    130         assert name.endswith('.isolated.gen.json'), name
    131         targets.append(name[:-len('.isolated.gen.json')])
    132 
    133     # No isolated tests found.
    134     if not targets:  # pragma: no cover
    135       return
    136 
    137     batch_targets = []
    138     exparchive_targets = []
    139     for t in targets:
    140       if t.endswith('_exparchive') or always_use_exparchive:
    141         exparchive_targets.append(t)
    142       else:
    143         batch_targets.append(t)
    144 
    145     isolate_steps = []
    146     try:
    147       args = [
    148           self.m.swarming_client.path,
    149           'exparchive',
    150           '--dump-json', self.m.json.output(),
    151           '--isolate-server', self._isolate_server,
    152           '--eventlog-endpoint', 'prod',
    153       ] + (['--verbose'] if verbose else [])
    154 
    155       for target in exparchive_targets:
    156         isolate_steps.append(
    157             self.m.python(
    158                 'isolate %s' % target,
    159                 self.resource('isolate.py'),
    160                 args + [
    161                     '--isolate', build_dir.join('%s.isolate' % target),
    162                     '--isolated', build_dir.join('%s.isolated' % target),
    163                 ],
    164                 step_test_data=lambda: self.test_api.output_json([target]),
    165                 **kwargs))
    166 
    167       if batch_targets:
    168         # TODO(vadimsh): Differentiate between bad *.isolate and upload errors.
    169         # Raise InfraFailure on upload errors.
    170         args = [
    171             self.m.swarming_client.path,
    172             'batcharchive',
    173             '--dump-json', self.m.json.output(),
    174             '--isolate-server', self._isolate_server,
    175         ] + (['--verbose'] if verbose else []) +  [
    176             build_dir.join('%s.isolated.gen.json' % t) for t in batch_targets
    177         ]
    178         isolate_steps.append(
    179             self.m.python(
    180                 'isolate tests', self.resource('isolate.py'), args,
    181                 step_test_data=lambda: self.test_api.output_json(batch_targets),
    182                 **kwargs))
    183 
    184       # TODO(tansell): Change this to return a dummy "isolate results" or the
    185       # top level master step.
    186       return isolate_steps[-1]
    187     finally:
    188       step_result = self.m.step.active_result
    189       swarm_hashes = {}
    190       for step in isolate_steps:
    191         if not step.json.output:
    192           continue  # pragma: no cover
    193 
    194         for k, v in step.json.output.iteritems():
    195           # TODO(tansell): Raise an error here when it can't clobber an
    196           # existing error. This code is currently inside a finally block,
    197           # meaning it could be executed when an existing error is occurring.
    198           # See https://chromium-review.googlesource.com/c/437024/
    199           #assert k not in swarm_hashes or swarm_hashes[k] == v, (
    200           #    "Duplicate hash for target %s was found at step %s."
    201           #    "Existing hash: %s, New hash: %s") % (
    202           #        k, step, swarm_hashes[k], v)
    203           swarm_hashes[k] = v
    204 
    205       if swarm_hashes:
    206         self._isolated_tests = swarm_hashes
    207 
    208       if set_swarm_hashes:
    209         step_result.presentation.properties['swarm_hashes'] = swarm_hashes
    210 
    211       missing = sorted(
    212           t for t, h in self._isolated_tests.iteritems() if not h)
    213       if missing:
    214         step_result.presentation.logs['failed to isolate'] = (
    215             ['Failed to isolate following targets:'] +
    216             missing +
    217             ['', 'See logs for more information.']
    218         )
    219         for k in missing:
    220           self._isolated_tests.pop(k)
    221 
    222   @property
    223   def isolated_tests(self):
    224     """The dictionary of 'target name -> isolated hash' for this run.
    225 
    226     These come either from the incoming swarm_hashes build property,
    227     or from calling find_isolated_tests, above, at some point during the run.
    228     """
    229     hashes = self.m.properties.get('swarm_hashes', self._isolated_tests)
    230     # Be robust in the case where swarm_hashes is an empty string
    231     # instead of an empty dictionary, or similar.
    232     if not hashes:
    233       return {} # pragma: no covergae
    234     return {
    235       k.encode('ascii'): v.encode('ascii')
    236       for k, v in hashes.iteritems()
    237     }
    238 
    239   @property
    240   def _run_isolated_path(self):
    241     """Returns the path to run_isolated.py."""
    242     return self.m.swarming_client.path.join('run_isolated.py')
    243 
    244   def run_isolated(self, name, isolate_hash, args=None, **kwargs):
    245     """Runs an isolated test."""
    246     cmd = [
    247         '--isolated', isolate_hash,
    248         '-I', self.isolate_server,
    249         '--verbose',
    250     ]
    251     if args:
    252       cmd.append('--')
    253       cmd.extend(args)
    254     self.m.python(name, self._run_isolated_path, cmd, **kwargs)
    255