Home | History | Annotate | Download | only in utils 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import fnmatch
      6 import glob
      7 import os
      8 import shutil
      9 import sys
     10 import tempfile
     11 
     12 from devil.utils import cmd_helper
     13 from pylib import constants
     14 from pylib.constants import host_paths
     15 
     16 
     17 _ISOLATE_SCRIPT = os.path.join(
     18     host_paths.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
     19 
     20 
     21 def DefaultPathVariables():
     22   return {
     23     'DEPTH': host_paths.DIR_SOURCE_ROOT,
     24     'PRODUCT_DIR': constants.GetOutDirectory(),
     25   }
     26 
     27 
     28 def DefaultConfigVariables():
     29   # Note: This list must match the --config-vars in build/isolate.gypi
     30   return {
     31     'CONFIGURATION_NAME': constants.GetBuildType(),
     32     'OS': 'android',
     33     'asan': '0',
     34     'branding': 'Chromium',
     35     'chromeos': '0',
     36     'component': 'static_library',
     37     'enable_pepper_cdms': '0',
     38     'enable_plugins': '0',
     39     'fastbuild': '0',
     40     'icu_use_data_file_flag': '1',
     41     'kasko': '0',
     42     'lsan': '0',
     43     'msan': '0',
     44     # TODO(maruel): This may not always be true.
     45     'target_arch': 'arm',
     46     'tsan': '0',
     47     'use_custom_libcxx': '0',
     48     'use_instrumented_libraries': '0',
     49     'use_prebuilt_instrumented_libraries': '0',
     50     'use_ozone': '0',
     51     'use_x11': '0',
     52     'v8_use_external_startup_data': '1',
     53     'msvs_version': '0',
     54   }
     55 
     56 
     57 def IsIsolateEmpty(isolate_path):
     58   """Returns whether there are no files in the .isolate."""
     59   with open(isolate_path) as f:
     60     return "'files': []" in f.read()
     61 
     62 
     63 class Isolator(object):
     64   """Manages calls to isolate.py for the android test runner scripts."""
     65 
     66   def __init__(self):
     67     self._isolate_deps_dir = tempfile.mkdtemp()
     68 
     69   @property
     70   def isolate_deps_dir(self):
     71     return self._isolate_deps_dir
     72 
     73   def Clear(self):
     74     """Deletes the isolate dependency directory."""
     75     if os.path.exists(self._isolate_deps_dir):
     76       shutil.rmtree(self._isolate_deps_dir)
     77 
     78   def Remap(self, isolate_abs_path, isolated_abs_path,
     79             path_variables=None, config_variables=None):
     80     """Remaps data dependencies into |self._isolate_deps_dir|.
     81 
     82     Args:
     83       isolate_abs_path: The absolute path to the .isolate file, which specifies
     84         data dependencies in the source tree.
     85       isolated_abs_path: The absolute path to the .isolated file, which is
     86         generated by isolate.py and specifies data dependencies in
     87         |self._isolate_deps_dir| and their digests.
     88       path_variables: A dict containing everything that should be passed
     89         as a |--path-variable| to the isolate script. Defaults to the return
     90         value of |DefaultPathVariables()|.
     91       config_variables: A dict containing everything that should be passed
     92         as a |--config-variable| to the isolate script. Defaults to the return
     93         value of |DefaultConfigVariables()|.
     94     Raises:
     95       Exception if the isolate command fails for some reason.
     96     """
     97     if not path_variables:
     98       path_variables = DefaultPathVariables()
     99     if not config_variables:
    100       config_variables = DefaultConfigVariables()
    101 
    102     isolate_cmd = [
    103       sys.executable, _ISOLATE_SCRIPT, 'remap',
    104       '--isolate', isolate_abs_path,
    105       '--isolated', isolated_abs_path,
    106       '--outdir', self._isolate_deps_dir,
    107     ]
    108     for k, v in path_variables.iteritems():
    109       isolate_cmd.extend(['--path-variable', k, v])
    110     for k, v in config_variables.iteritems():
    111       isolate_cmd.extend(['--config-variable', k, v])
    112 
    113     exit_code, _ = cmd_helper.GetCmdStatusAndOutput(isolate_cmd)
    114     if exit_code:
    115       raise Exception('isolate command failed: %s' % ' '.join(isolate_cmd))
    116 
    117   def VerifyHardlinks(self):
    118     """Checks |isolate_deps_dir| for a hardlink.
    119 
    120     Returns:
    121       True if a hardlink is found.
    122       False if nothing is found.
    123     Raises:
    124       Exception if a non-hardlink is found.
    125     """
    126     for root, _, filenames in os.walk(self._isolate_deps_dir):
    127       if filenames:
    128         linked_file = os.path.join(root, filenames[0])
    129         orig_file = os.path.join(
    130             self._isolate_deps_dir,
    131             os.path.relpath(linked_file, self._isolate_deps_dir))
    132         if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
    133           return True
    134         else:
    135           raise Exception('isolate remap command did not use hardlinks.')
    136     return False
    137 
    138   def PurgeExcluded(self, deps_exclusion_list):
    139     """Deletes anything on |deps_exclusion_list| from |self._isolate_deps_dir|.
    140 
    141     Args:
    142       deps_exclusion_list: A list of globs to exclude from the isolate
    143         dependency directory.
    144     """
    145     excluded_paths = (
    146         x for y in deps_exclusion_list
    147         for x in glob.glob(
    148             os.path.abspath(os.path.join(self._isolate_deps_dir, y))))
    149     for p in excluded_paths:
    150       if os.path.isdir(p):
    151         shutil.rmtree(p)
    152       else:
    153         os.remove(p)
    154 
    155   @classmethod
    156   def _DestructiveMerge(cls, src, dest):
    157     if os.path.exists(dest) and os.path.isdir(dest):
    158       for p in os.listdir(src):
    159         cls._DestructiveMerge(os.path.join(src, p), os.path.join(dest, p))
    160       os.rmdir(src)
    161     else:
    162       shutil.move(src, dest)
    163 
    164 
    165   def MoveOutputDeps(self):
    166     """Moves files from the output directory to the top level of
    167       |self._isolate_deps_dir|.
    168 
    169     Moves pak files from the output directory to to <isolate_deps_dir>/paks
    170     Moves files from the product directory to <isolate_deps_dir>
    171     """
    172     # On Android, all pak files need to be in the top-level 'paks' directory.
    173     paks_dir = os.path.join(self._isolate_deps_dir, 'paks')
    174     os.mkdir(paks_dir)
    175 
    176     deps_out_dir = os.path.join(
    177         self._isolate_deps_dir,
    178         os.path.relpath(os.path.join(constants.GetOutDirectory(), os.pardir),
    179                         host_paths.DIR_SOURCE_ROOT))
    180     for root, _, filenames in os.walk(deps_out_dir):
    181       for filename in fnmatch.filter(filenames, '*.pak'):
    182         shutil.move(os.path.join(root, filename), paks_dir)
    183 
    184     # Move everything in PRODUCT_DIR to top level.
    185     deps_product_dir = os.path.join(
    186         deps_out_dir, os.path.basename(constants.GetOutDirectory()))
    187     if os.path.isdir(deps_product_dir):
    188       for p in os.listdir(deps_product_dir):
    189         Isolator._DestructiveMerge(os.path.join(deps_product_dir, p),
    190                                    os.path.join(self._isolate_deps_dir, p))
    191       os.rmdir(deps_product_dir)
    192       os.rmdir(deps_out_dir)
    193