Home | History | Annotate | Download | only in gypfiles 
      1 #!/usr/bin/env python
      2 # Copyright 2014 the V8 project authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """
      7 This script runs every build as the first hook (See DEPS). If it detects that
      8 the build should be clobbered, it will delete the contents of the build
      9 directory.
     10 
     11 A landmine is tripped when a builder checks out a different revision, and the
     12 diff between the new landmines and the old ones is non-null. At this point, the
     13 build is clobbered.
     14 """
     15 
     16 import difflib
     17 import errno
     18 import gyp_environment
     19 import logging
     20 import optparse
     21 import os
     22 import re
     23 import shutil
     24 import sys
     25 import subprocess
     26 import time
     27 
     28 import landmine_utils
     29 
     30 
     31 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     32 
     33 
     34 def get_build_dir(build_tool, is_iphone=False):
     35   """
     36   Returns output directory absolute path dependent on build and targets.
     37   Examples:
     38     r'c:\b\build\slave\win\build\src\out'
     39     '/mnt/data/b/build/slave/linux/build/src/out'
     40     '/b/build/slave/ios_rel_device/build/src/xcodebuild'
     41 
     42   Keep this function in sync with tools/build/scripts/slave/compile.py
     43   """
     44   ret = None
     45   if build_tool == 'xcode':
     46     ret = os.path.join(SRC_DIR, 'xcodebuild')
     47   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
     48     if 'CHROMIUM_OUT_DIR' in os.environ:
     49       output_dir = os.environ.get('CHROMIUM_OUT_DIR').strip()
     50       if not output_dir:
     51         raise Error('CHROMIUM_OUT_DIR environment variable is set but blank!')
     52     else:
     53       output_dir = landmine_utils.gyp_generator_flags().get('output_dir', 'out')
     54     ret = os.path.join(SRC_DIR, output_dir)
     55   elif build_tool in ['msvs', 'vs', 'ib']:
     56     ret = os.path.join(SRC_DIR, 'build')
     57   else:
     58     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
     59   return os.path.abspath(ret)
     60 
     61 
     62 def extract_gn_build_commands(build_ninja_file):
     63   """Extracts from a build.ninja the commands to run GN.
     64 
     65   The commands to run GN are the gn rule and build.ninja build step at the
     66   top of the build.ninja file. We want to keep these when deleting GN builds
     67   since we want to preserve the command-line flags to GN.
     68 
     69   On error, returns the empty string."""
     70   result = ""
     71   with open(build_ninja_file, 'r') as f:
     72     # Read until the second blank line. The first thing GN writes to the file
     73     # is the "rule gn" and the second is the section for "build build.ninja",
     74     # separated by blank lines.
     75     num_blank_lines = 0
     76     while num_blank_lines < 2:
     77       line = f.readline()
     78       if len(line) == 0:
     79         return ''  # Unexpected EOF.
     80       result += line
     81       if line[0] == '\n':
     82         num_blank_lines = num_blank_lines + 1
     83   return result
     84 
     85 def delete_build_dir(build_dir):
     86   # GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
     87   build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
     88   if not os.path.exists(build_ninja_d_file):
     89     shutil.rmtree(build_dir)
     90     return
     91 
     92   # GN builds aren't automatically regenerated when you sync. To avoid
     93   # messing with the GN workflow, erase everything but the args file, and
     94   # write a dummy build.ninja file that will automatically rerun GN the next
     95   # time Ninja is run.
     96   build_ninja_file = os.path.join(build_dir, 'build.ninja')
     97   build_commands = extract_gn_build_commands(build_ninja_file)
     98 
     99   try:
    100     gn_args_file = os.path.join(build_dir, 'args.gn')
    101     with open(gn_args_file, 'r') as f:
    102       args_contents = f.read()
    103   except IOError:
    104     args_contents = ''
    105 
    106   shutil.rmtree(build_dir)
    107 
    108   # Put back the args file (if any).
    109   os.mkdir(build_dir)
    110   if args_contents != '':
    111     with open(gn_args_file, 'w') as f:
    112       f.write(args_contents)
    113 
    114   # Write the build.ninja file sufficiently to regenerate itself.
    115   with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
    116     if build_commands != '':
    117       f.write(build_commands)
    118     else:
    119       # Couldn't parse the build.ninja file, write a default thing.
    120       f.write('''rule gn
    121 command = gn -q gen //out/%s/
    122 description = Regenerating ninja files
    123 
    124 build build.ninja: gn
    125 generator = 1
    126 depfile = build.ninja.d
    127 ''' % (os.path.split(build_dir)[1]))
    128 
    129   # Write a .d file for the build which references a nonexistant file. This
    130   # will make Ninja always mark the build as dirty.
    131   with open(build_ninja_d_file, 'w') as f:
    132     f.write('build.ninja: nonexistant_file.gn\n')
    133 
    134 
    135 def needs_clobber(landmines_path, new_landmines):
    136   if os.path.exists(landmines_path):
    137     with open(landmines_path, 'r') as f:
    138       old_landmines = f.readlines()
    139     if old_landmines != new_landmines:
    140       old_date = time.ctime(os.stat(landmines_path).st_ctime)
    141       diff = difflib.unified_diff(old_landmines, new_landmines,
    142           fromfile='old_landmines', tofile='new_landmines',
    143           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
    144       sys.stdout.write('Clobbering due to:\n')
    145       sys.stdout.writelines(diff)
    146       return True
    147   else:
    148     sys.stdout.write('Clobbering due to missing landmines file.\n')
    149     return True
    150   return False
    151 
    152 
    153 def clobber_if_necessary(new_landmines):
    154   """Does the work of setting, planting, and triggering landmines."""
    155   out_dir = get_build_dir(landmine_utils.builder())
    156   landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
    157   try:
    158     os.makedirs(out_dir)
    159   except OSError as e:
    160     if e.errno == errno.EEXIST:
    161       pass
    162 
    163   if needs_clobber(landmines_path, new_landmines):
    164     # Clobber contents of build directory but not directory itself: some
    165     # checkouts have the build directory mounted.
    166     for f in os.listdir(out_dir):
    167       path = os.path.join(out_dir, f)
    168       if os.path.basename(out_dir) == 'build':
    169         # Only delete build directories and files for MSVS builds as the folder
    170         # shares some checked out files and directories.
    171         if (os.path.isdir(path) and
    172             re.search(r'(?:[Rr]elease)|(?:[Dd]ebug)', f)):
    173           delete_build_dir(path)
    174         elif (os.path.isfile(path) and
    175               (path.endswith('.sln') or
    176                path.endswith('.vcxproj') or
    177                path.endswith('.vcxproj.user'))):
    178           os.unlink(path)
    179       else:
    180         if os.path.isfile(path):
    181           os.unlink(path)
    182         elif os.path.isdir(path):
    183           delete_build_dir(path)
    184     if os.path.basename(out_dir) == 'xcodebuild':
    185       # Xcodebuild puts an additional project file structure into build,
    186       # while the output folder is xcodebuild.
    187       project_dir = os.path.join(SRC_DIR, 'build', 'all.xcodeproj')
    188       if os.path.exists(project_dir) and os.path.isdir(project_dir):
    189         delete_build_dir(project_dir)
    190 
    191   # Save current set of landmines for next time.
    192   with open(landmines_path, 'w') as f:
    193     f.writelines(new_landmines)
    194 
    195 
    196 def process_options():
    197   """Returns a list of landmine emitting scripts."""
    198   parser = optparse.OptionParser()
    199   parser.add_option(
    200       '-s', '--landmine-scripts', action='append',
    201       default=[os.path.join(SRC_DIR, 'gypfiles', 'get_landmines.py')],
    202       help='Path to the script which emits landmines to stdout. The target '
    203            'is passed to this script via option -t. Note that an extra '
    204            'script can be specified via an env var EXTRA_LANDMINES_SCRIPT.')
    205   parser.add_option('-v', '--verbose', action='store_true',
    206       default=('LANDMINES_VERBOSE' in os.environ),
    207       help=('Emit some extra debugging information (default off). This option '
    208           'is also enabled by the presence of a LANDMINES_VERBOSE environment '
    209           'variable.'))
    210 
    211   options, args = parser.parse_args()
    212 
    213   if args:
    214     parser.error('Unknown arguments %s' % args)
    215 
    216   logging.basicConfig(
    217       level=logging.DEBUG if options.verbose else logging.ERROR)
    218 
    219   extra_script = os.environ.get('EXTRA_LANDMINES_SCRIPT')
    220   if extra_script:
    221     return options.landmine_scripts + [extra_script]
    222   else:
    223     return options.landmine_scripts
    224 
    225 
    226 def main():
    227   landmine_scripts = process_options()
    228 
    229   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
    230     return 0
    231 
    232   gyp_environment.set_environment()
    233 
    234   landmines = []
    235   for s in landmine_scripts:
    236     proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
    237     output, _ = proc.communicate()
    238     landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
    239   clobber_if_necessary(landmines)
    240 
    241   return 0
    242 
    243 
    244 if __name__ == '__main__':
    245   sys.exit(main())
    246