Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Windows can't run .sh files, so this is a Python implementation of
      7 update.sh. This script should replace update.sh on all platforms eventually."""
      8 
      9 import os
     10 import re
     11 import shutil
     12 import subprocess
     13 import sys
     14 
     15 # Do NOT CHANGE this if you don't know what you're doing -- see
     16 # https://code.google.com/p/chromium/wiki/UpdatingClang
     17 # Reverting problematic clang rolls is safe, though.
     18 # Note: this revision is only used for Windows. Other platforms use update.sh.
     19 LLVM_WIN_REVISION = 'HEAD'
     20 
     21 # ASan on Windows is useful enough to use it even while the clang/win is still
     22 # in bringup. Use a pinned revision to make it slightly more stable.
     23 if (re.search(r'\b(asan)=1', os.environ.get('GYP_DEFINES', '')) and
     24     not 'LLVM_FORCE_HEAD_REVISION' in os.environ):
     25   LLVM_WIN_REVISION = '210586'
     26 
     27 # Path constants. (All of these should be absolute paths.)
     28 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
     29 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
     30 LLVM_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm')
     31 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
     32                               'Release+Asserts')
     33 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
     34 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
     35 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
     36 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
     37 
     38 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
     39 if 'LLVM_REPO_URL' in os.environ:
     40   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
     41 
     42 
     43 def ReadStampFile():
     44   """Return the contents of the stamp file, or '' if it doesn't exist."""
     45   try:
     46     with open(STAMP_FILE, 'r') as f:
     47       return f.read();
     48   except IOError:
     49     return ''
     50 
     51 
     52 def WriteStampFile(s):
     53   """Write s to the stamp file."""
     54   if not os.path.exists(LLVM_BUILD_DIR):
     55     os.makedirs(LLVM_BUILD_DIR)
     56   with open(STAMP_FILE, 'w') as f:
     57     f.write(s)
     58 
     59 
     60 def DeleteFiles(dir, pattern):
     61   """Delete all files in dir matching pattern."""
     62   n = 0
     63   regex = re.compile(r'^' + pattern + r'$')
     64   for root, _, files in os.walk(dir):
     65     for f in files:
     66       if regex.match(f):
     67         os.remove(os.path.join(root, f))
     68         n += 1
     69   return n
     70 
     71 
     72 def ClobberChromiumBuildFiles():
     73   """Clobber Chomium build files."""
     74   print 'Clobbering Chromium build files...'
     75   out_dir = os.path.join(CHROMIUM_DIR, 'out')
     76   if os.path.isdir(out_dir):
     77     shutil.rmtree(out_dir)
     78     print 'Removed Chromium out dir: %s.' % (out_dir)
     79 
     80 
     81 def RunCommand(command, tries=1):
     82   """Run a command, possibly with multiple retries."""
     83   for i in range(0, tries):
     84     print 'Running %s (try #%d)' % (str(command), i + 1)
     85     if subprocess.call(command, shell=True) == 0:
     86       return
     87     print 'Failed.'
     88   sys.exit(1)
     89 
     90 def CopyFile(src, dst):
     91   """Copy a file from src to dst."""
     92   shutil.copy(src, dst)
     93   print "Copying %s to %s" % (src, dst)
     94 
     95 def Checkout(name, url, dir):
     96   """Checkout the SVN module at url into dir. Use name for the log message."""
     97   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
     98   RunCommand(['svn', 'checkout', '--force',
     99               url + '@' + LLVM_WIN_REVISION, dir], tries=2)
    100 
    101 
    102 vs_version = None
    103 def GetVSVersion():
    104   global vs_version
    105   if not vs_version:
    106     # TODO(hans): Find a less hacky way to find the MSVS installation.
    107     sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
    108     import gyp.MSVSVersion
    109     # We request VS 2013 because Clang won't build with 2010, and 2013 will be
    110     # the default for Chromium soon anyway.
    111     vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
    112   return vs_version
    113 
    114 
    115 def UpdateClang():
    116   print 'Updating Clang to %s...' % (LLVM_WIN_REVISION)
    117   if LLVM_WIN_REVISION != 'HEAD' and ReadStampFile() == LLVM_WIN_REVISION:
    118     print 'Already up to date.'
    119     return 0
    120 
    121   ClobberChromiumBuildFiles()
    122 
    123   # Reset the stamp file in case the build is unsuccessful.
    124   WriteStampFile('')
    125 
    126   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
    127   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
    128   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
    129 
    130   if not os.path.exists(LLVM_BUILD_DIR):
    131     os.makedirs(LLVM_BUILD_DIR)
    132   os.chdir(LLVM_BUILD_DIR)
    133 
    134   if not re.search(r'cmake', os.environ['PATH'], flags=re.IGNORECASE):
    135     # If CMake is not on the path, try looking in a standard location.
    136     os.environ['PATH'] += os.pathsep + 'C:\\Program Files (x86)\\CMake 2.8\\bin'
    137 
    138   RunCommand(GetVSVersion().SetupScript('x64') +
    139              ['&&', 'cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=Release',
    140               '-DLLVM_ENABLE_ASSERTIONS=ON', LLVM_DIR])
    141   RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'ninja', 'all'])
    142 
    143   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
    144   # TODO(hans): Remove once the regular build above produces this.
    145   if not os.path.exists(COMPILER_RT_BUILD_DIR):
    146     os.makedirs(COMPILER_RT_BUILD_DIR)
    147   os.chdir(COMPILER_RT_BUILD_DIR)
    148   RunCommand(GetVSVersion().SetupScript('x86') +
    149              ['&&', 'cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=Release',
    150               '-DLLVM_ENABLE_ASSERTIONS=ON', LLVM_DIR])
    151   RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'ninja', 'compiler-rt'])
    152 
    153   # TODO(hans): Make this (and the .gypi file) version number independent.
    154   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
    155                                      '3.5.0', 'lib', 'windows')
    156   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
    157                                      '3.5.0', 'lib', 'windows')
    158 
    159   if not os.path.exists(asan_rt_lib_dst_dir):
    160     os.makedirs(asan_rt_lib_dst_dir)
    161   for root, _, files in os.walk(asan_rt_lib_src_dir):
    162     for f in files:
    163       if re.match(r'^.*-i386\.lib$', f):
    164         CopyFile(os.path.join(root, f), asan_rt_lib_dst_dir)
    165 
    166   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
    167            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
    168 
    169   # Make an extra copy of the sanitizer headers, to be put on the include path
    170   # of the fallback compiler.
    171   sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', '3.5.0',
    172                                        'include', 'sanitizer')
    173   aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
    174                                            '3.5.0', 'include_sanitizer',
    175                                            'sanitizer')
    176   if not os.path.exists(aux_sanitizer_include_dir):
    177     os.makedirs(aux_sanitizer_include_dir)
    178   for _, _, files in os.walk(sanitizer_include_dir):
    179     for f in files:
    180       CopyFile(os.path.join(sanitizer_include_dir, f),
    181                aux_sanitizer_include_dir)
    182 
    183   WriteStampFile(LLVM_WIN_REVISION)
    184   print 'Clang update was successful.'
    185   return 0
    186 
    187 
    188 def main():
    189   if not sys.platform in ['win32', 'cygwin']:
    190     # For non-Windows, fall back to update.sh.
    191     # TODO(hans): Make update.py replace update.sh completely.
    192 
    193     # This script is called by gclient. gclient opens its hooks subprocesses
    194     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
    195     # custom output processing that breaks printing '\r' characters for
    196     # single-line updating status messages as printed by curl and wget.
    197     # Work around this by setting stderr of the update.sh process to stdin (!):
    198     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
    199     # dup()ed sys.stdin is writable, try
    200     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
    201     # TODO: Fix gclient instead, http://crbug.com/95350
    202     return subprocess.call(
    203         [os.path.join(os.path.dirname(__file__), 'update.sh')] +  sys.argv[1:],
    204         stderr=os.fdopen(os.dup(sys.stdin.fileno())))
    205 
    206   if not re.search(r'\b(clang|asan)=1', os.environ.get('GYP_DEFINES', '')):
    207     print 'Skipping Clang update (clang=1 was not set in GYP_DEFINES).'
    208     return 0
    209 
    210   if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
    211     print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
    212     return 0
    213 
    214   return UpdateClang()
    215 
    216 
    217 if __name__ == '__main__':
    218   sys.exit(main())
    219