1 #!/usr/bin/python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Do all the steps required to build and test against nacl.""" 7 8 9 import optparse 10 import os.path 11 import re 12 import shutil 13 import subprocess 14 import sys 15 16 import find_chrome 17 18 THIS_DIR = os.path.abspath(os.path.dirname(__file__)) 19 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) 20 sys.path.append(os.path.join(CHROMIUM_DIR, 'build')) 21 import detect_host_arch 22 23 24 # Copied from buildbot/buildbot_lib.py 25 def TryToCleanContents(path, file_name_filter=lambda fn: True): 26 """ 27 Remove the contents of a directory without touching the directory itself. 28 Ignores all failures. 29 """ 30 if os.path.exists(path): 31 for fn in os.listdir(path): 32 TryToCleanPath(os.path.join(path, fn), file_name_filter) 33 34 35 # Copied from buildbot/buildbot_lib.py 36 def TryToCleanPath(path, file_name_filter=lambda fn: True): 37 """ 38 Removes a file or directory. 39 Ignores all failures. 40 """ 41 if os.path.exists(path): 42 if file_name_filter(path): 43 print 'Trying to remove %s' % path 44 if os.path.isdir(path): 45 shutil.rmtree(path, ignore_errors=True) 46 else: 47 try: 48 os.remove(path) 49 except Exception: 50 pass 51 else: 52 print 'Skipping %s' % path 53 54 55 # TODO(ncbray): this is somewhat unsafe. We should fix the underlying problem. 56 def CleanTempDir(): 57 # Only delete files and directories like: 58 # a) C:\temp\83C4.tmp 59 # b) /tmp/.org.chromium.Chromium.EQrEzl 60 file_name_re = re.compile( 61 r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$') 62 file_name_filter = lambda fn: file_name_re.search(fn) is not None 63 64 path = os.environ.get('TMP', os.environ.get('TEMP', '/tmp')) 65 if len(path) >= 4 and os.path.isdir(path): 66 print 67 print "Cleaning out the temp directory." 68 print 69 TryToCleanContents(path, file_name_filter) 70 else: 71 print 72 print "Cannot find temp directory, not cleaning it." 73 print 74 75 76 def RunCommand(cmd, cwd, env): 77 sys.stdout.write('\nRunning %s\n\n' % ' '.join(cmd)) 78 sys.stdout.flush() 79 retcode = subprocess.call(cmd, cwd=cwd, env=env) 80 if retcode != 0: 81 sys.stdout.write('\nFailed: %s\n\n' % ' '.join(cmd)) 82 sys.exit(retcode) 83 84 85 def RunTests(name, cmd, nacl_dir, env): 86 sys.stdout.write('\n\nBuilding files needed for %s testing...\n\n' % name) 87 RunCommand(cmd + ['do_not_run_tests=1', '-j8'], nacl_dir, env) 88 sys.stdout.write('\n\nRunning %s tests...\n\n' % name) 89 RunCommand(cmd, nacl_dir, env) 90 91 92 def BuildAndTest(options): 93 # Refuse to run under cygwin. 94 if sys.platform == 'cygwin': 95 raise Exception('I do not work under cygwin, sorry.') 96 97 # By default, use the version of Python is being used to run this script. 98 python = sys.executable 99 if sys.platform == 'darwin': 100 # Mac 10.5 bots tend to use a particularlly old version of Python, look for 101 # a newer version. 102 macpython27 = '/Library/Frameworks/Python.framework/Versions/2.7/bin/python' 103 if os.path.exists(macpython27): 104 python = macpython27 105 106 script_dir = os.path.dirname(os.path.abspath(__file__)) 107 src_dir = os.path.dirname(os.path.dirname(os.path.dirname(script_dir))) 108 nacl_dir = os.path.join(src_dir, 'native_client') 109 110 # Decide platform specifics. 111 if options.browser_path: 112 chrome_filename = options.browser_path 113 else: 114 chrome_filename = find_chrome.FindChrome(src_dir, [options.mode]) 115 if chrome_filename is None: 116 raise Exception('Cannot find a chrome binary - specify one with ' 117 '--browser_path?') 118 119 env = dict(os.environ) 120 if sys.platform in ['win32', 'cygwin']: 121 if options.bits == 64: 122 bits = 64 123 elif options.bits == 32: 124 bits = 32 125 elif '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \ 126 '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''): 127 bits = 64 128 else: 129 bits = 32 130 msvs_path = ';'.join([ 131 r'c:\Program Files\Microsoft Visual Studio 9.0\VC', 132 r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC', 133 r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools', 134 r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools', 135 r'c:\Program Files\Microsoft Visual Studio 8\VC', 136 r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC', 137 r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools', 138 r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools', 139 ]) 140 env['PATH'] += ';' + msvs_path 141 scons = [python, 'scons.py'] 142 elif sys.platform == 'darwin': 143 if options.bits == 64: 144 bits = 64 145 elif options.bits == 32: 146 bits = 32 147 else: 148 p = subprocess.Popen(['file', chrome_filename], stdout=subprocess.PIPE) 149 (p_stdout, _) = p.communicate() 150 assert p.returncode == 0 151 if p_stdout.find('executable x86_64') >= 0: 152 bits = 64 153 else: 154 bits = 32 155 scons = [python, 'scons.py'] 156 else: 157 if options.bits == 64: 158 bits = 64 159 elif options.bits == 32: 160 bits = 32 161 elif '64' in detect_host_arch.HostArch(): 162 bits = 64 163 else: 164 bits = 32 165 # xvfb-run has a 2-second overhead per invocation, so it is cheaper to wrap 166 # the entire build step rather than each test (browser_headless=1). 167 # We also need to make sure that there are at least 24 bits per pixel. 168 # https://code.google.com/p/chromium/issues/detail?id=316687 169 scons = [ 170 'xvfb-run', 171 '--auto-servernum', 172 '--server-args', '-screen 0 1024x768x24', 173 python, 'scons.py', 174 ] 175 176 if options.jobs > 1: 177 scons.append('-j%d' % options.jobs) 178 179 scons.append('disable_tests=%s' % options.disable_tests) 180 181 if options.buildbot is not None: 182 scons.append('buildbot=%s' % (options.buildbot,)) 183 184 # Clean the output of the previous build. 185 # Incremental builds can get wedged in weird ways, so we're trading speed 186 # for reliability. 187 shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True) 188 189 # check that the HOST (not target) is 64bit 190 # this is emulating what msvs_env.bat is doing 191 if '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \ 192 '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''): 193 # 64bit HOST 194 env['VS90COMNTOOLS'] = ('c:\\Program Files (x86)\\' 195 'Microsoft Visual Studio 9.0\\Common7\\Tools\\') 196 env['VS80COMNTOOLS'] = ('c:\\Program Files (x86)\\' 197 'Microsoft Visual Studio 8.0\\Common7\\Tools\\') 198 else: 199 # 32bit HOST 200 env['VS90COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 9.0\\' 201 'Common7\\Tools\\') 202 env['VS80COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 8.0\\' 203 'Common7\\Tools\\') 204 205 # Run nacl/chrome integration tests. 206 # Note that we have to add nacl_irt_test to --mode in order to get 207 # inbrowser_test_runner to run. 208 # TODO(mseaborn): Change it so that inbrowser_test_runner is not a 209 # special case. 210 cmd = scons + ['--verbose', '-k', 'platform=x86-%d' % bits, 211 '--mode=opt-host,nacl,nacl_irt_test', 212 'chrome_browser_path=%s' % chrome_filename, 213 ] 214 if not options.integration_bot and not options.morenacl_bot: 215 cmd.append('disable_flaky_tests=1') 216 cmd.append('chrome_browser_tests') 217 218 # Propagate path to JSON output if present. 219 # Note that RunCommand calls sys.exit on errors, so potential errors 220 # from one command won't be overwritten by another one. Overwriting 221 # a successful results file with either success or failure is fine. 222 if options.json_build_results_output_file: 223 cmd.append('json_build_results_output_file=%s' % 224 options.json_build_results_output_file) 225 226 # Download the toolchain(s). 227 pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version') 228 RunCommand([python, os.path.join(pkg_ver_dir, 'package_version.py'), 229 '--exclude', 'arm_trusted', 230 '--exclude', 'pnacl_newlib', 231 '--exclude', 'nacl_arm_newlib', 232 'sync', '--extract'], 233 nacl_dir, os.environ) 234 235 CleanTempDir() 236 237 if options.enable_newlib: 238 RunTests('nacl-newlib', cmd, nacl_dir, env) 239 240 if options.enable_glibc: 241 RunTests('nacl-glibc', cmd + ['--nacl_glibc'], nacl_dir, env) 242 243 244 def MakeCommandLineParser(): 245 parser = optparse.OptionParser() 246 parser.add_option('-m', '--mode', dest='mode', default='Debug', 247 help='Debug/Release mode') 248 parser.add_option('-j', dest='jobs', default=1, type='int', 249 help='Number of parallel jobs') 250 251 parser.add_option('--enable_newlib', dest='enable_newlib', default=-1, 252 type='int', help='Run newlib tests?') 253 parser.add_option('--enable_glibc', dest='enable_glibc', default=-1, 254 type='int', help='Run glibc tests?') 255 256 parser.add_option('--json_build_results_output_file', 257 help='Path to a JSON file for machine-readable output.') 258 259 # Deprecated, but passed to us by a script in the Chrome repo. 260 # Replaced by --enable_glibc=0 261 parser.add_option('--disable_glibc', dest='disable_glibc', 262 action='store_true', default=False, 263 help='Do not test using glibc.') 264 265 parser.add_option('--disable_tests', dest='disable_tests', 266 type='string', default='', 267 help='Comma-separated list of tests to omit') 268 builder_name = os.environ.get('BUILDBOT_BUILDERNAME', '') 269 is_integration_bot = 'nacl-chrome' in builder_name 270 parser.add_option('--integration_bot', dest='integration_bot', 271 type='int', default=int(is_integration_bot), 272 help='Is this an integration bot?') 273 is_morenacl_bot = ( 274 'More NaCl' in builder_name or 275 'naclmore' in builder_name) 276 parser.add_option('--morenacl_bot', dest='morenacl_bot', 277 type='int', default=int(is_morenacl_bot), 278 help='Is this a morenacl bot?') 279 280 # Not used on the bots, but handy for running the script manually. 281 parser.add_option('--bits', dest='bits', action='store', 282 type='int', default=None, 283 help='32/64') 284 parser.add_option('--browser_path', dest='browser_path', action='store', 285 type='string', default=None, 286 help='Path to the chrome browser.') 287 parser.add_option('--buildbot', dest='buildbot', action='store', 288 type='string', default=None, 289 help='Value passed to scons as buildbot= option.') 290 return parser 291 292 293 def Main(): 294 parser = MakeCommandLineParser() 295 options, args = parser.parse_args() 296 if options.integration_bot and options.morenacl_bot: 297 parser.error('ERROR: cannot be both an integration bot and a morenacl bot') 298 299 # Set defaults for enabling newlib. 300 if options.enable_newlib == -1: 301 options.enable_newlib = 1 302 303 # Set defaults for enabling glibc. 304 if options.enable_glibc == -1: 305 if options.integration_bot or options.morenacl_bot: 306 options.enable_glibc = 1 307 else: 308 options.enable_glibc = 0 309 310 if args: 311 parser.error('ERROR: invalid argument') 312 BuildAndTest(options) 313 314 315 if __name__ == '__main__': 316 Main() 317
