1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import collections 8 import copy 9 import json 10 import os 11 import pipes 12 import re 13 import subprocess 14 import sys 15 16 import bb_utils 17 18 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 19 from pylib import constants 20 21 22 CHROMIUM_COVERAGE_BUCKET = 'chromium-code-coverage' 23 24 _BotConfig = collections.namedtuple( 25 'BotConfig', ['bot_id', 'host_obj', 'test_obj']) 26 27 HostConfig = collections.namedtuple( 28 'HostConfig', 29 ['script', 'host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch']) 30 31 TestConfig = collections.namedtuple('Tests', ['script', 'tests', 'extra_args']) 32 33 34 def BotConfig(bot_id, host_object, test_object=None): 35 return _BotConfig(bot_id, host_object, test_object) 36 37 38 def DictDiff(d1, d2): 39 diff = [] 40 for key in sorted(set(d1.keys() + d2.keys())): 41 if key in d1 and d1[key] != d2.get(key): 42 diff.append('- %s=%s' % (key, pipes.quote(d1[key]))) 43 if key in d2 and d2[key] != d1.get(key): 44 diff.append('+ %s=%s' % (key, pipes.quote(d2[key]))) 45 return '\n'.join(diff) 46 47 48 def GetEnvironment(host_obj, testing, extra_env_vars=None): 49 init_env = dict(os.environ) 50 init_env['GYP_GENERATORS'] = 'ninja' 51 if extra_env_vars: 52 init_env.update(extra_env_vars) 53 envsetup_cmd = '. build/android/envsetup.sh' 54 if testing: 55 # Skip envsetup to avoid presubmit dependence on android deps. 56 print 'Testing mode - skipping "%s"' % envsetup_cmd 57 envsetup_cmd = ':' 58 else: 59 print 'Running %s' % envsetup_cmd 60 proc = subprocess.Popen(['bash', '-exc', 61 envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'], 62 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 63 cwd=bb_utils.CHROME_SRC, env=init_env) 64 json_env, envsetup_output = proc.communicate() 65 if proc.returncode != 0: 66 print >> sys.stderr, 'FATAL Failure in envsetup.' 67 print >> sys.stderr, envsetup_output 68 sys.exit(1) 69 env = json.loads(json_env) 70 env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + \ 71 ' OS=android fastbuild=1 use_goma=1 gomadir=%s' % bb_utils.GOMA_DIR 72 if host_obj.target_arch: 73 env['GYP_DEFINES'] += ' target_arch=%s' % host_obj.target_arch 74 extra_gyp = host_obj.extra_gyp_defines 75 if extra_gyp: 76 env['GYP_DEFINES'] += ' %s' % extra_gyp 77 if re.search('(asan|clang)=1', extra_gyp): 78 env.pop('CXX_target', None) 79 80 # Bots checkout chrome in /b/build/slave/<name>/build/src 81 build_internal_android = os.path.abspath(os.path.join( 82 bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', 83 'scripts', 'slave', 'android')) 84 if os.path.exists(build_internal_android): 85 env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']]) 86 return env 87 88 89 def GetCommands(options, bot_config): 90 """Get a formatted list of commands. 91 92 Args: 93 options: Options object. 94 bot_config: A BotConfig named tuple. 95 host_step_script: Host step script. 96 device_step_script: Device step script. 97 Returns: 98 list of Command objects. 99 """ 100 property_args = bb_utils.EncodeProperties(options) 101 commands = [[bot_config.host_obj.script, 102 '--steps=%s' % ','.join(bot_config.host_obj.host_steps)] + 103 property_args + (bot_config.host_obj.extra_args or [])] 104 105 test_obj = bot_config.test_obj 106 if test_obj: 107 run_test_cmd = [test_obj.script] + property_args 108 for test in test_obj.tests: 109 run_test_cmd.extend(['-f', test]) 110 if test_obj.extra_args: 111 run_test_cmd.extend(test_obj.extra_args) 112 commands.append(run_test_cmd) 113 return commands 114 115 116 def GetBotStepMap(): 117 compile_step = ['compile'] 118 python_unittests = ['python_unittests'] 119 std_host_tests = ['check_webview_licenses'] 120 std_build_steps = ['compile', 'zip_build'] 121 std_test_steps = ['extract_build'] 122 std_tests = ['ui', 'unit'] 123 trial_tests = [ 124 'base_junit_tests', 125 'components_browsertests', 126 'gfx_unittests', 127 'gl_unittests', 128 ] 129 flakiness_server = ( 130 '--flakiness-server=%s' % constants.UPSTREAM_FLAKINESS_SERVER) 131 experimental = ['--experimental'] 132 run_mb = ['--run-mb'] 133 bisect_chrome_output_dir = os.path.abspath( 134 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 135 os.pardir, 'bisect', 'src', 'out')) 136 B = BotConfig 137 H = (lambda steps, extra_args=None, extra_gyp=None, target_arch=None: 138 HostConfig('build/android/buildbot/bb_host_steps.py', steps, extra_args, 139 extra_gyp, target_arch)) 140 T = (lambda tests, extra_args=None: 141 TestConfig('build/android/buildbot/bb_device_steps.py', tests, 142 extra_args)) 143 144 bot_configs = [ 145 # Main builders 146 B('main-builder-dbg', H(std_build_steps + std_host_tests)), 147 B('main-builder-rel', H(std_build_steps)), 148 B('main-clang-builder', 149 H(compile_step, extra_gyp='clang=1 component=shared_library')), 150 B('main-clobber', H(compile_step)), 151 B('main-tests-rel', H(std_test_steps), 152 T(std_tests, ['--cleanup', flakiness_server])), 153 B('main-tests', H(std_test_steps), 154 T(std_tests, ['--cleanup', flakiness_server])), 155 156 # Other waterfalls 157 B('asan-builder-tests', H(compile_step, 158 extra_gyp='asan=1 component=shared_library'), 159 T(std_tests, ['--asan', '--asan-symbolize'])), 160 B('blink-try-builder', H(compile_step)), 161 B('chromedriver-fyi-tests-dbg', H(std_test_steps), 162 T(['chromedriver'], 163 ['--install=ChromePublic', '--install=ChromeDriverWebViewShell', 164 '--skip-wipe', '--disable-location', '--cleanup'])), 165 B('fyi-x86-builder-dbg', 166 H(compile_step + std_host_tests, experimental, target_arch='ia32')), 167 B('fyi-builder-dbg', 168 H(std_build_steps + std_host_tests, experimental, 169 extra_gyp='emma_coverage=1')), 170 B('x86-builder-dbg', 171 H(compile_step + std_host_tests, target_arch='ia32')), 172 B('fyi-builder-rel', H(std_build_steps, experimental)), 173 B('fyi-tests', H(std_test_steps), 174 T(std_tests + python_unittests, 175 ['--experimental', flakiness_server, 176 '--coverage-bucket', CHROMIUM_COVERAGE_BUCKET, 177 '--cleanup'])), 178 B('user-build-fyi-tests-dbg', H(std_test_steps), 179 T(sorted(trial_tests))), 180 B('fyi-component-builder-tests-dbg', 181 H(compile_step, extra_gyp='component=shared_library'), 182 T(std_tests, ['--experimental', flakiness_server])), 183 B('gpu-builder-tests-dbg', 184 H(compile_step, extra_args=run_mb), 185 T(['gpu'], ['--install=ContentShell'])), 186 # Pass empty T([]) so that logcat monitor and device status check are run. 187 B('perf-bisect-builder-tests-dbg', 188 H(['bisect_perf_regression']), 189 T([], ['--chrome-output-dir', bisect_chrome_output_dir])), 190 B('perf-tests-rel', H(std_test_steps), 191 T([], ['--cleanup'])), 192 B('webkit-latest-webkit-tests', H(std_test_steps), 193 T(['webkit_layout', 'webkit'], ['--cleanup', '--auto-reconnect'])), 194 B('webkit-latest-contentshell', H(compile_step), 195 T(['webkit_layout'], ['--auto-reconnect'])), 196 B('builder-unit-tests', H(compile_step), T(['unit'])), 197 198 # Generic builder config (for substring match). 199 B('builder', H(std_build_steps)), 200 ] 201 202 bot_map = dict((config.bot_id, config) for config in bot_configs) 203 204 # These bots have identical configuration to ones defined earlier. 205 copy_map = [ 206 ('lkgr-clobber', 'main-clobber'), 207 ('try-builder-dbg', 'main-builder-dbg'), 208 ('try-builder-rel', 'main-builder-rel'), 209 ('try-clang-builder', 'main-clang-builder'), 210 ('try-fyi-builder-dbg', 'fyi-builder-dbg'), 211 ('try-x86-builder-dbg', 'x86-builder-dbg'), 212 ('try-tests-rel', 'main-tests-rel'), 213 ('try-tests', 'main-tests'), 214 ('try-fyi-tests', 'fyi-tests'), 215 ('webkit-latest-tests', 'main-tests'), 216 ] 217 for to_id, from_id in copy_map: 218 assert to_id not in bot_map 219 # pylint: disable=W0212 220 bot_map[to_id] = copy.deepcopy(bot_map[from_id])._replace(bot_id=to_id) 221 222 # Trybots do not upload to flakiness dashboard. They should be otherwise 223 # identical in configuration to their trunk building counterparts. 224 test_obj = bot_map[to_id].test_obj 225 if to_id.startswith('try') and test_obj: 226 extra_args = test_obj.extra_args 227 if extra_args and flakiness_server in extra_args: 228 extra_args.remove(flakiness_server) 229 return bot_map 230 231 232 # Return an object from the map, looking first for an exact id match. 233 # If this fails, look for an id which is a substring of the specified id. 234 # Choose the longest of all substring matches. 235 # pylint: disable=W0622 236 def GetBestMatch(id_map, id): 237 config = id_map.get(id) 238 if not config: 239 substring_matches = [x for x in id_map.iterkeys() if x in id] 240 if substring_matches: 241 max_id = max(substring_matches, key=len) 242 print 'Using config from id="%s" (substring match).' % max_id 243 config = id_map[max_id] 244 return config 245 246 247 def GetRunBotOptParser(): 248 parser = bb_utils.GetParser() 249 parser.add_option('--bot-id', help='Specify bot id directly.') 250 parser.add_option('--testing', action='store_true', 251 help='For testing: print, but do not run commands') 252 253 return parser 254 255 256 def GetBotConfig(options, bot_step_map): 257 bot_id = options.bot_id or options.factory_properties.get('android_bot_id') 258 if not bot_id: 259 print (sys.stderr, 260 'A bot id must be specified through option or factory_props.') 261 return 262 263 bot_config = GetBestMatch(bot_step_map, bot_id) 264 if not bot_config: 265 print 'Error: config for id="%s" cannot be inferred.' % bot_id 266 return bot_config 267 268 269 def RunBotCommands(options, commands, env): 270 print 'Environment changes:' 271 print DictDiff(dict(os.environ), env) 272 273 for command in commands: 274 print bb_utils.CommandToString(command) 275 sys.stdout.flush() 276 if options.testing: 277 env['BUILDBOT_TESTING'] = '1' 278 return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env) 279 if return_code != 0: 280 return return_code 281 282 283 def main(argv): 284 proc = subprocess.Popen( 285 ['/bin/hostname', '-f'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 286 hostname_stdout, hostname_stderr = proc.communicate() 287 if proc.returncode == 0: 288 print 'Running on: ' + hostname_stdout 289 else: 290 print >> sys.stderr, 'WARNING: failed to run hostname' 291 print >> sys.stderr, hostname_stdout 292 print >> sys.stderr, hostname_stderr 293 sys.exit(1) 294 295 parser = GetRunBotOptParser() 296 options, args = parser.parse_args(argv[1:]) 297 if args: 298 parser.error('Unused args: %s' % args) 299 300 bot_config = GetBotConfig(options, GetBotStepMap()) 301 if not bot_config: 302 sys.exit(1) 303 304 print 'Using config:', bot_config 305 306 commands = GetCommands(options, bot_config) 307 for command in commands: 308 print 'Will run: ', bb_utils.CommandToString(command) 309 print 310 311 env = GetEnvironment(bot_config.host_obj, options.testing) 312 return RunBotCommands(options, commands, env) 313 314 315 if __name__ == '__main__': 316 sys.exit(main(sys.argv)) 317
