1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import collections 8 import copy 9 import json 10 import os 11 import pipes 12 import re 13 import subprocess 14 import sys 15 16 import bb_utils 17 18 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 19 from pylib import constants 20 21 22 _BotConfig = collections.namedtuple( 23 'BotConfig', ['bot_id', 'host_obj', 'test_obj']) 24 25 HostConfig = collections.namedtuple( 26 'HostConfig', 27 ['script', 'host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch']) 28 29 TestConfig = collections.namedtuple('Tests', ['script', 'tests', 'extra_args']) 30 31 32 def BotConfig(bot_id, host_object, test_object=None): 33 return _BotConfig(bot_id, host_object, test_object) 34 35 36 def DictDiff(d1, d2): 37 diff = [] 38 for key in sorted(set(d1.keys() + d2.keys())): 39 if key in d1 and d1[key] != d2.get(key): 40 diff.append('- %s=%s' % (key, pipes.quote(d1[key]))) 41 if key in d2 and d2[key] != d1.get(key): 42 diff.append('+ %s=%s' % (key, pipes.quote(d2[key]))) 43 return '\n'.join(diff) 44 45 46 def GetEnvironment(host_obj, testing): 47 init_env = dict(os.environ) 48 init_env['GYP_GENERATORS'] = 'ninja' 49 init_env['GOMA_DIR'] = bb_utils.GOMA_DIR 50 envsetup_cmd = '. build/android/envsetup.sh' 51 if host_obj.target_arch: 52 envsetup_cmd += ' --target-arch=%s' % host_obj.target_arch 53 if testing: 54 # Skip envsetup to avoid presubmit dependence on android deps. 55 print 'Testing mode - skipping "%s"' % envsetup_cmd 56 envsetup_cmd = ':' 57 else: 58 print 'Running %s' % envsetup_cmd 59 proc = subprocess.Popen(['bash', '-exc', 60 envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'], 61 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 62 cwd=bb_utils.CHROME_SRC, env=init_env) 63 json_env, envsetup_output = proc.communicate() 64 if proc.returncode != 0: 65 print >> sys.stderr, 'FATAL Failure in envsetup.' 66 print >> sys.stderr, envsetup_output 67 sys.exit(1) 68 env = json.loads(json_env) 69 env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + ' fastbuild=1' 70 extra_gyp = host_obj.extra_gyp_defines 71 if extra_gyp: 72 env['GYP_DEFINES'] += ' %s' % extra_gyp 73 if re.search('(asan|clang)=1', extra_gyp): 74 env.pop('CXX_target', None) 75 76 # Bots checkout chrome in /b/build/slave/<name>/build/src 77 build_internal_android = os.path.abspath(os.path.join( 78 bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', 79 'scripts', 'slave', 'android')) 80 if os.path.exists(build_internal_android): 81 env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']]) 82 return env 83 84 85 def GetCommands(options, bot_config): 86 """Get a formatted list of commands. 87 88 Args: 89 options: Options object. 90 bot_config: A BotConfig named tuple. 91 host_step_script: Host step script. 92 device_step_script: Device step script. 93 Returns: 94 list of Command objects. 95 """ 96 property_args = bb_utils.EncodeProperties(options) 97 commands = [[bot_config.host_obj.script, 98 '--steps=%s' % ','.join(bot_config.host_obj.host_steps)] + 99 property_args + (bot_config.host_obj.extra_args or [])] 100 101 test_obj = bot_config.test_obj 102 if test_obj: 103 run_test_cmd = [test_obj.script, '--reboot'] + property_args 104 for test in test_obj.tests: 105 run_test_cmd.extend(['-f', test]) 106 if test_obj.extra_args: 107 run_test_cmd.extend(test_obj.extra_args) 108 commands.append(run_test_cmd) 109 return commands 110 111 112 def GetBotStepMap(): 113 compile_step = ['compile'] 114 std_host_tests = ['check_webview_licenses', 'findbugs'] 115 std_build_steps = ['compile', 'zip_build'] 116 std_test_steps = ['extract_build'] 117 std_tests = ['ui', 'unit'] 118 flakiness_server = ( 119 '--flakiness-server=%s' % constants.UPSTREAM_FLAKINESS_SERVER) 120 experimental = ['--experimental'] 121 122 B = BotConfig 123 H = (lambda steps, extra_args=None, extra_gyp=None, target_arch=None : 124 HostConfig('build/android/buildbot/bb_host_steps.py', steps, extra_args, 125 extra_gyp, target_arch)) 126 T = (lambda tests, extra_args=None : 127 TestConfig('build/android/buildbot/bb_device_steps.py', tests, 128 extra_args)) 129 130 bot_configs = [ 131 # Main builders 132 B('main-builder-dbg', H(std_build_steps + std_host_tests)), 133 B('main-builder-rel', H(std_build_steps)), 134 B('main-clang-builder', 135 H(compile_step, extra_gyp='clang=1 component=shared_library')), 136 B('main-clobber', H(compile_step)), 137 B('main-tests', H(std_test_steps), T(std_tests, [flakiness_server])), 138 139 # Other waterfalls 140 B('asan-builder-tests', H(compile_step, extra_gyp='asan=1'), 141 T(std_tests, ['--asan'])), 142 B('chromedriver-fyi-tests-dbg', H(std_test_steps), 143 T(['chromedriver'], ['--install=ChromiumTestShell'])), 144 B('fyi-x86-builder-dbg', 145 H(compile_step + std_host_tests, experimental, target_arch='x86')), 146 B('fyi-builder-dbg', 147 H(std_build_steps + std_host_tests, experimental)), 148 B('x86-builder-dbg', 149 H(compile_step + std_host_tests, target_arch='x86')), 150 B('fyi-builder-rel', H(std_build_steps, experimental)), 151 B('fyi-tests', H(std_test_steps), 152 T(std_tests, ['--experimental', flakiness_server])), 153 B('fyi-component-builder-tests-dbg', 154 H(compile_step, extra_gyp='component=shared_library'), 155 T(std_tests, ['--experimental', flakiness_server])), 156 B('perf-bisect-builder-tests-dbg', H(['bisect_perf_regression'])), 157 B('perf-tests-rel', H(std_test_steps), 158 T([], ['--install=ChromiumTestShell'])), 159 B('webkit-latest-webkit-tests', H(std_test_steps), 160 T(['webkit_layout', 'webkit'], ['--auto-reconnect'])), 161 B('webkit-latest-contentshell', H(compile_step), 162 T(['webkit_layout'], ['--auto-reconnect'])), 163 B('builder-unit-tests', H(compile_step), T(['unit'])), 164 B('webrtc-tests', H(std_test_steps), T(['webrtc'], [flakiness_server])), 165 166 # Generic builder config (for substring match). 167 B('builder', H(std_build_steps)), 168 ] 169 170 bot_map = dict((config.bot_id, config) for config in bot_configs) 171 172 # These bots have identical configuration to ones defined earlier. 173 copy_map = [ 174 ('lkgr-clobber', 'main-clobber'), 175 ('try-builder-dbg', 'main-builder-dbg'), 176 ('try-builder-rel', 'main-builder-rel'), 177 ('try-clang-builder', 'main-clang-builder'), 178 ('try-fyi-builder-dbg', 'fyi-builder-dbg'), 179 ('try-x86-builder-dbg', 'x86-builder-dbg'), 180 ('try-tests', 'main-tests'), 181 ('try-fyi-tests', 'fyi-tests'), 182 ('webkit-latest-tests', 'main-tests'), 183 ('webrtc-builder', 'main-builder-rel'), 184 ] 185 for to_id, from_id in copy_map: 186 assert to_id not in bot_map 187 # pylint: disable=W0212 188 bot_map[to_id] = copy.deepcopy(bot_map[from_id])._replace(bot_id=to_id) 189 190 # Trybots do not upload to flakiness dashboard. They should be otherwise 191 # identical in configuration to their trunk building counterparts. 192 test_obj = bot_map[to_id].test_obj 193 if to_id.startswith('try') and test_obj: 194 extra_args = test_obj.extra_args 195 if extra_args and flakiness_server in extra_args: 196 extra_args.remove(flakiness_server) 197 return bot_map 198 199 200 # Return an object from the map, looking first for an exact id match. 201 # If this fails, look for an id which is a substring of the specified id. 202 # Choose the longest of all substring matches. 203 # pylint: disable=W0622 204 def GetBestMatch(id_map, id): 205 config = id_map.get(id) 206 if not config: 207 substring_matches = filter(lambda x: x in id, id_map.iterkeys()) 208 if substring_matches: 209 max_id = max(substring_matches, key=len) 210 print 'Using config from id="%s" (substring match).' % max_id 211 config = id_map[max_id] 212 return config 213 214 215 def GetRunBotOptParser(): 216 parser = bb_utils.GetParser() 217 parser.add_option('--bot-id', help='Specify bot id directly.') 218 parser.add_option('--testing', action='store_true', 219 help='For testing: print, but do not run commands') 220 221 return parser 222 223 224 def GetBotConfig(options, bot_step_map): 225 bot_id = options.bot_id or options.factory_properties.get('android_bot_id') 226 if not bot_id: 227 print (sys.stderr, 228 'A bot id must be specified through option or factory_props.') 229 return 230 231 bot_config = GetBestMatch(bot_step_map, bot_id) 232 if not bot_config: 233 print 'Error: config for id="%s" cannot be inferred.' % bot_id 234 return bot_config 235 236 237 def RunBotCommands(options, commands, env): 238 print 'Environment changes:' 239 print DictDiff(dict(os.environ), env) 240 241 for command in commands: 242 print bb_utils.CommandToString(command) 243 sys.stdout.flush() 244 if options.testing: 245 env['BUILDBOT_TESTING'] = '1' 246 return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env) 247 if return_code != 0: 248 return return_code 249 250 251 def main(argv): 252 parser = GetRunBotOptParser() 253 options, args = parser.parse_args(argv[1:]) 254 if args: 255 parser.error('Unused args: %s' % args) 256 257 bot_config = GetBotConfig(options, GetBotStepMap()) 258 if not bot_config: 259 sys.exit(1) 260 261 print 'Using config:', bot_config 262 263 commands = GetCommands(options, bot_config) 264 for command in commands: 265 print 'Will run: ', bb_utils.CommandToString(command) 266 print 267 268 env = GetEnvironment(bot_config.host_obj, options.testing) 269 return RunBotCommands(options, commands, env) 270 271 272 if __name__ == '__main__': 273 sys.exit(main(sys.argv)) 274
