1 # 2 # Copyright 2015 Google Inc. 3 # 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 # 7 8 #!/usr/bin/env python 9 10 usage = ''' 11 Write buildbot spec to outfile based on the bot name: 12 $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug 13 Or run self-tests: 14 $ python buildbot_spec.py test 15 ''' 16 17 import inspect 18 import json 19 import os 20 import sys 21 22 import builder_name_schema 23 import dm_flags 24 import nanobench_flags 25 26 27 CONFIG_DEBUG = 'Debug' 28 CONFIG_RELEASE = 'Release' 29 30 31 def lineno(): 32 caller = inspect.stack()[1] # Up one level to our caller. 33 return inspect.getframeinfo(caller[0]).lineno 34 35 # Since we don't actually start coverage until we're in the self-test, 36 # some function def lines aren't reported as covered. Add them to this 37 # list so that we can ignore them. 38 cov_skip = [] 39 40 cov_start = lineno()+1 # We care about coverage starting just past this def. 41 def gyp_defines(builder_dict): 42 gyp_defs = {} 43 44 # skia_arch_type. 45 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: 46 arch = builder_dict['target_arch'] 47 elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: 48 arch = None 49 else: 50 arch = builder_dict['arch'] 51 52 arch_types = { 53 'x86': 'x86', 54 'x86_64': 'x86_64', 55 'Arm7': 'arm', 56 'Arm64': 'arm64', 57 'Mips': 'mips32', 58 'Mips64': 'mips64', 59 'MipsDSP2': 'mips32', 60 } 61 if arch in arch_types: 62 gyp_defs['skia_arch_type'] = arch_types[arch] 63 64 # housekeeper: build shared lib. 65 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: 66 gyp_defs['skia_shared_lib'] = '1' 67 68 # skia_gpu. 69 if builder_dict.get('cpu_or_gpu') == 'CPU': 70 gyp_defs['skia_gpu'] = '0' 71 72 # skia_warnings_as_errors. 73 werr = False 74 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: 75 if 'Win' in builder_dict.get('os', ''): 76 if not ('GDI' in builder_dict.get('extra_config', '') or 77 'Exceptions' in builder_dict.get('extra_config', '')): 78 werr = True 79 elif ('Mac' in builder_dict.get('os', '') and 80 'Android' in builder_dict.get('extra_config', '')): 81 werr = False 82 else: 83 werr = True 84 gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' 85 86 # Win debugger. 87 if 'Win' in builder_dict.get('os', ''): 88 gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp' 89 90 # Qt SDK (Win). 91 if 'Win' in builder_dict.get('os', ''): 92 if builder_dict.get('os') == 'Win8': 93 gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/' 94 else: 95 gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/' 96 97 # ANGLE. 98 if builder_dict.get('extra_config') == 'ANGLE': 99 gyp_defs['skia_angle'] = '1' 100 if builder_dict.get('os', '') in ('Ubuntu', 'Linux'): 101 gyp_defs['use_x11'] = '1' 102 gyp_defs['chromeos'] = '0' 103 104 # GDI. 105 if builder_dict.get('extra_config') == 'GDI': 106 gyp_defs['skia_gdi'] = '1' 107 108 # Build with Exceptions on Windows. 109 if ('Win' in builder_dict.get('os', '') and 110 builder_dict.get('extra_config') == 'Exceptions'): 111 gyp_defs['skia_win_exceptions'] = '1' 112 113 # iOS. 114 if (builder_dict.get('os') == 'iOS' or 115 builder_dict.get('extra_config') == 'iOS'): 116 gyp_defs['skia_os'] = 'ios' 117 118 # Shared library build. 119 if builder_dict.get('extra_config') == 'Shared': 120 gyp_defs['skia_shared_lib'] = '1' 121 122 # Build fastest Skia possible. 123 if builder_dict.get('extra_config') == 'Fast': 124 gyp_defs['skia_fast'] = '1' 125 126 # PDF viewer in GM. 127 if (builder_dict.get('os') == 'Mac10.8' and 128 builder_dict.get('arch') == 'x86_64' and 129 builder_dict.get('configuration') == 'Release'): 130 gyp_defs['skia_run_pdfviewer_in_gm'] = '1' 131 132 # Clang. 133 if builder_dict.get('compiler') == 'Clang': 134 gyp_defs['skia_clang_build'] = '1' 135 136 # Valgrind. 137 if 'Valgrind' in builder_dict.get('extra_config', ''): 138 gyp_defs['skia_release_optimization_level'] = '1' 139 140 # Link-time code generation just wastes time on compile-only bots. 141 if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and 142 builder_dict.get('compiler') == 'MSVC'): 143 gyp_defs['skia_win_ltcg'] = '0' 144 145 # Mesa. 146 if (builder_dict.get('extra_config') == 'Mesa' or 147 builder_dict.get('cpu_or_gpu_value') == 'Mesa'): 148 gyp_defs['skia_mesa'] = '1' 149 150 # VisualBench 151 if builder_dict.get('extra_config') == 'VisualBench': 152 gyp_defs['skia_use_sdl'] = '1' 153 154 # skia_use_android_framework_defines. 155 if builder_dict.get('extra_config') == 'Android_FrameworkDefs': 156 gyp_defs['skia_use_android_framework_defines'] = '1' 157 158 # Skia dump stats for perf tests and gpu 159 if (builder_dict.get('cpu_or_gpu') == 'GPU' and 160 builder_dict.get('role') == 'Perf'): 161 gyp_defs['skia_dump_stats'] = '1' 162 163 return gyp_defs 164 165 166 cov_skip.extend([lineno(), lineno() + 1]) 167 def get_extra_env_vars(builder_dict): 168 env = {} 169 if builder_dict.get('configuration') == 'Coverage': 170 # We have to use Clang 3.6 because earlier versions do not support the 171 # compile flags we use and 3.7 and 3.8 hit asserts during compilation. 172 env['CC'] = '/usr/bin/clang-3.6' 173 env['CXX'] = '/usr/bin/clang++-3.6' 174 elif builder_dict.get('compiler') == 'Clang': 175 env['CC'] = '/usr/bin/clang' 176 env['CXX'] = '/usr/bin/clang++' 177 178 # SKNX_NO_SIMD, SK_USE_DISCARDABLE_SCALEDIMAGECACHE, etc. 179 extra_config = builder_dict.get('extra_config', '') 180 if extra_config.startswith('SK') and extra_config.isupper(): 181 env['CPPFLAGS'] = '-D' + extra_config 182 183 return env 184 185 186 cov_skip.extend([lineno(), lineno() + 1]) 187 def build_targets_from_builder_dict(builder_dict, do_test_steps, do_perf_steps): 188 """Return a list of targets to build, depending on the builder type.""" 189 if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS': 190 return ['iOSShell'] 191 if builder_dict.get('extra_config') == 'Appurify': 192 return ['VisualBenchTest_APK'] 193 t = [] 194 if do_test_steps: 195 t.append('dm') 196 if do_perf_steps and builder_dict.get('extra_config') == 'VisualBench': 197 t.append('visualbench') 198 elif do_perf_steps: 199 t.append('nanobench') 200 if t: 201 return t 202 else: 203 return ['most'] 204 205 206 cov_skip.extend([lineno(), lineno() + 1]) 207 def device_cfg(builder_dict): 208 # Android. 209 if 'Android' in builder_dict.get('extra_config', ''): 210 if 'NoNeon' in builder_dict['extra_config']: 211 return 'arm_v7' 212 return { 213 'Arm64': 'arm64', 214 'x86': 'x86', 215 'x86_64': 'x86_64', 216 'Mips': 'mips', 217 'Mips64': 'mips64', 218 'MipsDSP2': 'mips_dsp2', 219 }.get(builder_dict['target_arch'], 'arm_v7_neon') 220 elif builder_dict.get('os') == 'Android': 221 return { 222 'AndroidOne': 'arm_v7_neon', 223 'GalaxyS3': 'arm_v7_neon', 224 'GalaxyS4': 'arm_v7_neon', 225 'NVIDIA_Shield': 'arm64', 226 'Nexus10': 'arm_v7_neon', 227 'Nexus5': 'arm_v7_neon', 228 'Nexus6': 'arm_v7_neon', 229 'Nexus7': 'arm_v7_neon', 230 'Nexus9': 'arm64', 231 'NexusPlayer': 'x86', 232 }[builder_dict['model']] 233 234 # ChromeOS. 235 if 'CrOS' in builder_dict.get('extra_config', ''): 236 if 'Link' in builder_dict['extra_config']: 237 return 'link' 238 if 'Daisy' in builder_dict['extra_config']: 239 return 'daisy' 240 elif builder_dict.get('os') == 'ChromeOS': 241 return { 242 'Link': 'link', 243 'Daisy': 'daisy', 244 }[builder_dict['model']] 245 246 return None 247 248 249 cov_skip.extend([lineno(), lineno() + 1]) 250 def get_builder_spec(builder_name): 251 builder_dict = builder_name_schema.DictForBuilderName(builder_name) 252 env = get_extra_env_vars(builder_dict) 253 gyp_defs = gyp_defines(builder_dict) 254 gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()] 255 gyp_defs_list.sort() 256 env['GYP_DEFINES'] = ' '.join(gyp_defs_list) 257 rv = { 258 'builder_cfg': builder_dict, 259 'dm_flags': dm_flags.get_args(builder_name), 260 'env': env, 261 'nanobench_flags': nanobench_flags.get_args(builder_name), 262 } 263 device = device_cfg(builder_dict) 264 if device: 265 rv['device_cfg'] = device 266 267 role = builder_dict['role'] 268 if role == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: 269 configuration = CONFIG_RELEASE 270 else: 271 configuration = builder_dict.get( 272 'configuration', CONFIG_DEBUG) 273 arch = (builder_dict.get('arch') or builder_dict.get('target_arch')) 274 if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'): 275 configuration += '_x64' 276 rv['configuration'] = configuration 277 rv['do_test_steps'] = role == builder_name_schema.BUILDER_ROLE_TEST 278 rv['do_perf_steps'] = (role == builder_name_schema.BUILDER_ROLE_PERF or 279 (role == builder_name_schema.BUILDER_ROLE_TEST and 280 configuration == CONFIG_DEBUG)) 281 if 'Valgrind' in builder_name: 282 rv['do_perf_steps'] = True 283 if 'GalaxyS4' in builder_name: 284 rv['do_perf_steps'] = False 285 286 rv['build_targets'] = build_targets_from_builder_dict( 287 builder_dict, rv['do_test_steps'], rv['do_perf_steps']) 288 289 # Do we upload perf results? 290 upload_perf_results = False 291 if role == builder_name_schema.BUILDER_ROLE_PERF: 292 upload_perf_results = True 293 rv['upload_perf_results'] = upload_perf_results 294 295 # Do we upload correctness results? 296 skip_upload_bots = [ 297 'ASAN', 298 'Coverage', 299 'MSAN', 300 'TSAN', 301 'UBSAN', 302 'Valgrind', 303 ] 304 upload_dm_results = True 305 for s in skip_upload_bots: 306 if s in builder_name: 307 upload_dm_results = False 308 break 309 rv['upload_dm_results'] = upload_dm_results 310 311 return rv 312 313 314 cov_end = lineno() # Don't care about code coverage past here. 315 316 317 def self_test(): 318 import coverage # This way the bots don't need coverage.py to be installed. 319 args = {} 320 cases = [ 321 'Build-Mac10.8-Clang-Arm7-Debug-Android', 322 'Build-Win-MSVC-x86-Debug', 323 'Build-Win-MSVC-x86-Debug-GDI', 324 'Build-Win-MSVC-x86-Debug-Exceptions', 325 'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs', 326 'Build-Ubuntu-GCC-Arm7-Debug-Android_NoNeon', 327 'Build-Ubuntu-GCC-Arm7-Debug-CrOS_Daisy', 328 'Build-Ubuntu-GCC-x86_64-Debug-CrOS_Link', 329 'Build-Ubuntu-GCC-x86_64-Release-Mesa', 330 'Build-Ubuntu-GCC-x86_64-Release-ANGLE', 331 'Housekeeper-PerCommit', 332 'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot', 333 'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release-VisualBench', 334 'Test-Android-GCC-GalaxyS4-GPU-SGX544-Arm7-Debug', 335 'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Release-Appurify', 336 'Test-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Debug', 337 'Test-ChromeOS-GCC-Link-CPU-AVX-x86_64-Debug', 338 'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug', 339 'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release', 340 'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage', 341 ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-' 342 'SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), 343 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD', 344 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Fast', 345 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared', 346 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind', 347 'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE', 348 'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug', 349 ] 350 351 cov = coverage.coverage() 352 cov.start() 353 for case in cases: 354 args[case] = get_builder_spec(case) 355 cov.stop() 356 357 this_file = os.path.basename(__file__) 358 _, _, not_run, _ = cov.analysis(this_file) 359 filtered = [line for line in not_run if 360 line > cov_start and line < cov_end and line not in cov_skip] 361 if filtered: 362 print 'Lines not covered by test cases: ', filtered 363 sys.exit(1) 364 365 golden = this_file.replace('.py', '.json') 366 with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f: 367 json.dump(args, f, indent=2, sort_keys=True) 368 369 370 if __name__ == '__main__': 371 if len(sys.argv) == 2 and sys.argv[1] == 'test': 372 self_test() 373 sys.exit(0) 374 375 if len(sys.argv) != 3: 376 print usage 377 sys.exit(1) 378 379 with open(sys.argv[1], 'w') as out: 380 json.dump(get_builder_spec(sys.argv[2]), out) 381
