1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 # Recipe module for Skia Swarming perf. 7 8 9 import calendar 10 import os 11 12 13 DEPS = [ 14 'core', 15 'env', 16 'flavor', 17 'recipe_engine/file', 18 'recipe_engine/json', 19 'recipe_engine/path', 20 'recipe_engine/platform', 21 'recipe_engine/properties', 22 'recipe_engine/raw_io', 23 'recipe_engine/step', 24 'recipe_engine/time', 25 'run', 26 'vars', 27 ] 28 29 30 def nanobench_flags(api, bot): 31 args = ['--pre_log'] 32 33 if 'GPU' in bot: 34 args.append('--images') 35 args.extend(['--gpuStatsDump', 'true']) 36 37 args.extend(['--scales', '1.0', '1.1']) 38 39 if 'iOS' in bot: 40 args.extend(['--skps', 'ignore_skps']) 41 42 configs = [] 43 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 44 args.append('--nogpu') 45 configs.extend(['8888', 'nonrendering']) 46 47 if '-arm-' not in bot: 48 # For Android CPU tests, these take too long and cause the task to time 49 # out. 50 configs += [ 'f16', 'srgb' ] 51 if '-GCE-' in bot: 52 configs += [ '565' ] 53 54 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 55 args.append('--nocpu') 56 57 gl_prefix = 'gl' 58 sample_count = '8' 59 if 'Android' in bot or 'iOS' in bot: 60 sample_count = '4' 61 # The NVIDIA_Shield has a regular OpenGL implementation. We bench that 62 # instead of ES. 63 if 'NVIDIA_Shield' not in bot: 64 gl_prefix = 'gles' 65 # The NP produces a long error stream when we run with MSAA. 66 # iOS crashes (skia:6399) 67 # Nexus7 (Tegra3) does not support MSAA. 68 if ('NexusPlayer' in bot or 69 'iOS' in bot or 70 'Nexus7' in bot): 71 sample_count = '' 72 elif 'Intel' in bot: 73 sample_count = '' 74 elif 'ChromeOS' in bot: 75 gl_prefix = 'gles' 76 77 configs.extend([gl_prefix, gl_prefix + 'srgb']) 78 if sample_count is not '': 79 configs.append(gl_prefix + 'msaa' + sample_count) 80 if ('TegraX1' in bot or 81 'Quadro' in bot or 82 'GTX' in bot or 83 ('GT610' in bot and 'Ubuntu17' not in bot)): 84 configs.extend([gl_prefix + 'nvpr' + sample_count]) 85 86 # We want to test both the OpenGL config and the GLES config on Linux Intel: 87 # GL is used by Chrome, GLES is used by ChromeOS. 88 if 'Intel' in bot and api.vars.is_linux: 89 configs.extend(['gles', 'glessrgb']) 90 91 # The following devices do not support glessrgb. 92 if 'glessrgb' in configs: 93 if ('IntelHD405' in bot or 94 'IntelIris640' in bot or 95 'IntelBayTrail' in bot or 96 'IntelHD2000' in bot or 97 'AndroidOne' in bot or 98 'Nexus7' in bot or 99 'NexusPlayer' in bot): 100 configs.remove('glessrgb') 101 102 if 'CommandBuffer' in bot: 103 configs = ['commandbuffer'] 104 if 'Vulkan' in bot: 105 configs = ['vk'] 106 107 if 'ANGLE' in bot: 108 # Test only ANGLE configs. 109 configs = ['angle_d3d11_es2'] 110 if sample_count is not '': 111 configs.append('angle_d3d11_es2_msaa' + sample_count) 112 113 if 'ChromeOS' in bot: 114 # Just run GLES for now - maybe add gles_msaa4 in the future 115 configs = ['gles'] 116 117 args.append('--config') 118 args.extend(configs) 119 120 # By default, we test with GPU threading enabled. Leave PixelC devices 121 # running without threads, just to get some coverage of that code path. 122 if 'PixelC' in bot: 123 args.extend(['--gpuThreads', '0']) 124 125 if 'Valgrind' in bot: 126 # Don't care about Valgrind performance. 127 args.extend(['--loops', '1']) 128 args.extend(['--samples', '1']) 129 # Ensure that the bot framework does not think we have timed out. 130 args.extend(['--keepAlive', 'true']) 131 132 # Some people don't like verbose output. 133 verbose = False 134 135 match = [] 136 if 'Android' in bot: 137 # Segfaults when run as GPU bench. Very large texture? 138 match.append('~blurroundrect') 139 match.append('~patch_grid') # skia:2847 140 match.append('~desk_carsvg') 141 if 'Nexus5' in bot: 142 match.append('~keymobi_shop_mobileweb_ebay_com.skp') # skia:5178 143 if 'iOS' in bot: 144 match.append('~blurroundrect') 145 match.append('~patch_grid') # skia:2847 146 match.append('~desk_carsvg') 147 match.append('~keymobi') 148 match.append('~path_hairline') 149 match.append('~GLInstancedArraysBench') # skia:4714 150 if 'IntelIris540' in bot and 'ANGLE' in bot: 151 match.append('~tile_image_filter_tiled_64') # skia:6082 152 if 'Vulkan' in bot and 'IntelIris540' in bot and 'Win' in bot: 153 # skia:6398 154 match.append('~GM_varied_text_clipped_lcd') 155 match.append('~GM_varied_text_ignorable_clip_lcd') 156 match.append('~blendmode_mask_DstATop') 157 match.append('~blendmode_mask_SrcIn') 158 match.append('~blendmode_mask_SrcOut') 159 match.append('~blendmode_mask_Src') 160 match.append('~fontscaler_lcd') 161 match.append('~rotated_rects_aa_alternating_transparent_and_opaque_src') 162 match.append('~rotated_rects_aa_changing_transparent_src') 163 match.append('~rotated_rects_aa_same_transparent_src') 164 match.append('~shadermask_LCD_FF') 165 match.append('~srcmode_rects_1') 166 match.append('~text_16_LCD_88') 167 match.append('~text_16_LCD_BK') 168 match.append('~text_16_LCD_FF') 169 match.append('~text_16_LCD_WT') 170 # skia:6863 171 match.append('~desk_skbug6850overlay2') 172 match.append('~desk_googlespreadsheet') 173 match.append('~desk_carsvg') 174 if ('Vulkan' in bot and ('RadeonR9M470X' in bot or 'RadeonHD7770' in bot) and 175 'Win' in bot): 176 # skia:7677 177 match.append('~path_text_clipped_uncached') 178 if ('Intel' in bot and api.vars.is_linux and not 'Vulkan' in bot): 179 # TODO(dogben): Track down what's causing bots to die. 180 verbose = True 181 if 'IntelHD405' in bot and api.vars.is_linux and 'Vulkan' in bot: 182 # skia:7322 183 match.append('~desk_tiger8svg.skp_1') 184 match.append('~keymobi_techcrunch_com.skp_1.1') 185 match.append('~tabl_gamedeksiam.skp_1.1') 186 match.append('~tabl_pravda.skp_1') 187 match.append('~top25desk_ebay_com.skp_1.1') 188 if 'Vulkan' in bot and 'NexusPlayer' in bot: 189 match.append('~blendmode_') # skia:6691 190 if ('ASAN' in bot or 'UBSAN' in bot) and 'CPU' in bot: 191 # floor2int_undef benches undefined behavior, so ASAN correctly complains. 192 match.append('~^floor2int_undef$') 193 194 # We do not need or want to benchmark the decodes of incomplete images. 195 # In fact, in nanobench we assert that the full image decode succeeds. 196 match.append('~inc0.gif') 197 match.append('~inc1.gif') 198 match.append('~incInterlaced.gif') 199 match.append('~inc0.jpg') 200 match.append('~incGray.jpg') 201 match.append('~inc0.wbmp') 202 match.append('~inc1.wbmp') 203 match.append('~inc0.webp') 204 match.append('~inc1.webp') 205 match.append('~inc0.ico') 206 match.append('~inc1.ico') 207 match.append('~inc0.png') 208 match.append('~inc1.png') 209 match.append('~inc2.png') 210 match.append('~inc12.png') 211 match.append('~inc13.png') 212 match.append('~inc14.png') 213 match.append('~inc0.webp') 214 match.append('~inc1.webp') 215 216 if match: 217 args.append('--match') 218 args.extend(match) 219 220 if verbose: 221 args.append('--verbose') 222 223 return args 224 225 226 def perf_steps(api): 227 """Run Skia benchmarks.""" 228 if api.vars.upload_perf_results: 229 api.flavor.create_clean_device_dir( 230 api.flavor.device_dirs.perf_data_dir) 231 232 # Run nanobench. 233 properties = [ 234 '--properties', 235 'gitHash', api.vars.got_revision, 236 ] 237 if api.vars.is_trybot: 238 properties.extend([ 239 'issue', api.vars.issue, 240 'patchset', api.vars.patchset, 241 'patch_storage', api.vars.patch_storage, 242 ]) 243 properties.extend(['swarming_bot_id', api.vars.swarming_bot_id]) 244 properties.extend(['swarming_task_id', api.vars.swarming_task_id]) 245 246 target = 'nanobench' 247 args = [ 248 target, 249 '-i', api.flavor.device_dirs.resource_dir, 250 '--skps', api.flavor.device_dirs.skp_dir, 251 '--images', api.flavor.device_path_join( 252 api.flavor.device_dirs.images_dir, 'nanobench'), 253 ] 254 255 # Do not run svgs on Valgrind. 256 if 'Valgrind' not in api.vars.builder_name: 257 args.extend(['--svgs', api.flavor.device_dirs.svg_dir]) 258 259 args.extend(nanobench_flags(api, api.vars.builder_name)) 260 261 if 'Chromecast' in api.vars.builder_cfg.get('os', ''): 262 # Due to limited disk space, run a watered down perf run on Chromecast. 263 args = [target] 264 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 265 args.extend(['--nogpu', '--config', '8888']) 266 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 267 args.extend(['--nocpu', '--config', 'gles']) 268 args.extend([ 269 '-i', api.flavor.device_dirs.resource_dir, 270 '--images', api.flavor.device_path_join( 271 api.flavor.device_dirs.resource_dir, 'images', 'color_wheel.jpg'), 272 '--skps', api.flavor.device_dirs.skp_dir, 273 '--pre_log', 274 '--match', # skia:6687 275 '~matrixconvolution', 276 '~blur_image_filter', 277 '~blur_0.01', 278 '~GM_animated-image-blurs', 279 '~blendmode_mask_', 280 '~desk_carsvg.skp', 281 '~^path_text_clipped', # Bot times out; skia:7190 282 '~shapes_rrect_inner_rrect_50_500x500', # skia:7551 283 ]) 284 285 if api.vars.upload_perf_results: 286 now = api.time.utcnow() 287 ts = int(calendar.timegm(now.utctimetuple())) 288 json_path = api.flavor.device_path_join( 289 api.flavor.device_dirs.perf_data_dir, 290 'nanobench_%s_%d.json' % (api.vars.got_revision, ts)) 291 args.extend(['--outResultsFile', json_path]) 292 args.extend(properties) 293 294 keys_blacklist = ['configuration', 'role', 'test_filter'] 295 args.append('--key') 296 for k in sorted(api.vars.builder_cfg.keys()): 297 if not k in keys_blacklist: 298 args.extend([k, api.vars.builder_cfg[k]]) 299 300 # See skia:2789. 301 if 'AbandonGpuContext' in api.vars.extra_tokens: 302 args.extend(['--abandonGpuContext']) 303 304 api.run(api.flavor.step, target, cmd=args, 305 abort_on_failure=False) 306 307 # Copy results to swarming out dir. 308 if api.vars.upload_perf_results: 309 api.file.ensure_directory('makedirs perf_dir', 310 api.path.dirname(api.vars.perf_data_dir)) 311 api.flavor.copy_directory_contents_to_host( 312 api.flavor.device_dirs.perf_data_dir, 313 api.vars.perf_data_dir) 314 315 316 def RunSteps(api): 317 api.core.setup() 318 env = {} 319 if 'iOS' in api.vars.builder_name: 320 env['IOS_BUNDLE_ID'] = 'com.google.nanobench' 321 env['IOS_MOUNT_POINT'] = api.vars.slave_dir.join('mnt_iosdevice') 322 with api.env(env): 323 try: 324 if 'Chromecast' in api.vars.builder_name: 325 api.flavor.install(resources=True, skps=True) 326 else: 327 api.flavor.install_everything() 328 perf_steps(api) 329 finally: 330 api.flavor.cleanup_steps() 331 api.run.check_failure() 332 333 334 TEST_BUILDERS = [ 335 ('Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-All-' 336 'Android_Vulkan'), 337 'Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-All-Android', 338 'Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android', 339 'Perf-Android-Clang-Nexus7-CPU-Tegra3-arm-Release-All-Android', 340 'Perf-Android-Clang-Nexus7-GPU-Tegra3-arm-Release-All-Android', 341 'Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-All-Android', 342 'Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-All-Android_Vulkan', 343 'Perf-Android-Clang-PixelC-GPU-TegraX1-arm64-Release-All-Android_Skpbench', 344 'Perf-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Release-All', 345 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All', 346 'Perf-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All', 347 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN', 348 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All', 349 'Perf-Mac-Clang-MacMini7.1-CPU-AVX-x86_64-Release-All', 350 'Perf-Mac-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All', 351 ('Perf-Mac-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All-' 352 'CommandBuffer'), 353 'Perf-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug-All-Vulkan', 354 'Perf-Ubuntu16-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Debug-All-Vulkan', 355 'Perf-Ubuntu16-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Release-All', 356 ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 357 'Valgrind_AbandonGpuContext_SK_CPU_LIMIT_SSE41'), 358 ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 359 'Valgrind_SK_CPU_LIMIT_SSE41'), 360 'Perf-Win10-Clang-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-All-ANGLE', 361 'Perf-Win10-Clang-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-All-Vulkan', 362 'Perf-Win10-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-All-ANGLE', 363 'Perf-Win10-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release-All-Vulkan', 364 'Perf-Win10-Clang-ShuttleC-GPU-GTX960-x86_64-Release-All-ANGLE', 365 'Perf-Win2016-MSVC-GCE-CPU-AVX2-x86_64-Debug-All', 366 'Perf-Win2016-MSVC-GCE-CPU-AVX2-x86_64-Release-All', 367 'Perf-iOS-Clang-iPadPro-GPU-GT7800-arm64-Release-All', 368 ] 369 370 371 def GenTests(api): 372 for builder in TEST_BUILDERS: 373 test = ( 374 api.test(builder) + 375 api.properties(buildername=builder, 376 revision='abc123', 377 path_config='kitchen', 378 swarm_out_dir='[SWARM_OUT_DIR]') + 379 api.path.exists( 380 api.path['start_dir'].join('skia'), 381 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 382 'skimage', 'VERSION'), 383 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 384 'skp', 'VERSION'), 385 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 386 ) + 387 api.step_data('get swarming bot id', 388 stdout=api.raw_io.output('skia-bot-123')) + 389 api.step_data('get swarming task id', 390 stdout=api.raw_io.output('123456')) 391 ) 392 if 'Win' in builder: 393 test += api.platform('win', 64) 394 395 if 'Chromecast' in builder: 396 test += api.step_data( 397 'read chromecast ip', 398 stdout=api.raw_io.output('192.168.1.2:5555')) 399 400 if 'ChromeOS' in builder: 401 test += api.step_data( 402 'read chromeos ip', 403 stdout=api.raw_io.output('{"user_ip":"foo (at] 127.0.0.1"}')) 404 405 yield test 406 407 builder = 'Perf-Win10-Clang-NUCD34010WYKH-GPU-IntelHD4400-x86_64-Release-All' 408 yield ( 409 api.test('trybot') + 410 api.properties(buildername=builder, 411 revision='abc123', 412 path_config='kitchen', 413 swarm_out_dir='[SWARM_OUT_DIR]') + 414 api.properties(patch_storage='gerrit') + 415 api.properties.tryserver( 416 buildername=builder, 417 gerrit_project='skia', 418 gerrit_url='https://skia-review.googlesource.com/', 419 )+ 420 api.path.exists( 421 api.path['start_dir'].join('skia'), 422 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 423 'skimage', 'VERSION'), 424 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 425 'skp', 'VERSION'), 426 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 427 'svg', 'VERSION'), 428 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 429 ) 430 ) 431 432 builder = ('Perf-Android-Clang-NexusPlayer-CPU-Moorefield-x86-Debug-All-' + 433 'Android') 434 yield ( 435 api.test('failed_push') + 436 api.properties(buildername=builder, 437 revision='abc123', 438 path_config='kitchen', 439 swarm_out_dir='[SWARM_OUT_DIR]') + 440 api.path.exists( 441 api.path['start_dir'].join('skia'), 442 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 443 'skimage', 'VERSION'), 444 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 445 'skp', 'VERSION'), 446 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 447 'svg', 'VERSION'), 448 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 449 ) + 450 api.step_data('push [START_DIR]/skia/resources/* '+ 451 '/sdcard/revenge_of_the_skiabot/resources', retcode=1) 452 ) 453 454 yield ( 455 api.test('cpu_scale_failed_once') + 456 api.properties(buildername=builder, 457 revision='abc123', 458 path_config='kitchen', 459 swarm_out_dir='[SWARM_OUT_DIR]') + 460 api.path.exists( 461 api.path['start_dir'].join('skia'), 462 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 463 'skimage', 'VERSION'), 464 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 465 'skp', 'VERSION'), 466 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 467 'svg', 'VERSION'), 468 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 469 ) + 470 api.step_data('Scale CPU 0 to 0.600000', retcode=1) 471 ) 472 473 yield ( 474 api.test('cpu_scale_failed') + 475 api.properties(buildername=builder, 476 revision='abc123', 477 path_config='kitchen', 478 swarm_out_dir='[SWARM_OUT_DIR]') + 479 api.path.exists( 480 api.path['start_dir'].join('skia'), 481 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 482 'skimage', 'VERSION'), 483 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 484 'skp', 'VERSION'), 485 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 486 'svg', 'VERSION'), 487 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 488 ) + 489 api.step_data('get swarming bot id', 490 stdout=api.raw_io.output('skia-rpi-022')) + 491 api.step_data('Scale CPU 0 to 0.600000', retcode=1)+ 492 api.step_data('Scale CPU 0 to 0.600000 (attempt 2)', retcode=1)+ 493 api.step_data('Scale CPU 0 to 0.600000 (attempt 3)', retcode=1) 494 ) 495 496 builder = ('Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release' 497 '-All-Android') 498 yield ( 499 api.test('cpu_scale_failed_golo') + 500 api.properties(buildername=builder, 501 revision='abc123', 502 path_config='kitchen', 503 swarm_out_dir='[SWARM_OUT_DIR]') + 504 api.path.exists( 505 api.path['start_dir'].join('skia'), 506 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 507 'skimage', 'VERSION'), 508 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 509 'skp', 'VERSION'), 510 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 511 'svg', 'VERSION'), 512 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 513 ) + 514 api.step_data('get swarming bot id', 515 stdout=api.raw_io.output('build123-m2--device5')) + 516 api.step_data('Scale CPU 4 to 0.600000', retcode=1)+ 517 api.step_data('Scale CPU 4 to 0.600000 (attempt 2)', retcode=1)+ 518 api.step_data('Scale CPU 4 to 0.600000 (attempt 3)', retcode=1) 519 ) 520
