1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 # Recipe module for Skia Swarming perf. 7 8 9 import calendar 10 import os 11 12 13 DEPS = [ 14 'env', 15 'flavor', 16 'recipe_engine/file', 17 'recipe_engine/json', 18 'recipe_engine/path', 19 'recipe_engine/platform', 20 'recipe_engine/properties', 21 'recipe_engine/raw_io', 22 'recipe_engine/step', 23 'recipe_engine/time', 24 'run', 25 'vars', 26 ] 27 28 29 def upload_perf_results(buildername): 30 if 'Release' not in buildername: 31 return False 32 skip_upload_bots = [ 33 'ASAN', 34 'Coverage', 35 'MSAN', 36 'TSAN', 37 'UBSAN', 38 'Valgrind', 39 ] 40 for s in skip_upload_bots: 41 if s in buildername: 42 return False 43 return True 44 45 46 def nanobench_flags(api, bot): 47 args = ['--pre_log'] 48 49 if 'GPU' in bot: 50 args.append('--images') 51 args.extend(['--gpuStatsDump', 'true']) 52 53 args.extend(['--scales', '1.0', '1.1']) 54 55 if 'iOS' in bot: 56 args.extend(['--skps', 'ignore_skps']) 57 58 configs = [] 59 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 60 args.append('--nogpu') 61 configs.extend(['8888', 'nonrendering']) 62 63 if 'BonusConfigs' in bot or ('SAN' in bot and 'GCE' in bot): 64 configs += [ 65 'f16', 66 'srgb', 67 'esrgb', 68 'narrow', 69 'enarrow', 70 ] 71 72 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 73 args.append('--nocpu') 74 75 gl_prefix = 'gl' 76 sample_count = '8' 77 if 'Android' in bot or 'iOS' in bot: 78 sample_count = '4' 79 # The NVIDIA_Shield has a regular OpenGL implementation. We bench that 80 # instead of ES. 81 if 'NVIDIA_Shield' not in bot: 82 gl_prefix = 'gles' 83 # The NP produces a long error stream when we run with MSAA. 84 # iOS crashes (skia:6399) 85 # Nexus7 (Tegra3) does not support MSAA. 86 if ('NexusPlayer' in bot or 87 'iOS' in bot or 88 'Nexus7' in bot): 89 sample_count = '' 90 elif 'Intel' in bot: 91 sample_count = '' 92 elif 'ChromeOS' in bot: 93 gl_prefix = 'gles' 94 95 configs.extend([gl_prefix, gl_prefix + 'srgb']) 96 if sample_count is not '': 97 configs.append(gl_prefix + 'msaa' + sample_count) 98 if ('TegraX1' in bot or 99 'Quadro' in bot or 100 'GTX' in bot or 101 ('GT610' in bot and 'Ubuntu17' not in bot)): 102 configs.extend([gl_prefix + 'nvpr' + sample_count]) 103 104 # We want to test both the OpenGL config and the GLES config on Linux Intel: 105 # GL is used by Chrome, GLES is used by ChromeOS. 106 if 'Intel' in bot and api.vars.is_linux: 107 configs.extend(['gles', 'glessrgb']) 108 109 if 'CommandBuffer' in bot: 110 configs = ['commandbuffer'] 111 if 'Vulkan' in bot: 112 configs = ['vk'] 113 114 if 'ANGLE' in bot: 115 # Test only ANGLE configs. 116 configs = ['angle_d3d11_es2'] 117 if sample_count is not '': 118 configs.append('angle_d3d11_es2_msaa' + sample_count) 119 if 'QuadroP400' in bot: 120 # See skia:7823 and chromium:693090. 121 configs.append('angle_gl_es2') 122 if sample_count is not '': 123 configs.append('angle_gl_es2_msaa' + sample_count) 124 125 if 'ChromeOS' in bot: 126 # Just run GLES for now - maybe add gles_msaa4 in the future 127 configs = ['gles'] 128 129 args.append('--config') 130 args.extend(configs) 131 132 # By default, we test with GPU threading enabled, unless specifically 133 # disabled. 134 if 'NoGPUThreads' in bot: 135 args.extend(['--gpuThreads', '0']) 136 137 if 'Valgrind' in bot: 138 # Don't care about Valgrind performance. 139 args.extend(['--loops', '1']) 140 args.extend(['--samples', '1']) 141 # Ensure that the bot framework does not think we have timed out. 142 args.extend(['--keepAlive', 'true']) 143 144 if ('QuadroP400' in bot or 145 'Adreno540' in bot or 146 'IntelHD2000' in bot or # gen 6 - sandy bridge 147 'IntelHD4400' in bot or # gen 7 - haswell 148 'IntelHD405' in bot or # gen 8 - cherryview braswell 149 'IntelIris6100' in bot or # gen 8 - broadwell 150 'IntelIris540' in bot or # gen 9 - skylake 151 'IntelIris640' in bot or # gen 9 - kaby lake 152 'IntelIris655' in bot or # gen 9 - coffee lake 153 'MaliT760' in bot or 154 'MaliT860' in bot or 155 'MaliT880' in bot): 156 args.extend(['--reduceOpListSplitting']) 157 158 # Some people don't like verbose output. 159 verbose = False 160 161 match = [] 162 if 'Android' in bot: 163 # Segfaults when run as GPU bench. Very large texture? 164 match.append('~blurroundrect') 165 match.append('~patch_grid') # skia:2847 166 match.append('~desk_carsvg') 167 if 'Nexus5' in bot: 168 match.append('~keymobi_shop_mobileweb_ebay_com.skp') # skia:5178 169 if 'iOS' in bot: 170 match.append('~blurroundrect') 171 match.append('~patch_grid') # skia:2847 172 match.append('~desk_carsvg') 173 match.append('~keymobi') 174 match.append('~path_hairline') 175 match.append('~GLInstancedArraysBench') # skia:4714 176 if 'MoltenVK' in bot: 177 # skbug.com/7962 178 match.append('~^path_text_clipped_uncached$') 179 match.append('~^path_text_uncached$') 180 if ('Intel' in bot and api.vars.is_linux and not 'Vulkan' in bot): 181 # TODO(dogben): Track down what's causing bots to die. 182 verbose = True 183 if 'IntelHD405' in bot and api.vars.is_linux and 'Vulkan' in bot: 184 # skia:7322 185 match.append('~desk_carsvg.skp_1') 186 match.append('~desk_googlehome.skp') 187 match.append('~desk_tiger8svg.skp_1') 188 match.append('~desk_wowwiki.skp') 189 match.append('~desk_ynevsvg.skp_1.1') 190 match.append('~desk_nostroke_tiger8svg.skp') 191 match.append('~keymobi_booking_com.skp_1') 192 match.append('~keymobi_booking_com.skp_1_mpd') 193 match.append('~keymobi_cnn_article.skp_1') 194 match.append('~keymobi_cnn_article.skp_1_mpd') 195 match.append('~keymobi_forecast_io.skp_1') 196 match.append('~keymobi_forecast_io.skp_1_mpd') 197 match.append('~keymobi_sfgate.skp_1') 198 match.append('~keymobi_techcrunch_com.skp_1.1') 199 match.append('~keymobi_techcrunch.skp_1.1') 200 match.append('~keymobi_techcrunch.skp_1.1_mpd') 201 match.append('~svgparse_Seal_of_California.svg_1.1') 202 match.append('~svgparse_NewYork-StateSeal.svg_1.1') 203 match.append('~svgparse_Vermont_state_seal.svg_1') 204 match.append('~tabl_gamedeksiam.skp_1.1') 205 match.append('~tabl_pravda.skp_1') 206 match.append('~top25desk_ebay_com.skp_1.1') 207 match.append('~top25desk_ebay.skp_1.1') 208 match.append('~top25desk_ebay.skp_1.1_mpd') 209 if 'MacBook10.1' in bot and 'CommandBuffer' in bot: 210 match.append('~^desk_micrographygirlsvg.skp_1.1$') 211 if 'IntelIris655' in bot and 'Win10' in bot and 'Vulkan' in bot: 212 # skia:8587 213 match.append('~^GM_varied_text_clipped_lcd$') 214 match.append('~^GM_varied_text_ignorable_clip_lcd$') 215 match.append('~^fontscaler_lcd$') 216 match.append('~^rotated_rects_aa_changing_transparent_src$') 217 match.append('~^rotated_rects_aa_same_transparent_src$') 218 match.append('~^srcmode_rects_1_aa$') 219 match.append('~^desk_skbug6850overlay2.skp_1$') 220 match.append('~^desk_skbug6850overlay2.skp_1.1$') 221 match.append('~^desk_skbug6850overlay2.skp_1.1_mpd$') 222 match.append('~^desk_skbug6850overlay2.skp_1_mpd$') 223 # skia:8659 224 match.append('~^blendmode_mask_DstATop$') 225 match.append('~^blendmode_mask_Src$') 226 match.append('~^blendmode_mask_SrcIn$') 227 match.append('~^blendmode_mask_SrcOut$') 228 match.append('~^desk_carsvg.skp_1$') 229 match.append('~^desk_carsvg.skp_1.1$') 230 match.append('~^desk_carsvg.skp_1.1_mpd$') 231 match.append('~^desk_carsvg.skp_1_mpd$') 232 match.append('~^desk_googlespreadsheet.skp_1$') 233 match.append('~^desk_googlespreadsheet.skp_1.1$') 234 match.append('~^desk_googlespreadsheet.skp_1.1_mpd$') 235 match.append('~^desk_googlespreadsheet.skp_1_mpd$') 236 if 'Release' in bot: 237 match.append('~^rotated_rects_aa_alternating_transparent_and_opaque_src$') 238 match.append('~^shadermask_LCD_FF$') 239 match.append('~^text_16_LCD_88$') 240 match.append('~^text_16_LCD_BK$') 241 match.append('~^text_16_LCD_FF$') 242 match.append('~^text_16_LCD_WT$') 243 if ('ASAN' in bot or 'UBSAN' in bot) and 'CPU' in bot: 244 # floor2int_undef benches undefined behavior, so ASAN correctly complains. 245 match.append('~^floor2int_undef$') 246 247 # We do not need or want to benchmark the decodes of incomplete images. 248 # In fact, in nanobench we assert that the full image decode succeeds. 249 match.append('~inc0.gif') 250 match.append('~inc1.gif') 251 match.append('~incInterlaced.gif') 252 match.append('~inc0.jpg') 253 match.append('~incGray.jpg') 254 match.append('~inc0.wbmp') 255 match.append('~inc1.wbmp') 256 match.append('~inc0.webp') 257 match.append('~inc1.webp') 258 match.append('~inc0.ico') 259 match.append('~inc1.ico') 260 match.append('~inc0.png') 261 match.append('~inc1.png') 262 match.append('~inc2.png') 263 match.append('~inc12.png') 264 match.append('~inc13.png') 265 match.append('~inc14.png') 266 match.append('~inc0.webp') 267 match.append('~inc1.webp') 268 269 if match: 270 args.append('--match') 271 args.extend(match) 272 273 if verbose: 274 args.append('--verbose') 275 276 return args 277 278 279 def perf_steps(api): 280 """Run Skia benchmarks.""" 281 b = api.properties['buildername'] 282 if upload_perf_results(b): 283 api.flavor.create_clean_device_dir( 284 api.flavor.device_dirs.perf_data_dir) 285 286 # Run nanobench. 287 properties = [ 288 '--properties', 289 'gitHash', api.properties['revision'], 290 ] 291 if api.vars.is_trybot: 292 properties.extend([ 293 'issue', api.vars.issue, 294 'patchset', api.vars.patchset, 295 'patch_storage', api.vars.patch_storage, 296 ]) 297 properties.extend(['swarming_bot_id', api.vars.swarming_bot_id]) 298 properties.extend(['swarming_task_id', api.vars.swarming_task_id]) 299 300 target = 'nanobench' 301 args = [ 302 target, 303 '-i', api.flavor.device_dirs.resource_dir, 304 '--skps', api.flavor.device_dirs.skp_dir, 305 '--images', api.flavor.device_path_join( 306 api.flavor.device_dirs.images_dir, 'nanobench'), 307 ] 308 309 # Do not run svgs on Valgrind. 310 if 'Valgrind' not in api.vars.builder_name: 311 args.extend(['--svgs', api.flavor.device_dirs.svg_dir]) 312 313 args.extend(nanobench_flags(api, api.vars.builder_name)) 314 315 if 'Chromecast' in api.vars.builder_cfg.get('os', ''): 316 # Due to limited disk space, run a watered down perf run on Chromecast. 317 args = [target] 318 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 319 args.extend(['--nogpu', '--config', '8888']) 320 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 321 args.extend(['--nocpu', '--config', 'gles']) 322 args.extend([ 323 '-i', api.flavor.device_dirs.resource_dir, 324 '--images', api.flavor.device_path_join( 325 api.flavor.device_dirs.resource_dir, 'images', 'color_wheel.jpg'), 326 '--skps', api.flavor.device_dirs.skp_dir, 327 '--pre_log', 328 '--match', # skia:6687 329 '~matrixconvolution', 330 '~blur_image_filter', 331 '~blur_0.01', 332 '~GM_animated-image-blurs', 333 '~blendmode_mask_', 334 '~desk_carsvg.skp', 335 '~^path_text_clipped', # Bot times out; skia:7190 336 '~shapes_rrect_inner_rrect_50_500x500', # skia:7551 337 '~compositing_images', 338 ]) 339 340 if upload_perf_results(b): 341 now = api.time.utcnow() 342 ts = int(calendar.timegm(now.utctimetuple())) 343 json_path = api.flavor.device_path_join( 344 api.flavor.device_dirs.perf_data_dir, 345 'nanobench_%s_%d.json' % (api.properties['revision'], ts)) 346 args.extend(['--outResultsFile', json_path]) 347 args.extend(properties) 348 349 keys_blacklist = ['configuration', 'role', 'test_filter'] 350 args.append('--key') 351 for k in sorted(api.vars.builder_cfg.keys()): 352 if not k in keys_blacklist: 353 args.extend([k, api.vars.builder_cfg[k]]) 354 355 # See skia:2789. 356 if 'AbandonGpuContext' in api.vars.extra_tokens: 357 args.extend(['--abandonGpuContext']) 358 359 api.run(api.flavor.step, target, cmd=args, 360 abort_on_failure=False) 361 362 # Copy results to swarming out dir. 363 if upload_perf_results(b): 364 api.file.ensure_directory( 365 'makedirs perf_dir', 366 api.flavor.host_dirs.perf_data_dir) 367 api.flavor.copy_directory_contents_to_host( 368 api.flavor.device_dirs.perf_data_dir, 369 api.flavor.host_dirs.perf_data_dir) 370 371 372 def RunSteps(api): 373 api.vars.setup() 374 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 375 api.flavor.setup() 376 377 env = {} 378 if 'iOS' in api.vars.builder_name: 379 env['IOS_BUNDLE_ID'] = 'com.google.nanobench' 380 env['IOS_MOUNT_POINT'] = api.vars.slave_dir.join('mnt_iosdevice') 381 with api.env(env): 382 try: 383 if 'Chromecast' in api.vars.builder_name: 384 api.flavor.install(resources=True, skps=True) 385 else: 386 api.flavor.install(skps=True, images=True, svgs=True, resources=True) 387 perf_steps(api) 388 finally: 389 api.flavor.cleanup_steps() 390 api.run.check_failure() 391 392 393 TEST_BUILDERS = [ 394 'Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-All-Android', 395 ('Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-' 396 'Android_NoGPUThreads'), 397 'Perf-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Release-All', 398 'Perf-Chromecast-Clang-Chorizo-CPU-Cortex_A7-arm-Debug-All', 399 'Perf-Chromecast-Clang-Chorizo-GPU-Cortex_A7-arm-Release-All', 400 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All', 401 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN', 402 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-BonusConfigs', 403 'Perf-Debian9-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug-All-Vulkan', 404 'Perf-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Release-All', 405 ('Perf-Mac-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All-' 406 'CommandBuffer'), 407 'Perf-Mac-Clang-MacBook10.1-GPU-IntelHD615-x86_64-Release-All-CommandBuffer', 408 ('Perf-Mac-Clang-MacBookPro11.5-GPU-RadeonHD8870M-x86_64-Release-All-' 409 'MoltenVK_Vulkan'), 410 ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 411 'Valgrind_AbandonGpuContext_SK_CPU_LIMIT_SSE41'), 412 'Perf-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ANGLE', 413 'Perf-Win10-Clang-NUC8i5BEK-GPU-IntelIris655-x86_64-Release-All-Vulkan', 414 'Perf-iOS-Clang-iPadPro-GPU-PowerVRGT7800-arm64-Release-All', 415 ] 416 417 418 def GenTests(api): 419 for builder in TEST_BUILDERS: 420 test = ( 421 api.test(builder) + 422 api.properties(buildername=builder, 423 revision='abc123', 424 path_config='kitchen', 425 swarm_out_dir='[SWARM_OUT_DIR]') + 426 api.path.exists( 427 api.path['start_dir'].join('skia'), 428 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 429 'skimage', 'VERSION'), 430 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 431 'skp', 'VERSION'), 432 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 433 ) + 434 api.step_data('get swarming bot id', 435 stdout=api.raw_io.output('skia-bot-123')) + 436 api.step_data('get swarming task id', 437 stdout=api.raw_io.output('123456')) 438 ) 439 if 'Win' in builder: 440 test += api.platform('win', 64) 441 442 if 'Chromecast' in builder: 443 test += api.step_data( 444 'read chromecast ip', 445 stdout=api.raw_io.output('192.168.1.2:5555')) 446 447 if 'ChromeOS' in builder: 448 test += api.step_data( 449 'read chromeos ip', 450 stdout=api.raw_io.output('{"user_ip":"foo (at] 127.0.0.1"}')) 451 452 yield test 453 454 builder = 'Perf-Win10-Clang-NUCD34010WYKH-GPU-IntelHD4400-x86_64-Release-All' 455 yield ( 456 api.test('trybot') + 457 api.properties(buildername=builder, 458 revision='abc123', 459 path_config='kitchen', 460 swarm_out_dir='[SWARM_OUT_DIR]') + 461 api.properties(patch_storage='gerrit') + 462 api.properties.tryserver( 463 buildername=builder, 464 gerrit_project='skia', 465 gerrit_url='https://skia-review.googlesource.com/', 466 )+ 467 api.path.exists( 468 api.path['start_dir'].join('skia'), 469 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 470 'skimage', 'VERSION'), 471 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 472 'skp', 'VERSION'), 473 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 474 'svg', 'VERSION'), 475 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 476 ) 477 ) 478
