1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 from recipe_engine import recipe_api 7 8 from . import default 9 import subprocess # TODO(borenet): No! Remove this. 10 11 12 """Android flavor, used for running code on Android.""" 13 14 15 class AndroidFlavor(default.DefaultFlavor): 16 def __init__(self, m): 17 super(AndroidFlavor, self).__init__(m) 18 self._ever_ran_adb = False 19 self.ADB_BINARY = '/usr/bin/adb.1.0.35' 20 self.ADB_PUB_KEY = '/home/chrome-bot/.android/adbkey' 21 if 'skia' not in self.m.vars.swarming_bot_id: 22 self.ADB_BINARY = '/opt/infra-android/tools/adb' 23 self.ADB_PUB_KEY = ('/home/chrome-bot/.android/' 24 'chrome_infrastructure_adbkey') 25 26 # Data should go in android_data_dir, which may be preserved across runs. 27 android_data_dir = '/sdcard/revenge_of_the_skiabot/' 28 self.device_dirs = default.DeviceDirs( 29 bin_dir = '/data/local/tmp/', 30 dm_dir = android_data_dir + 'dm_out', 31 perf_data_dir = android_data_dir + 'perf', 32 resource_dir = android_data_dir + 'resources', 33 images_dir = android_data_dir + 'images', 34 lotties_dir = android_data_dir + 'lotties', 35 skp_dir = android_data_dir + 'skps', 36 svg_dir = android_data_dir + 'svgs', 37 tmp_dir = android_data_dir) 38 39 # A list of devices we can't root. If rooting fails and a device is not 40 # on the list, we fail the task to avoid perf inconsistencies. 41 self.rootable_blacklist = ['GalaxyS6', 'GalaxyS7_G930FD', 'GalaxyS9', 42 'MotoG4', 'NVIDIA_Shield'] 43 44 # Maps device type -> CPU ids that should be scaled for nanobench. 45 # Many devices have two (or more) different CPUs (e.g. big.LITTLE 46 # on Nexus5x). The CPUs listed are the biggest cpus on the device. 47 # The CPUs are grouped together, so we only need to scale one of them 48 # (the one listed) in order to scale them all. 49 # E.g. Nexus5x has cpu0-3 as one chip and cpu4-5 as the other. Thus, 50 # if one wants to run a single-threaded application (e.g. nanobench), one 51 # can disable cpu0-3 and scale cpu 4 to have only cpu4 and 5 at the same 52 # frequency. See also disable_for_nanobench. 53 self.cpus_to_scale = { 54 'Nexus5x': [4], 55 'NexusPlayer': [0, 2], # has 2 identical chips, so scale them both. 56 'Pixel': [2], 57 'Pixel2XL': [4] 58 } 59 60 # Maps device type -> CPU ids that should be turned off when running 61 # single-threaded applications like nanobench. The devices listed have 62 # multiple, differnt CPUs. We notice a lot of noise that seems to be 63 # caused by nanobench running on the slow CPU, then the big CPU. By 64 # disabling this, we see less of that noise by forcing the same CPU 65 # to be used for the performance testing every time. 66 self.disable_for_nanobench = { 67 'Nexus5x': range(0, 4), 68 'Pixel': range(0, 2), 69 'Pixel2XL': range(0, 4) 70 } 71 72 self.gpu_scaling = { 73 "Nexus5": 450000000, 74 "Nexus5x": 600000000, 75 } 76 77 def _run(self, title, *cmd, **kwargs): 78 with self.m.context(cwd=self.m.path['start_dir'].join('skia')): 79 return self.m.run(self.m.step, title, cmd=list(cmd), **kwargs) 80 81 def _adb(self, title, *cmd, **kwargs): 82 # The only non-infra adb steps (dm / nanobench) happen to not use _adb(). 83 if 'infra_step' not in kwargs: 84 kwargs['infra_step'] = True 85 86 self._ever_ran_adb = True 87 # ADB seems to be occasionally flaky on every device, so always retry. 88 attempts = 3 89 90 def wait_for_device(attempt): 91 self.m.run(self.m.step, 92 'kill adb server after failure of \'%s\' (attempt %d)' % ( 93 title, attempt), 94 cmd=[self.ADB_BINARY, 'kill-server'], 95 infra_step=True, timeout=30, abort_on_failure=False, 96 fail_build_on_failure=False) 97 self.m.run(self.m.step, 98 'wait for device after failure of \'%s\' (attempt %d)' % ( 99 title, attempt), 100 cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True, 101 timeout=180, abort_on_failure=False, 102 fail_build_on_failure=False) 103 104 with self.m.context(cwd=self.m.path['start_dir'].join('skia')): 105 with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}): 106 return self.m.run.with_retry(self.m.step, title, attempts, 107 cmd=[self.ADB_BINARY]+list(cmd), 108 between_attempts_fn=wait_for_device, 109 **kwargs) 110 111 def _scale_for_dm(self): 112 device = self.m.vars.builder_cfg.get('model') 113 if (device in self.rootable_blacklist or 114 self.m.vars.internal_hardware_label): 115 return 116 117 # This is paranoia... any CPUs we disabled while running nanobench 118 # ought to be back online now that we've restarted the device. 119 for i in self.disable_for_nanobench.get(device, []): 120 self._set_cpu_online(i, 1) # enable 121 122 scale_up = self.cpus_to_scale.get(device, [0]) 123 # For big.LITTLE devices, make sure we scale the LITTLE cores up; 124 # there is a chance they are still in powersave mode from when 125 # swarming slows things down for cooling down and charging. 126 if 0 not in scale_up: 127 scale_up.append(0) 128 for i in scale_up: 129 # AndroidOne doesn't support ondemand governor. hotplug is similar. 130 if device == 'AndroidOne': 131 self._set_governor(i, 'hotplug') 132 else: 133 self._set_governor(i, 'ondemand') 134 135 def _scale_for_nanobench(self): 136 device = self.m.vars.builder_cfg.get('model') 137 if (device in self.rootable_blacklist or 138 self.m.vars.internal_hardware_label): 139 return 140 141 for i in self.cpus_to_scale.get(device, [0]): 142 self._set_governor(i, 'userspace') 143 self._scale_cpu(i, 0.6) 144 145 for i in self.disable_for_nanobench.get(device, []): 146 self._set_cpu_online(i, 0) # disable 147 148 if device in self.gpu_scaling: 149 #https://developer.qualcomm.com/qfile/28823/lm80-p0436-11_adb_commands.pdf 150 # Section 3.2.1 Commands to put the GPU in performance mode 151 # Nexus 5 is 320000000 by default 152 # Nexus 5x is 180000000 by default 153 gpu_freq = self.gpu_scaling[device] 154 self.m.run.with_retry(self.m.python.inline, 155 "Lock GPU to %d (and other perf tweaks)" % gpu_freq, 156 3, # attempts 157 program=""" 158 import os 159 import subprocess 160 import sys 161 import time 162 ADB = sys.argv[1] 163 freq = sys.argv[2] 164 idle_timer = "10000" 165 166 log = subprocess.check_output([ADB, 'root']) 167 # check for message like 'adbd cannot run as root in production builds' 168 print log 169 if 'cannot' in log: 170 raise Exception('adb root failed') 171 172 subprocess.check_output([ADB, 'shell', 'stop', 'thermald']) 173 174 subprocess.check_output([ADB, 'shell', 'echo "%s" > ' 175 '/sys/class/kgsl/kgsl-3d0/gpuclk' % freq]) 176 177 actual_freq = subprocess.check_output([ADB, 'shell', 'cat ' 178 '/sys/class/kgsl/kgsl-3d0/gpuclk']).strip() 179 if actual_freq != freq: 180 raise Exception('Frequency (actual, expected) (%s, %s)' 181 % (actual_freq, freq)) 182 183 subprocess.check_output([ADB, 'shell', 'echo "%s" > ' 184 '/sys/class/kgsl/kgsl-3d0/idle_timer' % idle_timer]) 185 186 actual_timer = subprocess.check_output([ADB, 'shell', 'cat ' 187 '/sys/class/kgsl/kgsl-3d0/idle_timer']).strip() 188 if actual_timer != idle_timer: 189 raise Exception('idle_timer (actual, expected) (%s, %s)' 190 % (actual_timer, idle_timer)) 191 192 for s in ['force_bus_on', 'force_rail_on', 'force_clk_on']: 193 subprocess.check_output([ADB, 'shell', 'echo "1" > ' 194 '/sys/class/kgsl/kgsl-3d0/%s' % s]) 195 actual_set = subprocess.check_output([ADB, 'shell', 'cat ' 196 '/sys/class/kgsl/kgsl-3d0/%s' % s]).strip() 197 if actual_set != "1": 198 raise Exception('%s (actual, expected) (%s, 1)' 199 % (s, actual_set)) 200 """, 201 args = [self.ADB_BINARY, gpu_freq], 202 infra_step=True, 203 timeout=30) 204 205 def _set_governor(self, cpu, gov): 206 self._ever_ran_adb = True 207 self.m.run.with_retry(self.m.python.inline, 208 "Set CPU %d's governor to %s" % (cpu, gov), 209 3, # attempts 210 program=""" 211 import os 212 import subprocess 213 import sys 214 import time 215 ADB = sys.argv[1] 216 cpu = int(sys.argv[2]) 217 gov = sys.argv[3] 218 219 log = subprocess.check_output([ADB, 'root']) 220 # check for message like 'adbd cannot run as root in production builds' 221 print log 222 if 'cannot' in log: 223 raise Exception('adb root failed') 224 225 subprocess.check_output([ADB, 'shell', 'echo "%s" > ' 226 '/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor' % (gov, cpu)]) 227 actual_gov = subprocess.check_output([ADB, 'shell', 'cat ' 228 '/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor' % cpu]).strip() 229 if actual_gov != gov: 230 raise Exception('(actual, expected) (%s, %s)' 231 % (actual_gov, gov)) 232 """, 233 args = [self.ADB_BINARY, cpu, gov], 234 infra_step=True, 235 timeout=30) 236 237 238 def _set_cpu_online(self, cpu, value): 239 """Set /sys/devices/system/cpu/cpu{N}/online to value (0 or 1).""" 240 self._ever_ran_adb = True 241 msg = 'Disabling' 242 if value: 243 msg = 'Enabling' 244 self.m.run.with_retry(self.m.python.inline, 245 '%s CPU %d' % (msg, cpu), 246 3, # attempts 247 program=""" 248 import os 249 import subprocess 250 import sys 251 import time 252 ADB = sys.argv[1] 253 cpu = int(sys.argv[2]) 254 value = int(sys.argv[3]) 255 256 log = subprocess.check_output([ADB, 'root']) 257 # check for message like 'adbd cannot run as root in production builds' 258 print log 259 if 'cannot' in log: 260 raise Exception('adb root failed') 261 262 # If we try to echo 1 to an already online cpu, adb returns exit code 1. 263 # So, check the value before trying to write it. 264 prior_status = subprocess.check_output([ADB, 'shell', 'cat ' 265 '/sys/devices/system/cpu/cpu%d/online' % cpu]).strip() 266 if prior_status == str(value): 267 print 'CPU %d online already %d' % (cpu, value) 268 sys.exit() 269 270 subprocess.check_output([ADB, 'shell', 'echo %s > ' 271 '/sys/devices/system/cpu/cpu%d/online' % (value, cpu)]) 272 actual_status = subprocess.check_output([ADB, 'shell', 'cat ' 273 '/sys/devices/system/cpu/cpu%d/online' % cpu]).strip() 274 if actual_status != str(value): 275 raise Exception('(actual, expected) (%s, %d)' 276 % (actual_status, value)) 277 """, 278 args = [self.ADB_BINARY, cpu, value], 279 infra_step=True, 280 timeout=30) 281 282 283 def _scale_cpu(self, cpu, target_percent): 284 self._ever_ran_adb = True 285 self.m.run.with_retry(self.m.python.inline, 286 'Scale CPU %d to %f' % (cpu, target_percent), 287 3, # attempts 288 program=""" 289 import os 290 import subprocess 291 import sys 292 import time 293 ADB = sys.argv[1] 294 target_percent = float(sys.argv[2]) 295 cpu = int(sys.argv[3]) 296 log = subprocess.check_output([ADB, 'root']) 297 # check for message like 'adbd cannot run as root in production builds' 298 print log 299 if 'cannot' in log: 300 raise Exception('adb root failed') 301 302 root = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu 303 304 # All devices we test on give a list of their available frequencies. 305 available_freqs = subprocess.check_output([ADB, 'shell', 306 'cat %s/scaling_available_frequencies' % root]) 307 308 # Check for message like '/system/bin/sh: file not found' 309 if available_freqs and '/system/bin/sh' not in available_freqs: 310 available_freqs = sorted( 311 int(i) for i in available_freqs.strip().split()) 312 else: 313 raise Exception('Could not get list of available frequencies: %s' % 314 available_freqs) 315 316 maxfreq = available_freqs[-1] 317 target = int(round(maxfreq * target_percent)) 318 freq = maxfreq 319 for f in reversed(available_freqs): 320 if f <= target: 321 freq = f 322 break 323 324 print 'Setting frequency to %d' % freq 325 326 # If scaling_max_freq is lower than our attempted setting, it won't take. 327 # We must set min first, because if we try to set max to be less than min 328 # (which sometimes happens after certain devices reboot) it returns a 329 # perplexing permissions error. 330 subprocess.check_output([ADB, 'shell', 'echo 0 > ' 331 '%s/scaling_min_freq' % root]) 332 subprocess.check_output([ADB, 'shell', 'echo %d > ' 333 '%s/scaling_max_freq' % (freq, root)]) 334 subprocess.check_output([ADB, 'shell', 'echo %d > ' 335 '%s/scaling_setspeed' % (freq, root)]) 336 time.sleep(5) 337 actual_freq = subprocess.check_output([ADB, 'shell', 'cat ' 338 '%s/scaling_cur_freq' % root]).strip() 339 if actual_freq != str(freq): 340 raise Exception('(actual, expected) (%s, %d)' 341 % (actual_freq, freq)) 342 """, 343 args = [self.ADB_BINARY, str(target_percent), cpu], 344 infra_step=True, 345 timeout=30) 346 347 348 def install(self): 349 self._adb('mkdir ' + self.device_dirs.resource_dir, 350 'shell', 'mkdir', '-p', self.device_dirs.resource_dir) 351 if 'ASAN' in self.m.vars.extra_tokens: 352 self._ever_ran_adb = True 353 asan_setup = self.m.vars.slave_dir.join( 354 'android_ndk_linux', 'toolchains', 'llvm', 'prebuilt', 355 'linux-x86_64', 'lib64', 'clang', '8.0.2', 'bin', 356 'asan_device_setup') 357 self.m.run(self.m.python.inline, 'Setting up device to run ASAN', 358 program=""" 359 import os 360 import subprocess 361 import sys 362 import time 363 ADB = sys.argv[1] 364 ASAN_SETUP = sys.argv[2] 365 366 def wait_for_device(): 367 while True: 368 time.sleep(5) 369 print 'Waiting for device' 370 subprocess.check_output([ADB, 'wait-for-device']) 371 bit1 = subprocess.check_output([ADB, 'shell', 'getprop', 372 'dev.bootcomplete']) 373 bit2 = subprocess.check_output([ADB, 'shell', 'getprop', 374 'sys.boot_completed']) 375 if '1' in bit1 and '1' in bit2: 376 print 'Device detected' 377 break 378 379 log = subprocess.check_output([ADB, 'root']) 380 # check for message like 'adbd cannot run as root in production builds' 381 print log 382 if 'cannot' in log: 383 raise Exception('adb root failed') 384 385 output = subprocess.check_output([ADB, 'disable-verity']) 386 print output 387 388 if 'already disabled' not in output: 389 print 'Rebooting device' 390 subprocess.check_output([ADB, 'reboot']) 391 wait_for_device() 392 393 def installASAN(revert=False): 394 # ASAN setup script is idempotent, either it installs it or 395 # says it's installed. Returns True on success, false otherwise. 396 out = subprocess.check_output([ADB, 'wait-for-device']) 397 print out 398 cmd = [ASAN_SETUP] 399 if revert: 400 cmd = [ASAN_SETUP, '--revert'] 401 process = subprocess.Popen(cmd, env={'ADB': ADB}, 402 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 403 404 # this also blocks until command finishes 405 (stdout, stderr) = process.communicate() 406 print stdout 407 print 'Stderr: %s' % stderr 408 return process.returncode == 0 409 410 if not installASAN(): 411 print 'Trying to revert the ASAN install and then re-install' 412 # ASAN script sometimes has issues if it was interrupted or partially applied 413 # Try reverting it, then re-enabling it 414 if not installASAN(revert=True): 415 raise Exception('reverting ASAN install failed') 416 417 # Sleep because device does not reboot instantly 418 time.sleep(10) 419 420 if not installASAN(): 421 raise Exception('Tried twice to setup ASAN and failed.') 422 423 # Sleep because device does not reboot instantly 424 time.sleep(10) 425 wait_for_device() 426 # Sleep again to hopefully avoid error "secure_mkdirs failed: No such file or 427 # directory" when pushing resources to the device. 428 time.sleep(60) 429 """, 430 args = [self.ADB_BINARY, asan_setup], 431 infra_step=True, 432 timeout=300, 433 abort_on_failure=True) 434 435 436 def cleanup_steps(self): 437 if 'ASAN' in self.m.vars.extra_tokens: 438 self._ever_ran_adb = True 439 # Remove ASAN. 440 asan_setup = self.m.vars.slave_dir.join( 441 'android_ndk_linux', 'toolchains', 'llvm', 'prebuilt', 442 'linux-x86_64', 'lib64', 'clang', '8.0.2', 'bin', 443 'asan_device_setup') 444 self.m.run(self.m.step, 445 'wait for device before uninstalling ASAN', 446 cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True, 447 timeout=180, abort_on_failure=False, 448 fail_build_on_failure=False) 449 self.m.run(self.m.step, 'uninstall ASAN', 450 cmd=[asan_setup, '--revert'], infra_step=True, timeout=300, 451 abort_on_failure=False, fail_build_on_failure=False) 452 453 if self._ever_ran_adb: 454 self.m.run(self.m.python.inline, 'dump log', program=""" 455 import os 456 import subprocess 457 import sys 458 out = sys.argv[1] 459 log = subprocess.check_output(['%s', 'logcat', '-d']) 460 for line in log.split('\\n'): 461 tokens = line.split() 462 if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc': 463 addr, path = tokens[-2:] 464 local = os.path.join(out, os.path.basename(path)) 465 if os.path.exists(local): 466 sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr]) 467 line = line.replace(addr, addr + ' ' + sym.strip()) 468 print line 469 """ % self.ADB_BINARY, 470 args=[self.host_dirs.bin_dir], 471 infra_step=True, 472 timeout=300, 473 abort_on_failure=False) 474 475 # Only quarantine the bot if the first failed step 476 # is an infra step. If, instead, we did this for any infra failures, we 477 # would do this too much. For example, if a Nexus 10 died during dm 478 # and the following pull step would also fail "device not found" - causing 479 # us to run the shutdown command when the device was probably not in a 480 # broken state; it was just rebooting. 481 if (self.m.run.failed_steps and 482 isinstance(self.m.run.failed_steps[0], recipe_api.InfraFailure)): 483 bot_id = self.m.vars.swarming_bot_id 484 self.m.file.write_text('Quarantining Bot', 485 '/home/chrome-bot/%s.force_quarantine' % bot_id, 486 ' ') 487 488 if self._ever_ran_adb: 489 self._adb('kill adb server', 'kill-server') 490 491 def step(self, name, cmd, **kwargs): 492 if (cmd[0] == 'nanobench'): 493 self._scale_for_nanobench() 494 else: 495 self._scale_for_dm() 496 app = self.host_dirs.bin_dir.join(cmd[0]) 497 self._adb('push %s' % cmd[0], 498 'push', app, self.device_dirs.bin_dir) 499 500 sh = '%s.sh' % cmd[0] 501 self.m.run.writefile(self.m.vars.tmp_dir.join(sh), 502 'set -x; %s%s; echo $? >%src' % ( 503 self.device_dirs.bin_dir, subprocess.list2cmdline(map(str, cmd)), 504 self.device_dirs.bin_dir)) 505 self._adb('push %s' % sh, 506 'push', self.m.vars.tmp_dir.join(sh), self.device_dirs.bin_dir) 507 508 self._adb('clear log', 'logcat', '-c') 509 self.m.python.inline('%s' % cmd[0], """ 510 import subprocess 511 import sys 512 bin_dir = sys.argv[1] 513 sh = sys.argv[2] 514 subprocess.check_call(['%s', 'shell', 'sh', bin_dir + sh]) 515 try: 516 sys.exit(int(subprocess.check_output(['%s', 'shell', 'cat', 517 bin_dir + 'rc']))) 518 except ValueError: 519 print "Couldn't read the return code. Probably killed for OOM." 520 sys.exit(1) 521 """ % (self.ADB_BINARY, self.ADB_BINARY), 522 args=[self.device_dirs.bin_dir, sh]) 523 524 def copy_file_to_device(self, host, device): 525 self._adb('push %s %s' % (host, device), 'push', host, device) 526 527 def copy_directory_contents_to_device(self, host, device): 528 # Copy the tree, avoiding hidden directories and resolving symlinks. 529 self.m.run(self.m.python.inline, 'push %s/* %s' % (host, device), 530 program=""" 531 import os 532 import subprocess 533 import sys 534 host = sys.argv[1] 535 device = sys.argv[2] 536 for d, _, fs in os.walk(host): 537 p = os.path.relpath(d, host) 538 if p != '.' and p.startswith('.'): 539 continue 540 for f in fs: 541 print os.path.join(p,f) 542 subprocess.check_call(['%s', 'push', 543 os.path.realpath(os.path.join(host, p, f)), 544 os.path.join(device, p, f)]) 545 """ % self.ADB_BINARY, args=[host, device], infra_step=True) 546 547 def copy_directory_contents_to_host(self, device, host): 548 # TODO(borenet): When all of our devices are on Android 6.0 and up, we can 549 # switch to using tar to zip up the results before pulling. 550 with self.m.step.nest('adb pull'): 551 with self.m.tempfile.temp_dir('adb_pull') as tmp: 552 self._adb('pull %s' % device, 'pull', device, tmp) 553 paths = self.m.file.glob_paths( 554 'list pulled files', 555 tmp, 556 self.m.path.basename(device) + self.m.path.sep + '*', 557 test_data=['%d.png' % i for i in (1, 2)]) 558 for p in paths: 559 self.m.file.copy('copy %s' % self.m.path.basename(p), p, host) 560 561 def read_file_on_device(self, path, **kwargs): 562 rv = self._adb('read %s' % path, 563 'shell', 'cat', path, stdout=self.m.raw_io.output(), 564 **kwargs) 565 return rv.stdout.rstrip() if rv and rv.stdout else None 566 567 def remove_file_on_device(self, path): 568 self._adb('rm %s' % path, 'shell', 'rm', '-f', path) 569 570 def create_clean_device_dir(self, path): 571 self._adb('rm %s' % path, 'shell', 'rm', '-rf', path) 572 self._adb('mkdir %s' % path, 'shell', 'mkdir', '-p', path) 573
