1 #!/usr/bin/python3 2 # 3 # Copyright (C) 2016 The Android Open-Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # This script is located in internal ARC repo. 18 # Search for android_libs/arc/push_to_device.py 19 20 from __future__ import print_function 21 22 import argparse 23 import atexit 24 import hashlib 25 import itertools 26 import logging 27 import os 28 import pipes 29 import re 30 import shutil 31 import string 32 import subprocess 33 import sys 34 import tempfile 35 import time 36 import xml.etree.cElementTree as ElementTree 37 import zipfile 38 39 import lib.util 40 41 42 _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) 43 44 _EXPECTED_TARGET_PRODUCTS = { 45 '^x86': ('cheets_x86', 'cheets_x86_64'), 46 '^arm': ('cheets_arm',), 47 '^aarch64$': ('cheets_arm',), 48 } 49 _ANDROID_ROOT = '/opt/google/containers/android' 50 _ANDROID_ROOT_STATEFUL = os.path.join('/usr/local', 51 os.path.relpath(_ANDROID_ROOT, '/')) 52 _CONTAINER_INSTANCE_ROOT_WILDCARD = '/run/containers/android_*' 53 _CONTAINER_ROOT = os.path.join(_ANDROID_ROOT, 'rootfs', 'root') 54 _RSYNC_COMMAND = ['rsync', '--inplace', '-v', '--progress'] 55 _SCP_COMMAND = ['scp'] 56 57 _BUILD_FILENAME = string.Template('${product}-img-${build_id}.zip') 58 _BUILD_TARGET = string.Template('${product}-${build_variant}') 59 60 _CHROMEOS_ARC_ANDROID_SDK_VERSION = 'CHROMEOS_ARC_ANDROID_SDK_VERSION=' 61 62 _GENERIC_DEVICE = 'generic_%(arch)s_cheets' 63 _RO_BUILD_TYPE = 'ro.build.type=' 64 _RO_BUILD_VERSION_SDK = 'ro.build.version.sdk=' 65 _RO_PRODUCT_DEVICE = 'ro.product.device=' 66 67 _ANDROID_REL_KEY_SIGNATURE_SUBSTRING = ( 68 '55b390dd7fdb9418631895d5f759f30112687ff621410c069308a') 69 _APK_KEY_DEBUG = 'debug-key' 70 _APK_KEY_RELEASE = 'release-key' 71 _APK_KEY_UNKNOWN = 'unknown' 72 _GMS_CORE_PACKAGE_NAME = 'com.google.android.gms' 73 74 _ANDROID_SDK_MAPPING = { 75 23: "M (API 23)", 76 24: "N (API 24)", 77 25: "N_MR1 (API 25)", 78 26: "O (API 26)", 79 } 80 81 class RemoteProxy(object): 82 """Proxy class to run command line on the remote test device.""" 83 84 def __init__(self, remote, dryrun): 85 self._remote = remote 86 self._dryrun = dryrun 87 self._sync_command = ( 88 _RSYNC_COMMAND if self._has_rsync_on_remote_device() else _SCP_COMMAND) 89 90 def check_call(self, remote_command): 91 """Runs |remote_command| on the remote test device via ssh.""" 92 command = self.get_ssh_commandline(remote_command) 93 lib.util.check_call(dryrun=self._dryrun, *command) 94 95 def check_output(self, remote_command): 96 """Runs |remote_command| on the remote test device via ssh, and returns 97 its output.""" 98 command = self.get_ssh_commandline(remote_command) 99 return lib.util.check_output(dryrun=self._dryrun, *command) 100 101 def sync(self, file_list, dest_dir): 102 """Copies |file_list| to the |dest_dir| on the remote test device.""" 103 target = 'root@%s:%s' % (self._remote, dest_dir) 104 command = self._sync_command + file_list + [target] 105 lib.util.check_call(dryrun=self._dryrun, *command) 106 107 def push(self, source_path, dest_path): 108 """Pushes |source_path| on the host, to |dest_path| on the remote test 109 device. 110 111 Args: 112 source_path: Host file path to be pushed. 113 dest_path: Path to the destination location on the remote test device. 114 """ 115 target = 'root@%s:%s' % (self._remote, dest_path) 116 command = _SCP_COMMAND + [source_path, target] 117 lib.util.check_call(dryrun=self._dryrun, *command) 118 119 def pull(self, source_path, dest_path): 120 """Pulls |source_path| from the remote test device, to |dest_path| on the 121 host. 122 123 Args: 124 source_path: Remote test device file path to be pulled. 125 dest_path: Path to the destination location on the host. 126 """ 127 target = 'root@%s:%s' % (self._remote, source_path) 128 command = _SCP_COMMAND + [target, dest_path] 129 return lib.util.check_call(dryrun=self._dryrun, *command) 130 131 def get_ssh_commandline(self, remote_command): 132 return ['ssh', 'root@' + self._remote, remote_command] 133 134 def _has_rsync_on_remote_device(self): 135 command = self.get_ssh_commandline('which rsync') 136 logging.debug('Calling: %s', lib.util.get_command_str(command)) 137 # Always return true for --dryrun. 138 return self._dryrun or subprocess.call(command) == 0 139 140 141 class TemporaryDirectory(object): 142 """A context object that has a temporary directory with the same lifetime.""" 143 144 def __init__(self): 145 self.name = None 146 147 def __enter__(self): 148 self.name = tempfile.mkdtemp() 149 return self 150 151 def __exit__(self, exception_type, exception_value, traceback): 152 shutil.rmtree(self.name) 153 154 155 class MountWrapper(object): 156 """A context object that mounts an image during the lifetime.""" 157 158 def __init__(self, image_path, mountpoint): 159 self._image_path = image_path 160 self._mountpoint = mountpoint 161 162 def __enter__(self): 163 lib.util.check_call('/usr/bin/sudo', '/bin/mount', '-o', 'loop', 164 self._image_path, self._mountpoint) 165 return self 166 167 def __exit__(self, exception_type, exception_value, traceback): 168 try: 169 lib.util.check_call('/usr/bin/sudo', '/bin/umount', self._mountpoint) 170 except Exception: 171 if not exception_type: 172 raise 173 # Instead of propagate the exception, record the one from exit body. 174 logging.exception('Failed to umount ' + self._mountpoint) 175 176 177 class Simg2img(object): 178 """Wrapper class of simg2img""" 179 180 def __init__(self, simg2img_path, dryrun): 181 self._path = simg2img_path 182 self._dryrun = dryrun 183 184 def convert(self, src, dest): 185 """Converts the image to the raw image by simg2img command line. 186 187 If |dryrun| is set, does not execute the commandline. 188 """ 189 lib.util.check_call(self._path, src, dest, dryrun=self._dryrun) 190 191 192 def _verify_machine_arch(remote_proxy, target_product, dryrun): 193 """Verifies if the data being pushed is build for the target architecture. 194 195 Args: 196 remote_proxy: RemoteProxy instance for the remote test device. 197 target_product: Target product name of the image being pushed. This is 198 usually set by "lunch" command. E.g. "cheets_x86" or "cheets_arm". 199 dryrun: If set, this function assumes the machine architectures match. 200 201 Raises: 202 AssertionError: If the pushing image does not match to the remote test 203 device. 204 """ 205 if dryrun: 206 logging.debug('Pretending machine architectures match') 207 return 208 remote_arch = remote_proxy.check_output('uname -m') 209 for arch_pattern, expected_set in _EXPECTED_TARGET_PRODUCTS.items(): 210 if re.search(arch_pattern, remote_arch): 211 expected = itertools.chain.from_iterable( 212 (expected, 'aosp_' + expected, expected + '_gmscore_next') for 213 expected in expected_set) 214 assert target_product in expected, ( 215 ('Architecture mismatch: Deploying \'%s\' to \'%s\' seems incorrect.' 216 % (target_product, remote_arch))) 217 return 218 logging.warning('Unknown remote machine type \'%s\'. Skipping ' 219 'architecture sanity check.', remote_arch) 220 221 222 def _convert_images(simg2img, out, push_vendor_image): 223 """Converts the images being pushed to the raw images. 224 225 Returns: 226 A tuple of (large_file_list, file_list). Each list consists of paths of 227 converted files. 228 """ 229 result = [] 230 result_large = [] 231 232 system_raw_img = os.path.join(out, 'system.raw.img') 233 simg2img.convert(os.path.join(out, 'system.img'), system_raw_img) 234 result_large.append(system_raw_img) 235 236 if push_vendor_image: 237 vendor_raw_img = os.path.join(out, 'vendor.raw.img') 238 simg2img.convert(os.path.join(out, 'vendor.img'), vendor_raw_img) 239 result.append(vendor_raw_img) 240 241 return (result_large, result) 242 243 244 def _update_build_fingerprint(remote_proxy, build_fingerprint): 245 """Updates CHROMEOS_ARC_VERSION in /etc/lsb-release. 246 247 Args: 248 remote_proxy: RemoteProxy instance connected to the test device. 249 build_fingerprint: The version code which should be embedded into 250 /etc/lsb-release. 251 """ 252 if not build_fingerprint: 253 logging.warning( 254 'Skipping version update. ARC version will be reported incorrectly') 255 return 256 257 # Replace the ARC version on disk with what we're pushing there. 258 logging.info('Updating CHROMEOS_ARC_VERSION...') 259 remote_proxy.check_call(' '.join([ 260 '/bin/sed', '-i', 261 # Note: we assume build_fingerprint does not contain any char which 262 # needs to be escaped. 263 r'"s/^\(CHROMEOS_ARC_VERSION=\).*/\1%(_BUILD_FINGERPRINT)s/"', 264 '/etc/lsb-release' 265 ]) % {'_BUILD_FINGERPRINT': build_fingerprint}) 266 267 268 def _get_remote_device_android_sdk_version(remote_proxy, dryrun): 269 """ Returns the Android SDK version on the remote device. 270 271 Args: 272 remote_proxy: RemoteProxy instance for the remote test device. 273 dryrun: If set, this function assumes Android SDK version is 1. 274 """ 275 if dryrun: 276 logging.debug('Pretending target device\'s Android SDK version is 1') 277 return 1 278 try: 279 line = remote_proxy.check_output( 280 'grep ^%s /etc/lsb-release' % _CHROMEOS_ARC_ANDROID_SDK_VERSION).strip() 281 except subprocess.CalledProcessError: 282 logging.exception('Failed to inspect /etc/lsb-release remotely') 283 return None 284 285 if not line.startswith(_CHROMEOS_ARC_ANDROID_SDK_VERSION): 286 logging.warning('Failed to find the correct string format.\n' 287 'Expected format: "%s"\nActual string: "%s"', 288 _CHROMEOS_ARC_ANDROID_SDK_VERSION, line) 289 return None 290 291 android_sdk_version = int( 292 line[len(_CHROMEOS_ARC_ANDROID_SDK_VERSION):].strip()) 293 logging.debug('Target device\'s Android SDK version: %d', android_sdk_version) 294 return android_sdk_version 295 296 297 def _verify_android_sdk_version(remote_proxy, provider, dryrun): 298 """Verifies if the Android SDK versions of the pushing image and the test 299 device are the same. 300 301 Args: 302 remote_proxy: RemoteProxy instance for the remote test device. 303 provider: Android image provider. 304 dryrun: If set, this function assumes Android SDK versions match. 305 306 Raises: 307 AssertionError: If the Android SDK version of pushing image does not match 308 the Android SDK version on the remote test device. 309 """ 310 if dryrun: 311 logging.debug('Pretending Android SDK versions match') 312 return 313 logging.debug('New image\'s Android SDK version: %d', 314 provider.get_build_version_sdk()) 315 316 device_android_sdk_version = _get_remote_device_android_sdk_version( 317 remote_proxy, dryrun) 318 319 if device_android_sdk_version is None: 320 if not boolean_prompt(('Unable to determine the target device\'s Android ' 321 'SDK version. Continue?'), False): 322 sys.exit(1) 323 else: 324 assert device_android_sdk_version == provider.get_build_version_sdk(), ( 325 'Android SDK versions do not match. The target device has {}, while ' 326 'the new image is {}'.format( 327 _android_sdk_version_to_string(device_android_sdk_version), 328 _android_sdk_version_to_string(provider.get_build_version_sdk()))) 329 330 331 def _android_sdk_version_to_string(android_sdk_version): 332 """Converts the |android_sdk_version| to a human readable string 333 334 Args: 335 android_sdk_version: The Android SDK version number as a string 336 """ 337 return _ANDROID_SDK_MAPPING.get( 338 android_sdk_version, 339 'Unknown SDK Version (API {})'.format(android_sdk_version)) 340 341 342 def _is_selinux_policy_updated(remote_proxy, out, dryrun): 343 """Returns True if SELinux policy is updated.""" 344 if dryrun: 345 logging.debug('Pretending sepolicy is not updated in dryrun mode') 346 return False 347 remote_sepolicy_sha1, _ = remote_proxy.check_output( 348 'sha1sum /etc/selinux/arc/policy/policy.30').split() 349 with open(os.path.join(out, 'root', 'sepolicy'), 'rb') as f: 350 host_sepolicy_sha1 = hashlib.sha1(f.read()).hexdigest() 351 return remote_sepolicy_sha1 != host_sepolicy_sha1 352 353 354 def _update_selinux_policy(remote_proxy, out): 355 """Updates the selinux policy file.""" 356 remote_proxy.push(os.path.join(out, 'root', 'sepolicy'), 357 '/etc/selinux/arc/policy/policy.30') 358 359 360 def _remount_rootfs_as_writable(remote_proxy): 361 """Remounts root file system to make it writable.""" 362 remote_proxy.check_call('mount -o remount,rw /') 363 364 365 def boolean_prompt(prompt, default=True, true_value='yes', false_value='no', 366 prolog=None): 367 """Helper function for processing boolean choice prompts. 368 369 Args: 370 prompt: The question to present to the user. 371 default: Boolean to return if the user just presses enter. 372 true_value: The text to display that represents a True returned. 373 false_value: The text to display that represents a False returned. 374 prolog: The text to display before prompt. 375 376 Returns: 377 True or False. 378 """ 379 true_value, false_value = true_value.lower(), false_value.lower() 380 true_text, false_text = true_value, false_value 381 if true_value == false_value: 382 raise ValueError('true_value and false_value must differ: got %r' 383 % true_value) 384 385 if default: 386 true_text = true_text[0].upper() + true_text[1:] 387 else: 388 false_text = false_text[0].upper() + false_text[1:] 389 390 prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text)) 391 392 if prolog: 393 prompt = ('\n%s\n%s' % (prolog, prompt)) 394 395 while True: 396 try: 397 response = input(prompt).lower() 398 except EOFError: 399 # If the user hits CTRL+D, or stdin is disabled, use the default. 400 print(file=sys.stderr) 401 response = None 402 except KeyboardInterrupt: 403 # If the user hits CTRL+C, just exit the process. 404 print(file=sys.stderr) 405 print('CTRL+C detected; exiting', file=sys.stderr) 406 raise 407 408 if not response: 409 return default 410 if true_value.startswith(response): 411 if not false_value.startswith(response): 412 return True 413 # common prefix between the two... 414 elif false_value.startswith(response): 415 return False 416 417 418 def _disable_rootfs_verification(force, remote_proxy): 419 make_dev_ssd_path = \ 420 '/usr/libexec/debugd/helpers/dev_features_rootfs_verification' 421 make_dev_ssd_command = remote_proxy.get_ssh_commandline(make_dev_ssd_path) 422 if not force: 423 logging.error('Detected that the device has rootfs verification enabled.') 424 logging.info('This script can automatically remove the rootfs ' 425 'verification using `%s`, which requires that the device is ' 426 'rebooted afterwards.', 427 lib.util.get_command_str(make_dev_ssd_command)) 428 logging.info('Skip this prompt by specifying --force.') 429 if not boolean_prompt('Remove rootfs verification?', False): 430 return False 431 remote_proxy.check_call(make_dev_ssd_path) 432 reboot_time = time.time() 433 remote_proxy.check_call('reboot') 434 logging.debug('Waiting up to 10 seconds for the machine to reboot') 435 for _ in range(10): 436 time.sleep(1) 437 try: 438 device_boot_time = remote_proxy.check_output('grep btime /proc/stat | ' + 439 'cut -d" " -f2') 440 if int(device_boot_time) >= reboot_time: 441 return True 442 except subprocess.CalledProcessError: 443 pass 444 logging.error('Failed to detect whether the device had successfully rebooted') 445 return False 446 447 def _stop_ui(remote_proxy): 448 remote_proxy.check_call('\n'.join([ 449 # Stop UI if necessary. 450 'if ! (status ui | grep -q stop); then', 451 ' stop ui', 452 'fi', 453 454 # Unmount the container root/vendor and root if necessary. 455 'stop arc-system-mount', 456 # TODO(yusukes): Remove the manual umount below once everyone starts using 457 # arc-system-mount.conf with the post-stop script. 458 'if mountpoint -q %(_CONTAINER_ROOT)s/vendor; then', 459 ' umount %(_CONTAINER_ROOT)s/vendor', 460 'fi', 461 'if mountpoint -q %(_CONTAINER_ROOT)s; then', 462 ' umount %(_CONTAINER_ROOT)s', 463 'fi', 464 ]) % {'_CONTAINER_ROOT': _CONTAINER_ROOT}) 465 466 467 class ImageUpdateMode(object): 468 """Context object to manage remote host writable status.""" 469 def __init__(self, remote_proxy, is_selinux_policy_updated, push_to_stateful, 470 clobber_data, force): 471 self._remote_proxy = remote_proxy 472 self._is_selinux_policy_updated = is_selinux_policy_updated 473 self._push_to_stateful = push_to_stateful 474 self._clobber_data = clobber_data 475 self._force = force 476 477 def __enter__(self): 478 logging.info('Setting up ChromeOS device to image-writable...') 479 480 if self._clobber_data: 481 self._remote_proxy.check_call( 482 'if [ -e %(ANDROID_ROOT_WILDCARD)s/root/data ]; then' 483 ' kill -9 `cat %(ANDROID_ROOT_WILDCARD)s/container.pid`;' 484 ' find %(ANDROID_ROOT_WILDCARD)s/root/data' 485 ' %(ANDROID_ROOT_WILDCARD)s/root/cache -mindepth 1 -delete;' 486 'fi' % {'ANDROID_ROOT_WILDCARD': _CONTAINER_INSTANCE_ROOT_WILDCARD}) 487 488 _stop_ui(self._remote_proxy) 489 try: 490 _remount_rootfs_as_writable(self._remote_proxy) 491 except subprocess.CalledProcessError: 492 if not _disable_rootfs_verification(self._force, self._remote_proxy): 493 raise 494 _stop_ui(self._remote_proxy) 495 # Try to remount rootfs as writable. Bail out if it fails this time. 496 _remount_rootfs_as_writable(self._remote_proxy) 497 try: 498 self._remote_proxy.check_call('\n'.join([ 499 # Delete the image file if it is a symlink. 500 'test -L %(_ANDROID_ROOT)s/system.raw.img && ' 501 ' rm %(_ANDROID_ROOT)s/system.raw.img', 502 ]) % {'_ANDROID_ROOT': _ANDROID_ROOT}) 503 except Exception: 504 # Not a symlink. 505 pass 506 if self._push_to_stateful: 507 self._remote_proxy.check_call('\n'.join([ 508 # Create the destination directory in the stateful partition. 509 'mkdir -p %(_ANDROID_ROOT_STATEFUL)s', 510 ]) % {'_ANDROID_ROOT_STATEFUL': _ANDROID_ROOT_STATEFUL}) 511 512 def __exit__(self, exc_type, exc_value, traceback): 513 if self._push_to_stateful: 514 # Push the image to _ANDROID_ROOT_STATEFUL instead of _ANDROID_ROOT. 515 # Create a symlink so that arc-system-mount can handle it. 516 self._remote_proxy.check_call('\n'.join([ 517 'ln -sf %(_ANDROID_ROOT_STATEFUL)s/system.raw.img ' 518 ' %(_ANDROID_ROOT)s/system.raw.img', 519 ]) % {'_ANDROID_ROOT': _ANDROID_ROOT, 520 '_ANDROID_ROOT_STATEFUL': _ANDROID_ROOT_STATEFUL}) 521 522 if self._is_selinux_policy_updated: 523 logging.info('*** SELinux policy updated. ***') 524 else: 525 logging.info('*** SELinux policy is not updated. Restarting ui. ***') 526 try: 527 self._remote_proxy.check_call('\n'.join([ 528 # Make the whole invocation fail if any individual command does. 529 'set -e', 530 531 # Remount the root file system to readonly. 532 'mount -o remount,ro /', 533 534 # Restart UI. 535 'start ui', 536 537 # Mount the updated {system,vendor}.raw.img. This will also trigger 538 # android-ureadahead once it's done and should remove the packfile. 539 'start arc-system-mount', 540 ])) 541 return 542 except Exception: 543 # The above commands are just an optimization to avoid having to reboot 544 # every single time an image is pushed, which saves 6-10s. If any of 545 # them fail, the only safe thing to do is reboot the device. 546 logging.exception('Failed to cleanly restart ui, fall back to reboot') 547 548 logging.info('*** Reboot required. ***') 549 try: 550 self._remote_proxy.check_call('reboot') 551 except Exception: 552 if exc_type is None: 553 raise 554 # If the body block of a with statement also raises an error, here we 555 # just log the exception, so that the main exception will be propagated to 556 # the caller properly. 557 logging.exception('Failed to reboot the device') 558 559 560 class PreserveTimestamps(object): 561 """Context object to modify a file but preserve the original timestamp.""" 562 def __init__(self, path): 563 self.path = path 564 self._original_timestamp = None 565 566 def __enter__(self): 567 # Save the original timestamp 568 self._original_timestamp = os.stat(self.path) 569 return self 570 571 def __exit__(self, exception_type, exception_value, traceback): 572 # Apply the original timestamp 573 os.utime(self.path, (self._original_timestamp.st_atime, 574 self._original_timestamp.st_mtime)) 575 576 577 def _extract_artifact(simg2img, out_dir, filename): 578 with zipfile.ZipFile(filename, 'r') as z: 579 z.extract('system.img', out_dir) 580 z.extract('vendor.img', out_dir) 581 # Note that the same simg2img conversion is performed again for system.img 582 # later, but the extra run is acceptable (<2s). If this is important, we 583 # could try to change the program flow. 584 simg2img.convert(os.path.join(out_dir, 'system.img'), 585 os.path.join(out_dir, 'system.raw.img')) 586 # Extract the SELinux policy. 587 with TemporaryDirectory() as mnt_dir: 588 with MountWrapper(os.path.join(out_dir, 'system.raw.img'), 589 mnt_dir.name): 590 os.makedirs(os.path.join(out_dir, 'root')) 591 shutil.copyfile(os.path.join(mnt_dir.name, 'sepolicy'), 592 os.path.join(out_dir, 'root', 'sepolicy')) 593 shutil.copyfile(os.path.join(mnt_dir.name, 'system', 'build.prop'), 594 os.path.join(out_dir, 'build.prop')) 595 596 597 def _make_tempdir_deleted_on_exit(): 598 d = tempfile.mkdtemp() 599 atexit.register(shutil.rmtree, d, ignore_errors=True) 600 return d 601 602 603 def _detect_cert_inconsistency(remote_proxy, new_variant, dryrun): 604 """Prompt to ask for deleting data based on detected situation (best effort). 605 606 Detection is only accurate for active session, so it won't fix other profiles. 607 608 As GMS apps are signed with different key between user and non-user build, 609 the container won't run correctly if old key has been registered in /data. 610 """ 611 if dryrun: 612 return False 613 614 # Get current build variant on device. 615 cmd = 'grep %s %s' % (_RO_BUILD_TYPE, 616 os.path.join(_CONTAINER_ROOT, 'system/build.prop')) 617 try: 618 line = remote_proxy.check_output(cmd).strip() 619 except subprocess.CalledProcessError: 620 # Catch any error to avoid blocking the push. 621 logging.exception('Failed to inspect build property remotely') 622 return False 623 device_variant = line[len(_RO_BUILD_TYPE):] 624 625 device_apk_key = _APK_KEY_UNKNOWN 626 try: 627 device_apk_key = _get_remote_device_apk_key(remote_proxy) 628 except Exception as e: 629 logging.warning('There was an error getting the remote device APK ' 630 'key signature %s. Assuming APK key signature is ' 631 '\'unknown\'', e) 632 633 logging.debug('device apk key: %s; build variant: %s -> %s', device_apk_key, 634 device_variant, new_variant) 635 636 # GMS signature in /data is inconsistent with the new build. 637 is_inconsistent = ( 638 (device_apk_key == _APK_KEY_RELEASE and new_variant != 'user') or 639 (device_apk_key == _APK_KEY_DEBUG and new_variant == 'user')) 640 641 if is_inconsistent: 642 new_apk_key = _APK_KEY_RELEASE if new_variant == 'user' else _APK_KEY_DEBUG 643 return boolean_prompt( 644 'Detected apk signature change (%s -> %s[%s]) on current user. Delete ' 645 '/data and /cache?' % (device_apk_key, new_apk_key, new_variant), 646 default=True) 647 648 # Switching from/to user build. 649 if (device_variant == 'user') != (new_variant == 'user'): 650 logging.warn('\n\n** You are switching build variant (%s -> %s). If you ' 651 'have ever run with the old image, make sure to wipe out ' 652 '/data first before starting the container. **\n', 653 device_variant, new_variant) 654 return False 655 656 657 def _get_remote_device_apk_key(remote_proxy): 658 """Retrieves the APK key signature of the remote test device. 659 660 Args: 661 remote_proxy: RemoteProxy instance for the remote test device. 662 """ 663 remote_packages_xml = os.path.join(_CONTAINER_INSTANCE_ROOT_WILDCARD, 664 'root/data/system/packages.xml') 665 with TemporaryDirectory() as tmp_dir: 666 host_packages_xml = os.path.join(tmp_dir.name, 'packages.xml') 667 remote_proxy.pull(remote_packages_xml, host_packages_xml) 668 return _get_apk_key_from_xml(host_packages_xml) 669 670 671 def _get_apk_key_from_xml(xml_file): 672 """Parses |xml_file| to determine the APK key signature. 673 674 Args: 675 xml_file: The XML file to parse. 676 """ 677 if not os.path.exists(xml_file): 678 logging.warning('XML file doesn\'t exist: %s' % xml_file) 679 return _APK_KEY_UNKNOWN 680 681 root = ElementTree.parse(xml_file).getroot() 682 gms_core_elements = root.findall('package[@name=\'%s\']' 683 % _GMS_CORE_PACKAGE_NAME) 684 assert len(gms_core_elements) == 1, ('Invalid number of GmsCore package ' 685 'elements. Expected: 1 Actual: %d' 686 % len(gms_core_elements)) 687 gms_core_element = gms_core_elements[0] 688 sigs_element = gms_core_element.find('sigs') 689 assert sigs_element, ('Unable to find the |sigs| tag under the GmsCore ' 690 'package tag.') 691 sigs_count_attribute = int(sigs_element.get('count')) 692 assert sigs_count_attribute == 1, ('Invalid signature count. Expected: 1 ' 693 'Actual: %d' % sigs_count_attribute) 694 cert_element = sigs_element.find('cert') 695 gms_core_cert_index = int(cert_element.get('index', -1)) 696 logging.debug("GmsCore cert index: %d" % gms_core_cert_index) 697 if gms_core_cert_index == -1: 698 logging.warning('Invalid cert index (%d)' % gms_core_cert_index) 699 return _APK_KEY_UNKNOWN 700 701 cert_key = cert_element.get('key') 702 if cert_key: 703 return _get_android_key_type_from_cert_key(cert_key) 704 705 # The GmsCore package element for |cert| contains the cert index, but not the 706 # cert key. Find its the matching cert key. 707 for cert_element in root.findall('package/sigs/cert'): 708 cert_index = int(cert_element.get('index')) 709 cert_key = cert_element.get('key') 710 if cert_key and cert_index == gms_core_cert_index: 711 return _get_android_key_type_from_cert_key(cert_key) 712 logging.warning ('Unable to find a cert key matching index %d' % cert_index) 713 return _APK_KEY_UNKNOWN 714 715 716 def _get_android_key_type_from_cert_key(cert_key): 717 """Returns |_APK_KEY_RELEASE| if |cert_key| contains the Android release key 718 signature substring, otherwise it returns |_APK_KEY_DEBUG|.""" 719 if _ANDROID_REL_KEY_SIGNATURE_SUBSTRING in cert_key: 720 return _APK_KEY_RELEASE 721 else: 722 return _APK_KEY_DEBUG 723 724 725 def _find_build_property(line, build_property_name): 726 """Returns the value that matches |build_property_name| in |line|.""" 727 if line.startswith(build_property_name): 728 return line[len(build_property_name):].strip() 729 return None 730 731 732 class BaseProvider(object): 733 """Base class of image provider. 734 735 Subclass should provide a directory with images in it. 736 """ 737 738 def __init__(self): 739 self._build_variant = None 740 self._build_version_sdk = None 741 742 def prepare(self): 743 """Subclass should prepare image in its implementation. 744 745 Subclass must return the (image directory, product, fingerprint) tuple. 746 Product is a string like "cheets_arm". Fingerprint is the string that 747 will be updated to CHROMEOS_ARC_VERSION in /etc/lsb-release. 748 """ 749 raise NotImplementedError() 750 751 def get_build_variant(self): 752 """ Returns the extracted build variant.""" 753 return self._build_variant 754 755 def get_build_version_sdk(self): 756 """ Returns the extracted Android SDK version.""" 757 return self._build_version_sdk 758 759 def read_build_prop_file(self, build_prop_file, remove_file=True): 760 """ Reads the specified build property file, and extracts the 761 "ro.build.variant" and "ro.build.version.sdk" fields. This method optionally 762 deletes |build_prop_file| when done 763 764 Args: 765 build_prop_file: The fully qualified path to the build.prop file. 766 remove_file: Removes the |build_prop_file| when done. (default=True) 767 """ 768 logging.debug('Reading build prop file: %s', build_prop_file) 769 with open(build_prop_file, 'r') as f: 770 for line in f: 771 if self._build_version_sdk is None: 772 value = _find_build_property(line, _RO_BUILD_VERSION_SDK) 773 if value is not None: 774 self._build_version_sdk = int(value) 775 if self._build_variant is None: 776 value = _find_build_property(line, _RO_BUILD_TYPE) 777 if value is not None: 778 self._build_variant = value 779 if self._build_variant and self._build_version_sdk: 780 break 781 if remove_file: 782 logging.info('Deleting prop file: %s...', build_prop_file) 783 os.remove(build_prop_file) 784 785 786 class LocalPrebuiltProvider(BaseProvider): 787 """A provider that provides prebuilt image from a local file.""" 788 789 def __init__(self, prebuilt_file, simg2img): 790 super(LocalPrebuiltProvider, self).__init__() 791 self._prebuilt_file = prebuilt_file 792 self._simg2img = simg2img 793 794 def prepare(self): 795 out_dir = _make_tempdir_deleted_on_exit() 796 _extract_artifact(self._simg2img, out_dir, self._prebuilt_file) 797 798 build_prop_file = os.path.join(out_dir, 'build.prop') 799 self.read_build_prop_file(build_prop_file) 800 if self._build_variant is None: 801 self._build_variant = 'user' # default to non-eng 802 803 m = re.match(r'(cheets_\w+)-img-P?\d+\.zip', 804 os.path.basename(self._prebuilt_file)) 805 if not m: 806 sys.exit('Unrecognized file name of prebuilt image archive.') 807 product = m.group(1) 808 809 fingerprint = os.path.splitext(os.path.basename(self._prebuilt_file))[0] 810 return out_dir, product, fingerprint 811 812 813 class LocalBuildProvider(BaseProvider): 814 """A provider that provides local built image.""" 815 816 def __init__(self, build_fingerprint, skip_build_prop_update): 817 super(LocalBuildProvider, self).__init__() 818 self._build_fingerprint = build_fingerprint 819 self._skip_build_prop_update = skip_build_prop_update 820 expected_env = ('TARGET_BUILD_VARIANT', 'TARGET_PRODUCT', 'OUT') 821 if not all(var in os.environ for var in expected_env): 822 sys.exit('Did you run lunch?') 823 self._build_variant = os.environ.get('TARGET_BUILD_VARIANT') 824 self._target_product = os.environ.get('TARGET_PRODUCT') 825 self._out_dir = os.environ.get('OUT') 826 827 def prepare(self): 828 # Use build fingerprint if set. Otherwise, read it from the text file. 829 build_fingerprint = self._build_fingerprint 830 if not build_fingerprint: 831 fingerprint_filepath = os.path.join(self._out_dir, 832 'build_fingerprint.txt') 833 if os.path.isfile(fingerprint_filepath): 834 with open(fingerprint_filepath) as f: 835 build_fingerprint = f.read().strip().replace('/', '_') 836 837 # Find the absolute path of build.prop. 838 build_prop_file = os.path.join(self._out_dir, 'system/build.prop') 839 if not self._skip_build_prop_update: 840 self._update_local_build_prop_file(build_prop_file) 841 self.read_build_prop_file(build_prop_file, False) 842 return self._out_dir, self._target_product, build_fingerprint 843 844 def _update_local_build_prop_file(self, build_prop_file): 845 """Updates build.prop of the local prebuilt image.""" 846 847 if not build_prop_file: 848 logging.warning('Skipping. build_prop_file was not specified.') 849 return 850 # Create the generic device name by extracting the architecture type 851 # from the target product. 852 generic_device = _GENERIC_DEVICE % dict( 853 arch=lib.util.get_product_arch(self._target_product)) 854 # Get the current value of ro.product.device in build.prop. 855 current_prop_value = lib.util.check_output('grep', _RO_PRODUCT_DEVICE, 856 build_prop_file).strip() 857 new_prop_value = '%s%s' % (_RO_PRODUCT_DEVICE, generic_device) 858 # If build.prop contains the new value, return. 859 if current_prop_value == new_prop_value: 860 logging.info('build.prop does not need to be updated.') 861 return 862 863 logging.info('Setting "%s" to "%s" in build.prop...', 864 _RO_PRODUCT_DEVICE, generic_device) 865 with PreserveTimestamps(build_prop_file) as f: 866 # Make the changes to build.prop 867 lib.util.check_call( 868 '/bin/sed', '-i', 869 r's/^\(%(_KEY)s\).*/\1%(_VALUE)s/' 870 % {'_KEY': _RO_PRODUCT_DEVICE, '_VALUE': generic_device}, 871 f.path) 872 873 logging.info('Recreating the system image with the updated build.prop ' + 874 'file...') 875 system_dir = os.path.join(self._out_dir, 'system') 876 system_image_info_file = os.path.join( 877 self._out_dir, 878 'obj/PACKAGING/systemimage_intermediates/system_image_info.txt') 879 system_image_file = os.path.join(self._out_dir, 'system.img') 880 with PreserveTimestamps(system_image_file) as f: 881 # Recreate system.img 882 lib.util.check_call( 883 './build/tools/releasetools/build_image.py', 884 system_dir, 885 system_image_info_file, 886 f.path, 887 system_dir) 888 889 890 class NullProvider(BaseProvider): 891 """ Provider used for dry runs """ 892 893 def __init__(self): 894 super(NullProvider, self).__init__() 895 self._build_variant = 'user' 896 self._build_version_sdk = 1 897 898 def prepare(self): 899 return ('<dir>', '<product>', '<fingerprint>') 900 901 902 def _parse_prebuilt(param): 903 m = re.search(r'^(cheets_(?:arm|x86))/(user|userdebug|eng)/(P?\d+)$', param) 904 if not m: 905 sys.exit('Invalid format of --use-prebuilt') 906 return m.group(1), m.group(2), m.group(3) 907 908 909 def _default_simg2img_path(): 910 # Automatically resolve simg2img path if possible. 911 if 'ANDROID_HOST_OUT' in os.environ: 912 return os.path.join(os.environ.get('ANDROID_HOST_OUT'), 'bin', 'simg2img') 913 path = os.path.join(_SCRIPT_DIR, 'simg2img') 914 if os.path.isfile(path): 915 return path 916 return None 917 918 919 def _resolve_args(args): 920 if not args.simg2img_path: 921 sys.exit('Cannot determine the path of simg2img') 922 923 924 def _parse_args(): 925 """Parses the arguments.""" 926 parser = argparse.ArgumentParser( 927 formatter_class=argparse.RawDescriptionHelpFormatter, 928 description='Push image to Chromebook', 929 epilog="""Examples: 930 931 To push from local build 932 $ %(prog)s <remote> 933 934 To push from Android build prebuilt 935 $ %(prog)s --use-prebuilt cheets_arm/eng/123456 <remote> 936 937 To push from local prebuilt 938 $ %(prog)s --use-prebuilt-file path/to/cheets_arm-img-123456.zip <remote> 939 """) 940 parser.add_argument( 941 '--push-vendor-image', action='store_true', help='Push vendor image') 942 parser.add_argument( 943 '--use-prebuilt', metavar='PRODUCT/BUILD_VARIANT/BUILD_ID', 944 type=_parse_prebuilt, 945 help='Push prebuilt image instead. Example value: cheets_arm/eng/123456') 946 parser.add_argument( 947 '--use-prebuilt-file', dest='prebuilt_file', metavar='<path>', 948 help='The downloaded image path') 949 parser.add_argument( 950 '--build-fingerprint', default=os.environ.get('BUILD_FINGERPRINT'), 951 help='If set, embed this fingerprint data to the /etc/lsb-release ' 952 'as CHROMEOS_ARC_VERSION value.') 953 parser.add_argument( 954 '--dryrun', action='store_true', 955 help='Do not execute subprocesses.') 956 parser.add_argument( 957 '--loglevel', default='INFO', 958 choices=('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'), 959 help='Logging level.') 960 parser.add_argument( 961 '--simg2img-path', default=_default_simg2img_path(), 962 help='Executable path of simg2img') 963 parser.add_argument( 964 '--force', action='store_true', 965 help=('Skip all prompts (i.e., for disabling of rootfs verification). ' 966 'This may result in the target machine being rebooted')) 967 parser.add_argument( 968 '--try-clobber-data', action='store_true', 969 help='If currently logged in, also clobber /data and /cache') 970 parser.add_argument( 971 '--skip_build_prop_update', action='store_true', 972 help=('Do not change ro.product.device to "generic_cheets" for local ' 973 'builds')) 974 parser.add_argument( 975 '--push-to-stateful-partition', action='store_true', 976 help=('Place the system.raw.img on the stateful partition instead of /. ' 977 'This is always used for -eng builds since they do not fit on /.')) 978 parser.add_argument( 979 'remote', 980 help=('The target test device. This is passed to ssh command etc., ' 981 'so IP or the name registered in your .ssh/config file can be ' 982 'accepted.')) 983 args = parser.parse_args() 984 985 _resolve_args(args) 986 return args 987 988 989 def main(): 990 # Set up arguments. 991 args = _parse_args() 992 logging.basicConfig(level=getattr(logging, args.loglevel)) 993 994 simg2img = Simg2img(args.simg2img_path, args.dryrun) 995 996 # Prepare local source. A preparer is responsible to return an directory that 997 # contains necessary files to push. It also needs to return metadata like 998 # product (e.g. cheets_arm) and a build fingerprint. 999 if args.dryrun: 1000 provider = NullProvider() 1001 elif args.prebuilt_file: 1002 provider = LocalPrebuiltProvider(args.prebuilt_file, simg2img) 1003 else: 1004 provider = LocalBuildProvider(args.build_fingerprint, 1005 args.skip_build_prop_update) 1006 1007 # Actually prepare the files to push. 1008 out, product, fingerprint = provider.prepare() 1009 1010 # Update the image. 1011 remote_proxy = RemoteProxy(args.remote, args.dryrun) 1012 _verify_android_sdk_version(remote_proxy, provider, args.dryrun) 1013 _verify_machine_arch(remote_proxy, product, args.dryrun) 1014 1015 if args.try_clobber_data: 1016 clobber_data = True 1017 else: 1018 clobber_data = _detect_cert_inconsistency( 1019 remote_proxy, provider.get_build_variant(), args.dryrun) 1020 1021 logging.info('Converting images to raw images...') 1022 (large_image_list, image_list) = _convert_images( 1023 simg2img, out, args.push_vendor_image) 1024 1025 is_selinux_policy_updated = _is_selinux_policy_updated(remote_proxy, out, 1026 args.dryrun) 1027 push_to_stateful = (args.push_to_stateful_partition or 1028 'eng' == provider.get_build_variant()) 1029 1030 with ImageUpdateMode(remote_proxy, is_selinux_policy_updated, 1031 push_to_stateful, clobber_data, args.force): 1032 is_debuggable = 'user' != provider.get_build_variant() 1033 remote_proxy.check_call(' '.join([ 1034 '/bin/sed', '-i', 1035 r'"s/^\(env ANDROID_DEBUGGABLE=\).*/\1%(_IS_DEBUGGABLE)d/"', 1036 '/etc/init/arc-setup.conf' 1037 ]) % {'_IS_DEBUGGABLE': is_debuggable}) 1038 1039 logging.info('Syncing image files to ChromeOS...') 1040 if large_image_list: 1041 remote_proxy.sync(large_image_list, 1042 _ANDROID_ROOT_STATEFUL if push_to_stateful else 1043 _ANDROID_ROOT) 1044 if image_list: 1045 remote_proxy.sync(image_list, _ANDROID_ROOT) 1046 _update_build_fingerprint(remote_proxy, fingerprint) 1047 if is_selinux_policy_updated: 1048 _update_selinux_policy(remote_proxy, out) 1049 1050 1051 if __name__ == '__main__': 1052 main() 1053