1 # Copyright (C) 2008 The Android Open Source Project 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import copy 16 import errno 17 import getopt 18 import getpass 19 import imp 20 import os 21 import platform 22 import re 23 import shlex 24 import shutil 25 import subprocess 26 import sys 27 import tempfile 28 import threading 29 import time 30 import zipfile 31 32 try: 33 from hashlib import sha1 as sha1 34 except ImportError: 35 from sha import sha as sha1 36 37 # missing in Python 2.4 and before 38 if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41 class Options(object): pass 42 OPTIONS = Options() 43 OPTIONS.search_path = "out/host/linux-x86" 44 OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45 OPTIONS.extra_signapk_args = [] 46 OPTIONS.java_path = "java" # Use the one on the path by default. 47 OPTIONS.public_key_suffix = ".x509.pem" 48 OPTIONS.private_key_suffix = ".pk8" 49 OPTIONS.verbose = False 50 OPTIONS.tempfiles = [] 51 OPTIONS.device_specific = None 52 OPTIONS.extras = {} 53 OPTIONS.info_dict = None 54 55 56 # Values for "certificate" in apkcerts that mean special things. 57 SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60 class ExternalError(RuntimeError): pass 61 62 63 def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71 def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87 def LoadInfoDict(zip): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 d = {} 92 try: 93 for line in zip.read("META/misc_info.txt").split("\n"): 94 line = line.strip() 95 if not line or line.startswith("#"): continue 96 k, v = line.split("=", 1) 97 d[k] = v 98 except KeyError: 99 # ok if misc_info.txt doesn't exist 100 pass 101 102 # backwards compatibility: These values used to be in their own 103 # files. Look for them, in case we're processing an old 104 # target_files zip. 105 106 if "mkyaffs2_extra_flags" not in d: 107 try: 108 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 109 except KeyError: 110 # ok if flags don't exist 111 pass 112 113 if "recovery_api_version" not in d: 114 try: 115 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 116 except KeyError: 117 raise ValueError("can't find recovery API version in input target-files") 118 119 if "tool_extensions" not in d: 120 try: 121 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 122 except KeyError: 123 # ok if extensions don't exist 124 pass 125 126 if "fstab_version" not in d: 127 d["fstab_version"] = "1" 128 129 try: 130 data = zip.read("META/imagesizes.txt") 131 for line in data.split("\n"): 132 if not line: continue 133 name, value = line.split(" ", 1) 134 if not value: continue 135 if name == "blocksize": 136 d[name] = value 137 else: 138 d[name + "_size"] = value 139 except KeyError: 140 pass 141 142 def makeint(key): 143 if key in d: 144 d[key] = int(d[key], 0) 145 146 makeint("recovery_api_version") 147 makeint("blocksize") 148 makeint("system_size") 149 makeint("userdata_size") 150 makeint("cache_size") 151 makeint("recovery_size") 152 makeint("boot_size") 153 makeint("fstab_version") 154 155 d["fstab"] = LoadRecoveryFSTab(zip, d["fstab_version"]) 156 d["build.prop"] = LoadBuildProp(zip) 157 return d 158 159 def LoadBuildProp(zip): 160 try: 161 data = zip.read("SYSTEM/build.prop") 162 except KeyError: 163 print "Warning: could not find SYSTEM/build.prop in %s" % zip 164 data = "" 165 166 d = {} 167 for line in data.split("\n"): 168 line = line.strip() 169 if not line or line.startswith("#"): continue 170 name, value = line.split("=", 1) 171 d[name] = value 172 return d 173 174 def LoadRecoveryFSTab(zip, fstab_version): 175 class Partition(object): 176 pass 177 178 try: 179 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 180 except KeyError: 181 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip 182 data = "" 183 184 if fstab_version == 1: 185 d = {} 186 for line in data.split("\n"): 187 line = line.strip() 188 if not line or line.startswith("#"): continue 189 pieces = line.split() 190 if not (3 <= len(pieces) <= 4): 191 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 192 193 p = Partition() 194 p.mount_point = pieces[0] 195 p.fs_type = pieces[1] 196 p.device = pieces[2] 197 p.length = 0 198 options = None 199 if len(pieces) >= 4: 200 if pieces[3].startswith("/"): 201 p.device2 = pieces[3] 202 if len(pieces) >= 5: 203 options = pieces[4] 204 else: 205 p.device2 = None 206 options = pieces[3] 207 else: 208 p.device2 = None 209 210 if options: 211 options = options.split(",") 212 for i in options: 213 if i.startswith("length="): 214 p.length = int(i[7:]) 215 else: 216 print "%s: unknown option \"%s\"" % (p.mount_point, i) 217 218 d[p.mount_point] = p 219 220 elif fstab_version == 2: 221 d = {} 222 for line in data.split("\n"): 223 line = line.strip() 224 if not line or line.startswith("#"): continue 225 pieces = line.split() 226 if len(pieces) != 5: 227 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 228 229 # Ignore entries that are managed by vold 230 options = pieces[4] 231 if "voldmanaged=" in options: continue 232 233 # It's a good line, parse it 234 p = Partition() 235 p.device = pieces[0] 236 p.mount_point = pieces[1] 237 p.fs_type = pieces[2] 238 p.device2 = None 239 p.length = 0 240 241 options = options.split(",") 242 for i in options: 243 if i.startswith("length="): 244 p.length = int(i[7:]) 245 else: 246 # Ignore all unknown options in the unified fstab 247 continue 248 249 d[p.mount_point] = p 250 251 else: 252 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 253 254 return d 255 256 257 def DumpInfoDict(d): 258 for k, v in sorted(d.items()): 259 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 260 261 def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 262 """Take a kernel, cmdline, and ramdisk directory from the input (in 263 'sourcedir'), and turn them into a boot image. Return the image 264 data, or None if sourcedir does not appear to contains files for 265 building the requested image.""" 266 267 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 268 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 269 return None 270 271 if info_dict is None: 272 info_dict = OPTIONS.info_dict 273 274 ramdisk_img = tempfile.NamedTemporaryFile() 275 img = tempfile.NamedTemporaryFile() 276 277 if os.access(fs_config_file, os.F_OK): 278 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 279 else: 280 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 281 p1 = Run(cmd, stdout=subprocess.PIPE) 282 p2 = Run(["minigzip"], 283 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 284 285 p2.wait() 286 p1.wait() 287 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 288 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 289 290 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 291 292 fn = os.path.join(sourcedir, "cmdline") 293 if os.access(fn, os.F_OK): 294 cmd.append("--cmdline") 295 cmd.append(open(fn).read().rstrip("\n")) 296 297 fn = os.path.join(sourcedir, "base") 298 if os.access(fn, os.F_OK): 299 cmd.append("--base") 300 cmd.append(open(fn).read().rstrip("\n")) 301 302 fn = os.path.join(sourcedir, "pagesize") 303 if os.access(fn, os.F_OK): 304 cmd.append("--pagesize") 305 cmd.append(open(fn).read().rstrip("\n")) 306 307 args = info_dict.get("mkbootimg_args", None) 308 if args and args.strip(): 309 cmd.extend(args.split()) 310 311 cmd.extend(["--ramdisk", ramdisk_img.name, 312 "--output", img.name]) 313 314 p = Run(cmd, stdout=subprocess.PIPE) 315 p.communicate() 316 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 317 os.path.basename(sourcedir),) 318 319 img.seek(os.SEEK_SET, 0) 320 data = img.read() 321 322 ramdisk_img.close() 323 img.close() 324 325 return data 326 327 328 def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 329 info_dict=None): 330 """Return a File object (with name 'name') with the desired bootable 331 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 332 'prebuilt_name', otherwise construct it from the source files in 333 'unpack_dir'/'tree_subdir'.""" 334 335 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 336 if os.path.exists(prebuilt_path): 337 print "using prebuilt %s..." % (prebuilt_name,) 338 return File.FromLocalFile(name, prebuilt_path) 339 else: 340 print "building image from target_files %s..." % (tree_subdir,) 341 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 342 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 343 os.path.join(unpack_dir, fs_config), 344 info_dict)) 345 346 347 def UnzipTemp(filename, pattern=None): 348 """Unzip the given archive into a temporary directory and return the name. 349 350 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 351 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 352 353 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 354 main file), open for reading. 355 """ 356 357 tmp = tempfile.mkdtemp(prefix="targetfiles-") 358 OPTIONS.tempfiles.append(tmp) 359 360 def unzip_to_dir(filename, dirname): 361 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 362 if pattern is not None: 363 cmd.append(pattern) 364 p = Run(cmd, stdout=subprocess.PIPE) 365 p.communicate() 366 if p.returncode != 0: 367 raise ExternalError("failed to unzip input target-files \"%s\"" % 368 (filename,)) 369 370 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 371 if m: 372 unzip_to_dir(m.group(1), tmp) 373 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 374 filename = m.group(1) 375 else: 376 unzip_to_dir(filename, tmp) 377 378 return tmp, zipfile.ZipFile(filename, "r") 379 380 381 def GetKeyPasswords(keylist): 382 """Given a list of keys, prompt the user to enter passwords for 383 those which require them. Return a {key: password} dict. password 384 will be None if the key has no password.""" 385 386 no_passwords = [] 387 need_passwords = [] 388 key_passwords = {} 389 devnull = open("/dev/null", "w+b") 390 for k in sorted(keylist): 391 # We don't need a password for things that aren't really keys. 392 if k in SPECIAL_CERT_STRINGS: 393 no_passwords.append(k) 394 continue 395 396 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 397 "-inform", "DER", "-nocrypt"], 398 stdin=devnull.fileno(), 399 stdout=devnull.fileno(), 400 stderr=subprocess.STDOUT) 401 p.communicate() 402 if p.returncode == 0: 403 # Definitely an unencrypted key. 404 no_passwords.append(k) 405 else: 406 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 407 "-inform", "DER", "-passin", "pass:"], 408 stdin=devnull.fileno(), 409 stdout=devnull.fileno(), 410 stderr=subprocess.PIPE) 411 stdout, stderr = p.communicate() 412 if p.returncode == 0: 413 # Encrypted key with empty string as password. 414 key_passwords[k] = '' 415 elif stderr.startswith('Error decrypting key'): 416 # Definitely encrypted key. 417 # It would have said "Error reading key" if it didn't parse correctly. 418 need_passwords.append(k) 419 else: 420 # Potentially, a type of key that openssl doesn't understand. 421 # We'll let the routines in signapk.jar handle it. 422 no_passwords.append(k) 423 devnull.close() 424 425 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 426 key_passwords.update(dict.fromkeys(no_passwords, None)) 427 return key_passwords 428 429 430 def SignFile(input_name, output_name, key, password, align=None, 431 whole_file=False): 432 """Sign the input_name zip/jar/apk, producing output_name. Use the 433 given key and password (the latter may be None if the key does not 434 have a password. 435 436 If align is an integer > 1, zipalign is run to align stored files in 437 the output zip on 'align'-byte boundaries. 438 439 If whole_file is true, use the "-w" option to SignApk to embed a 440 signature that covers the whole file in the archive comment of the 441 zip file. 442 """ 443 444 if align == 0 or align == 1: 445 align = None 446 447 if align: 448 temp = tempfile.NamedTemporaryFile() 449 sign_name = temp.name 450 else: 451 sign_name = output_name 452 453 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 454 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 455 cmd.extend(OPTIONS.extra_signapk_args) 456 if whole_file: 457 cmd.append("-w") 458 cmd.extend([key + OPTIONS.public_key_suffix, 459 key + OPTIONS.private_key_suffix, 460 input_name, sign_name]) 461 462 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 463 if password is not None: 464 password += "\n" 465 p.communicate(password) 466 if p.returncode != 0: 467 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 468 469 if align: 470 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 471 p.communicate() 472 if p.returncode != 0: 473 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 474 temp.close() 475 476 477 def CheckSize(data, target, info_dict): 478 """Check the data string passed against the max size limit, if 479 any, for the given target. Raise exception if the data is too big. 480 Print a warning if the data is nearing the maximum size.""" 481 482 if target.endswith(".img"): target = target[:-4] 483 mount_point = "/" + target 484 485 if info_dict["fstab"]: 486 if mount_point == "/userdata": mount_point = "/data" 487 p = info_dict["fstab"][mount_point] 488 fs_type = p.fs_type 489 device = p.device 490 if "/" in device: 491 device = device[device.rfind("/")+1:] 492 limit = info_dict.get(device + "_size", None) 493 if not fs_type or not limit: return 494 495 if fs_type == "yaffs2": 496 # image size should be increased by 1/64th to account for the 497 # spare area (64 bytes per 2k page) 498 limit = limit / 2048 * (2048+64) 499 size = len(data) 500 pct = float(size) * 100.0 / limit 501 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 502 if pct >= 99.0: 503 raise ExternalError(msg) 504 elif pct >= 95.0: 505 print 506 print " WARNING: ", msg 507 print 508 elif OPTIONS.verbose: 509 print " ", msg 510 511 512 def ReadApkCerts(tf_zip): 513 """Given a target_files ZipFile, parse the META/apkcerts.txt file 514 and return a {package: cert} dict.""" 515 certmap = {} 516 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 517 line = line.strip() 518 if not line: continue 519 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 520 r'private_key="(.*)"$', line) 521 if m: 522 name, cert, privkey = m.groups() 523 public_key_suffix_len = len(OPTIONS.public_key_suffix) 524 private_key_suffix_len = len(OPTIONS.private_key_suffix) 525 if cert in SPECIAL_CERT_STRINGS and not privkey: 526 certmap[name] = cert 527 elif (cert.endswith(OPTIONS.public_key_suffix) and 528 privkey.endswith(OPTIONS.private_key_suffix) and 529 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 530 certmap[name] = cert[:-public_key_suffix_len] 531 else: 532 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 533 return certmap 534 535 536 COMMON_DOCSTRING = """ 537 -p (--path) <dir> 538 Prepend <dir>/bin to the list of places to search for binaries 539 run by this script, and expect to find jars in <dir>/framework. 540 541 -s (--device_specific) <file> 542 Path to the python module containing device-specific 543 releasetools code. 544 545 -x (--extra) <key=value> 546 Add a key/value pair to the 'extras' dict, which device-specific 547 extension code may look at. 548 549 -v (--verbose) 550 Show command lines being executed. 551 552 -h (--help) 553 Display this usage message and exit. 554 """ 555 556 def Usage(docstring): 557 print docstring.rstrip("\n") 558 print COMMON_DOCSTRING 559 560 561 def ParseOptions(argv, 562 docstring, 563 extra_opts="", extra_long_opts=(), 564 extra_option_handler=None): 565 """Parse the options in argv and return any arguments that aren't 566 flags. docstring is the calling module's docstring, to be displayed 567 for errors and -h. extra_opts and extra_long_opts are for flags 568 defined by the caller, which are processed by passing them to 569 extra_option_handler.""" 570 571 try: 572 opts, args = getopt.getopt( 573 argv, "hvp:s:x:" + extra_opts, 574 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 575 "java_path=", "public_key_suffix=", "private_key_suffix=", 576 "device_specific=", "extra="] + 577 list(extra_long_opts)) 578 except getopt.GetoptError, err: 579 Usage(docstring) 580 print "**", str(err), "**" 581 sys.exit(2) 582 583 path_specified = False 584 585 for o, a in opts: 586 if o in ("-h", "--help"): 587 Usage(docstring) 588 sys.exit() 589 elif o in ("-v", "--verbose"): 590 OPTIONS.verbose = True 591 elif o in ("-p", "--path"): 592 OPTIONS.search_path = a 593 elif o in ("--signapk_path",): 594 OPTIONS.signapk_path = a 595 elif o in ("--extra_signapk_args",): 596 OPTIONS.extra_signapk_args = shlex.split(a) 597 elif o in ("--java_path",): 598 OPTIONS.java_path = a 599 elif o in ("--public_key_suffix",): 600 OPTIONS.public_key_suffix = a 601 elif o in ("--private_key_suffix",): 602 OPTIONS.private_key_suffix = a 603 elif o in ("-s", "--device_specific"): 604 OPTIONS.device_specific = a 605 elif o in ("-x", "--extra"): 606 key, value = a.split("=", 1) 607 OPTIONS.extras[key] = value 608 else: 609 if extra_option_handler is None or not extra_option_handler(o, a): 610 assert False, "unknown option \"%s\"" % (o,) 611 612 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 613 os.pathsep + os.environ["PATH"]) 614 615 return args 616 617 618 def Cleanup(): 619 for i in OPTIONS.tempfiles: 620 if os.path.isdir(i): 621 shutil.rmtree(i) 622 else: 623 os.remove(i) 624 625 626 class PasswordManager(object): 627 def __init__(self): 628 self.editor = os.getenv("EDITOR", None) 629 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 630 631 def GetPasswords(self, items): 632 """Get passwords corresponding to each string in 'items', 633 returning a dict. (The dict may have keys in addition to the 634 values in 'items'.) 635 636 Uses the passwords in $ANDROID_PW_FILE if available, letting the 637 user edit that file to add more needed passwords. If no editor is 638 available, or $ANDROID_PW_FILE isn't define, prompts the user 639 interactively in the ordinary way. 640 """ 641 642 current = self.ReadFile() 643 644 first = True 645 while True: 646 missing = [] 647 for i in items: 648 if i not in current or not current[i]: 649 missing.append(i) 650 # Are all the passwords already in the file? 651 if not missing: return current 652 653 for i in missing: 654 current[i] = "" 655 656 if not first: 657 print "key file %s still missing some passwords." % (self.pwfile,) 658 answer = raw_input("try to edit again? [y]> ").strip() 659 if answer and answer[0] not in 'yY': 660 raise RuntimeError("key passwords unavailable") 661 first = False 662 663 current = self.UpdateAndReadFile(current) 664 665 def PromptResult(self, current): 666 """Prompt the user to enter a value (password) for each key in 667 'current' whose value is fales. Returns a new dict with all the 668 values. 669 """ 670 result = {} 671 for k, v in sorted(current.iteritems()): 672 if v: 673 result[k] = v 674 else: 675 while True: 676 result[k] = getpass.getpass("Enter password for %s key> " 677 % (k,)).strip() 678 if result[k]: break 679 return result 680 681 def UpdateAndReadFile(self, current): 682 if not self.editor or not self.pwfile: 683 return self.PromptResult(current) 684 685 f = open(self.pwfile, "w") 686 os.chmod(self.pwfile, 0600) 687 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 688 f.write("# (Additional spaces are harmless.)\n\n") 689 690 first_line = None 691 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 692 sorted.sort() 693 for i, (_, k, v) in enumerate(sorted): 694 f.write("[[[ %s ]]] %s\n" % (v, k)) 695 if not v and first_line is None: 696 # position cursor on first line with no password. 697 first_line = i + 4 698 f.close() 699 700 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 701 _, _ = p.communicate() 702 703 return self.ReadFile() 704 705 def ReadFile(self): 706 result = {} 707 if self.pwfile is None: return result 708 try: 709 f = open(self.pwfile, "r") 710 for line in f: 711 line = line.strip() 712 if not line or line[0] == '#': continue 713 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 714 if not m: 715 print "failed to parse password file: ", line 716 else: 717 result[m.group(2)] = m.group(1) 718 f.close() 719 except IOError, e: 720 if e.errno != errno.ENOENT: 721 print "error reading password file: ", str(e) 722 return result 723 724 725 def ZipWriteStr(zip, filename, data, perms=0644): 726 # use a fixed timestamp so the output is repeatable. 727 zinfo = zipfile.ZipInfo(filename=filename, 728 date_time=(2009, 1, 1, 0, 0, 0)) 729 zinfo.compress_type = zip.compression 730 zinfo.external_attr = perms << 16 731 zip.writestr(zinfo, data) 732 733 734 class DeviceSpecificParams(object): 735 module = None 736 def __init__(self, **kwargs): 737 """Keyword arguments to the constructor become attributes of this 738 object, which is passed to all functions in the device-specific 739 module.""" 740 for k, v in kwargs.iteritems(): 741 setattr(self, k, v) 742 self.extras = OPTIONS.extras 743 744 if self.module is None: 745 path = OPTIONS.device_specific 746 if not path: return 747 try: 748 if os.path.isdir(path): 749 info = imp.find_module("releasetools", [path]) 750 else: 751 d, f = os.path.split(path) 752 b, x = os.path.splitext(f) 753 if x == ".py": 754 f = b 755 info = imp.find_module(f, [d]) 756 self.module = imp.load_module("device_specific", *info) 757 except ImportError: 758 print "unable to load device-specific module; assuming none" 759 760 def _DoCall(self, function_name, *args, **kwargs): 761 """Call the named function in the device-specific module, passing 762 the given args and kwargs. The first argument to the call will be 763 the DeviceSpecific object itself. If there is no module, or the 764 module does not define the function, return the value of the 765 'default' kwarg (which itself defaults to None).""" 766 if self.module is None or not hasattr(self.module, function_name): 767 return kwargs.get("default", None) 768 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 769 770 def FullOTA_Assertions(self): 771 """Called after emitting the block of assertions at the top of a 772 full OTA package. Implementations can add whatever additional 773 assertions they like.""" 774 return self._DoCall("FullOTA_Assertions") 775 776 def FullOTA_InstallBegin(self): 777 """Called at the start of full OTA installation.""" 778 return self._DoCall("FullOTA_InstallBegin") 779 780 def FullOTA_InstallEnd(self): 781 """Called at the end of full OTA installation; typically this is 782 used to install the image for the device's baseband processor.""" 783 return self._DoCall("FullOTA_InstallEnd") 784 785 def IncrementalOTA_Assertions(self): 786 """Called after emitting the block of assertions at the top of an 787 incremental OTA package. Implementations can add whatever 788 additional assertions they like.""" 789 return self._DoCall("IncrementalOTA_Assertions") 790 791 def IncrementalOTA_VerifyBegin(self): 792 """Called at the start of the verification phase of incremental 793 OTA installation; additional checks can be placed here to abort 794 the script before any changes are made.""" 795 return self._DoCall("IncrementalOTA_VerifyBegin") 796 797 def IncrementalOTA_VerifyEnd(self): 798 """Called at the end of the verification phase of incremental OTA 799 installation; additional checks can be placed here to abort the 800 script before any changes are made.""" 801 return self._DoCall("IncrementalOTA_VerifyEnd") 802 803 def IncrementalOTA_InstallBegin(self): 804 """Called at the start of incremental OTA installation (after 805 verification is complete).""" 806 return self._DoCall("IncrementalOTA_InstallBegin") 807 808 def IncrementalOTA_InstallEnd(self): 809 """Called at the end of incremental OTA installation; typically 810 this is used to install the image for the device's baseband 811 processor.""" 812 return self._DoCall("IncrementalOTA_InstallEnd") 813 814 class File(object): 815 def __init__(self, name, data): 816 self.name = name 817 self.data = data 818 self.size = len(data) 819 self.sha1 = sha1(data).hexdigest() 820 821 @classmethod 822 def FromLocalFile(cls, name, diskname): 823 f = open(diskname, "rb") 824 data = f.read() 825 f.close() 826 return File(name, data) 827 828 def WriteToTemp(self): 829 t = tempfile.NamedTemporaryFile() 830 t.write(self.data) 831 t.flush() 832 return t 833 834 def AddToZip(self, z): 835 ZipWriteStr(z, self.name, self.data) 836 837 DIFF_PROGRAM_BY_EXT = { 838 ".gz" : "imgdiff", 839 ".zip" : ["imgdiff", "-z"], 840 ".jar" : ["imgdiff", "-z"], 841 ".apk" : ["imgdiff", "-z"], 842 ".img" : "imgdiff", 843 } 844 845 class Difference(object): 846 def __init__(self, tf, sf, diff_program=None): 847 self.tf = tf 848 self.sf = sf 849 self.patch = None 850 self.diff_program = diff_program 851 852 def ComputePatch(self): 853 """Compute the patch (as a string of data) needed to turn sf into 854 tf. Returns the same tuple as GetPatch().""" 855 856 tf = self.tf 857 sf = self.sf 858 859 if self.diff_program: 860 diff_program = self.diff_program 861 else: 862 ext = os.path.splitext(tf.name)[1] 863 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 864 865 ttemp = tf.WriteToTemp() 866 stemp = sf.WriteToTemp() 867 868 ext = os.path.splitext(tf.name)[1] 869 870 try: 871 ptemp = tempfile.NamedTemporaryFile() 872 if isinstance(diff_program, list): 873 cmd = copy.copy(diff_program) 874 else: 875 cmd = [diff_program] 876 cmd.append(stemp.name) 877 cmd.append(ttemp.name) 878 cmd.append(ptemp.name) 879 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 880 _, err = p.communicate() 881 if err or p.returncode != 0: 882 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 883 return None 884 diff = ptemp.read() 885 finally: 886 ptemp.close() 887 stemp.close() 888 ttemp.close() 889 890 self.patch = diff 891 return self.tf, self.sf, self.patch 892 893 894 def GetPatch(self): 895 """Return a tuple (target_file, source_file, patch_data). 896 patch_data may be None if ComputePatch hasn't been called, or if 897 computing the patch failed.""" 898 return self.tf, self.sf, self.patch 899 900 901 def ComputeDifferences(diffs): 902 """Call ComputePatch on all the Difference objects in 'diffs'.""" 903 print len(diffs), "diffs to compute" 904 905 # Do the largest files first, to try and reduce the long-pole effect. 906 by_size = [(i.tf.size, i) for i in diffs] 907 by_size.sort(reverse=True) 908 by_size = [i[1] for i in by_size] 909 910 lock = threading.Lock() 911 diff_iter = iter(by_size) # accessed under lock 912 913 def worker(): 914 try: 915 lock.acquire() 916 for d in diff_iter: 917 lock.release() 918 start = time.time() 919 d.ComputePatch() 920 dur = time.time() - start 921 lock.acquire() 922 923 tf, sf, patch = d.GetPatch() 924 if sf.name == tf.name: 925 name = tf.name 926 else: 927 name = "%s (%s)" % (tf.name, sf.name) 928 if patch is None: 929 print "patching failed! %s" % (name,) 930 else: 931 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 932 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 933 lock.release() 934 except Exception, e: 935 print e 936 raise 937 938 # start worker threads; wait for them all to finish. 939 threads = [threading.Thread(target=worker) 940 for i in range(OPTIONS.worker_threads)] 941 for th in threads: 942 th.start() 943 while threads: 944 threads.pop().join() 945 946 947 # map recovery.fstab's fs_types to mount/format "partition types" 948 PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 949 "ext4": "EMMC", "emmc": "EMMC" } 950 951 def GetTypeAndDevice(mount_point, info): 952 fstab = info["fstab"] 953 if fstab: 954 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 955 else: 956 return None 957 958 959 def ParseCertificate(data): 960 """Parse a PEM-format certificate.""" 961 cert = [] 962 save = False 963 for line in data.split("\n"): 964 if "--END CERTIFICATE--" in line: 965 break 966 if save: 967 cert.append(line) 968 if "--BEGIN CERTIFICATE--" in line: 969 save = True 970 cert = "".join(cert).decode('base64') 971 return cert 972
