1 # Copyright (C) 2008 The Android Open Source Project 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import copy 16 import errno 17 import getopt 18 import getpass 19 import imp 20 import os 21 import platform 22 import re 23 import shutil 24 import subprocess 25 import sys 26 import tempfile 27 import threading 28 import time 29 import zipfile 30 31 try: 32 from hashlib import sha1 as sha1 33 except ImportError: 34 from sha import sha as sha1 35 36 # missing in Python 2.4 and before 37 if not hasattr(os, "SEEK_SET"): 38 os.SEEK_SET = 0 39 40 class Options(object): pass 41 OPTIONS = Options() 42 OPTIONS.search_path = "out/host/linux-x86" 43 OPTIONS.verbose = False 44 OPTIONS.tempfiles = [] 45 OPTIONS.device_specific = None 46 OPTIONS.extras = {} 47 OPTIONS.info_dict = None 48 49 50 # Values for "certificate" in apkcerts that mean special things. 51 SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 52 53 54 class ExternalError(RuntimeError): pass 55 56 57 def Run(args, **kwargs): 58 """Create and return a subprocess.Popen object, printing the command 59 line on the terminal if -v was specified.""" 60 if OPTIONS.verbose: 61 print " running: ", " ".join(args) 62 return subprocess.Popen(args, **kwargs) 63 64 65 def CloseInheritedPipes(): 66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 67 before doing other work.""" 68 if platform.system() != "Darwin": 69 return 70 for d in range(3, 1025): 71 try: 72 stat = os.fstat(d) 73 if stat is not None: 74 pipebit = stat[0] & 0x1000 75 if pipebit != 0: 76 os.close(d) 77 except OSError: 78 pass 79 80 81 def LoadInfoDict(zip): 82 """Read and parse the META/misc_info.txt key/value pairs from the 83 input target files and return a dict.""" 84 85 d = {} 86 try: 87 for line in zip.read("META/misc_info.txt").split("\n"): 88 line = line.strip() 89 if not line or line.startswith("#"): continue 90 k, v = line.split("=", 1) 91 d[k] = v 92 except KeyError: 93 # ok if misc_info.txt doesn't exist 94 pass 95 96 # backwards compatibility: These values used to be in their own 97 # files. Look for them, in case we're processing an old 98 # target_files zip. 99 100 if "mkyaffs2_extra_flags" not in d: 101 try: 102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 103 except KeyError: 104 # ok if flags don't exist 105 pass 106 107 if "recovery_api_version" not in d: 108 try: 109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 110 except KeyError: 111 raise ValueError("can't find recovery API version in input target-files") 112 113 if "tool_extensions" not in d: 114 try: 115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 116 except KeyError: 117 # ok if extensions don't exist 118 pass 119 120 try: 121 data = zip.read("META/imagesizes.txt") 122 for line in data.split("\n"): 123 if not line: continue 124 name, value = line.split(" ", 1) 125 if not value: continue 126 if name == "blocksize": 127 d[name] = value 128 else: 129 d[name + "_size"] = value 130 except KeyError: 131 pass 132 133 def makeint(key): 134 if key in d: 135 d[key] = int(d[key], 0) 136 137 makeint("recovery_api_version") 138 makeint("blocksize") 139 makeint("system_size") 140 makeint("userdata_size") 141 makeint("recovery_size") 142 makeint("boot_size") 143 144 d["fstab"] = LoadRecoveryFSTab(zip) 145 return d 146 147 def LoadRecoveryFSTab(zip): 148 class Partition(object): 149 pass 150 151 try: 152 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 153 except KeyError: 154 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip 155 data = "" 156 157 d = {} 158 for line in data.split("\n"): 159 line = line.strip() 160 if not line or line.startswith("#"): continue 161 pieces = line.split() 162 if not (3 <= len(pieces) <= 4): 163 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 164 165 p = Partition() 166 p.mount_point = pieces[0] 167 p.fs_type = pieces[1] 168 p.device = pieces[2] 169 p.length = 0 170 options = None 171 if len(pieces) >= 4: 172 if pieces[3].startswith("/"): 173 p.device2 = pieces[3] 174 if len(pieces) >= 5: 175 options = pieces[4] 176 else: 177 p.device2 = None 178 options = pieces[3] 179 else: 180 p.device2 = None 181 182 if options: 183 options = options.split(",") 184 for i in options: 185 if i.startswith("length="): 186 p.length = int(i[7:]) 187 else: 188 print "%s: unknown option \"%s\"" % (p.mount_point, i) 189 190 d[p.mount_point] = p 191 return d 192 193 194 def DumpInfoDict(d): 195 for k, v in sorted(d.items()): 196 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 197 198 def BuildBootableImage(sourcedir): 199 """Take a kernel, cmdline, and ramdisk directory from the input (in 200 'sourcedir'), and turn them into a boot image. Return the image 201 data, or None if sourcedir does not appear to contains files for 202 building the requested image.""" 203 204 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 205 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 206 return None 207 208 ramdisk_img = tempfile.NamedTemporaryFile() 209 img = tempfile.NamedTemporaryFile() 210 211 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 212 stdout=subprocess.PIPE) 213 p2 = Run(["minigzip"], 214 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 215 216 p2.wait() 217 p1.wait() 218 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 219 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 220 221 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 222 223 fn = os.path.join(sourcedir, "cmdline") 224 if os.access(fn, os.F_OK): 225 cmd.append("--cmdline") 226 cmd.append(open(fn).read().rstrip("\n")) 227 228 fn = os.path.join(sourcedir, "base") 229 if os.access(fn, os.F_OK): 230 cmd.append("--base") 231 cmd.append(open(fn).read().rstrip("\n")) 232 233 fn = os.path.join(sourcedir, "pagesize") 234 if os.access(fn, os.F_OK): 235 cmd.append("--pagesize") 236 cmd.append(open(fn).read().rstrip("\n")) 237 238 cmd.extend(["--ramdisk", ramdisk_img.name, 239 "--output", img.name]) 240 241 p = Run(cmd, stdout=subprocess.PIPE) 242 p.communicate() 243 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 244 os.path.basename(sourcedir),) 245 246 img.seek(os.SEEK_SET, 0) 247 data = img.read() 248 249 ramdisk_img.close() 250 img.close() 251 252 return data 253 254 255 def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir): 256 """Return a File object (with name 'name') with the desired bootable 257 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 258 'prebuilt_name', otherwise construct it from the source files in 259 'unpack_dir'/'tree_subdir'.""" 260 261 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 262 if os.path.exists(prebuilt_path): 263 print "using prebuilt %s..." % (prebuilt_name,) 264 return File.FromLocalFile(name, prebuilt_path) 265 else: 266 print "building image from target_files %s..." % (tree_subdir,) 267 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir))) 268 269 270 def UnzipTemp(filename, pattern=None): 271 """Unzip the given archive into a temporary directory and return the name. 272 273 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 274 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 275 276 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 277 main file), open for reading. 278 """ 279 280 tmp = tempfile.mkdtemp(prefix="targetfiles-") 281 OPTIONS.tempfiles.append(tmp) 282 283 def unzip_to_dir(filename, dirname): 284 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 285 if pattern is not None: 286 cmd.append(pattern) 287 p = Run(cmd, stdout=subprocess.PIPE) 288 p.communicate() 289 if p.returncode != 0: 290 raise ExternalError("failed to unzip input target-files \"%s\"" % 291 (filename,)) 292 293 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 294 if m: 295 unzip_to_dir(m.group(1), tmp) 296 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 297 filename = m.group(1) 298 else: 299 unzip_to_dir(filename, tmp) 300 301 return tmp, zipfile.ZipFile(filename, "r") 302 303 304 def GetKeyPasswords(keylist): 305 """Given a list of keys, prompt the user to enter passwords for 306 those which require them. Return a {key: password} dict. password 307 will be None if the key has no password.""" 308 309 no_passwords = [] 310 need_passwords = [] 311 devnull = open("/dev/null", "w+b") 312 for k in sorted(keylist): 313 # We don't need a password for things that aren't really keys. 314 if k in SPECIAL_CERT_STRINGS: 315 no_passwords.append(k) 316 continue 317 318 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 319 "-inform", "DER", "-nocrypt"], 320 stdin=devnull.fileno(), 321 stdout=devnull.fileno(), 322 stderr=subprocess.STDOUT) 323 p.communicate() 324 if p.returncode == 0: 325 no_passwords.append(k) 326 else: 327 need_passwords.append(k) 328 devnull.close() 329 330 key_passwords = PasswordManager().GetPasswords(need_passwords) 331 key_passwords.update(dict.fromkeys(no_passwords, None)) 332 return key_passwords 333 334 335 def SignFile(input_name, output_name, key, password, align=None, 336 whole_file=False): 337 """Sign the input_name zip/jar/apk, producing output_name. Use the 338 given key and password (the latter may be None if the key does not 339 have a password. 340 341 If align is an integer > 1, zipalign is run to align stored files in 342 the output zip on 'align'-byte boundaries. 343 344 If whole_file is true, use the "-w" option to SignApk to embed a 345 signature that covers the whole file in the archive comment of the 346 zip file. 347 """ 348 349 if align == 0 or align == 1: 350 align = None 351 352 if align: 353 temp = tempfile.NamedTemporaryFile() 354 sign_name = temp.name 355 else: 356 sign_name = output_name 357 358 cmd = ["java", "-Xmx2048m", "-jar", 359 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 360 if whole_file: 361 cmd.append("-w") 362 cmd.extend([key + ".x509.pem", key + ".pk8", 363 input_name, sign_name]) 364 365 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 366 if password is not None: 367 password += "\n" 368 p.communicate(password) 369 if p.returncode != 0: 370 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 371 372 if align: 373 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 374 p.communicate() 375 if p.returncode != 0: 376 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 377 temp.close() 378 379 380 def CheckSize(data, target, info_dict): 381 """Check the data string passed against the max size limit, if 382 any, for the given target. Raise exception if the data is too big. 383 Print a warning if the data is nearing the maximum size.""" 384 385 if target.endswith(".img"): target = target[:-4] 386 mount_point = "/" + target 387 388 if info_dict["fstab"]: 389 if mount_point == "/userdata": mount_point = "/data" 390 p = info_dict["fstab"][mount_point] 391 fs_type = p.fs_type 392 limit = info_dict.get(p.device + "_size", None) 393 if not fs_type or not limit: return 394 395 if fs_type == "yaffs2": 396 # image size should be increased by 1/64th to account for the 397 # spare area (64 bytes per 2k page) 398 limit = limit / 2048 * (2048+64) 399 size = len(data) 400 pct = float(size) * 100.0 / limit 401 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 402 if pct >= 99.0: 403 raise ExternalError(msg) 404 elif pct >= 95.0: 405 print 406 print " WARNING: ", msg 407 print 408 elif OPTIONS.verbose: 409 print " ", msg 410 411 412 def ReadApkCerts(tf_zip): 413 """Given a target_files ZipFile, parse the META/apkcerts.txt file 414 and return a {package: cert} dict.""" 415 certmap = {} 416 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 417 line = line.strip() 418 if not line: continue 419 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 420 r'private_key="(.*)"$', line) 421 if m: 422 name, cert, privkey = m.groups() 423 if cert in SPECIAL_CERT_STRINGS and not privkey: 424 certmap[name] = cert 425 elif (cert.endswith(".x509.pem") and 426 privkey.endswith(".pk8") and 427 cert[:-9] == privkey[:-4]): 428 certmap[name] = cert[:-9] 429 else: 430 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 431 return certmap 432 433 434 COMMON_DOCSTRING = """ 435 -p (--path) <dir> 436 Prepend <dir>/bin to the list of places to search for binaries 437 run by this script, and expect to find jars in <dir>/framework. 438 439 -s (--device_specific) <file> 440 Path to the python module containing device-specific 441 releasetools code. 442 443 -x (--extra) <key=value> 444 Add a key/value pair to the 'extras' dict, which device-specific 445 extension code may look at. 446 447 -v (--verbose) 448 Show command lines being executed. 449 450 -h (--help) 451 Display this usage message and exit. 452 """ 453 454 def Usage(docstring): 455 print docstring.rstrip("\n") 456 print COMMON_DOCSTRING 457 458 459 def ParseOptions(argv, 460 docstring, 461 extra_opts="", extra_long_opts=(), 462 extra_option_handler=None): 463 """Parse the options in argv and return any arguments that aren't 464 flags. docstring is the calling module's docstring, to be displayed 465 for errors and -h. extra_opts and extra_long_opts are for flags 466 defined by the caller, which are processed by passing them to 467 extra_option_handler.""" 468 469 try: 470 opts, args = getopt.getopt( 471 argv, "hvp:s:x:" + extra_opts, 472 ["help", "verbose", "path=", "device_specific=", "extra="] + 473 list(extra_long_opts)) 474 except getopt.GetoptError, err: 475 Usage(docstring) 476 print "**", str(err), "**" 477 sys.exit(2) 478 479 path_specified = False 480 481 for o, a in opts: 482 if o in ("-h", "--help"): 483 Usage(docstring) 484 sys.exit() 485 elif o in ("-v", "--verbose"): 486 OPTIONS.verbose = True 487 elif o in ("-p", "--path"): 488 OPTIONS.search_path = a 489 elif o in ("-s", "--device_specific"): 490 OPTIONS.device_specific = a 491 elif o in ("-x", "--extra"): 492 key, value = a.split("=", 1) 493 OPTIONS.extras[key] = value 494 else: 495 if extra_option_handler is None or not extra_option_handler(o, a): 496 assert False, "unknown option \"%s\"" % (o,) 497 498 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 499 os.pathsep + os.environ["PATH"]) 500 501 return args 502 503 504 def Cleanup(): 505 for i in OPTIONS.tempfiles: 506 if os.path.isdir(i): 507 shutil.rmtree(i) 508 else: 509 os.remove(i) 510 511 512 class PasswordManager(object): 513 def __init__(self): 514 self.editor = os.getenv("EDITOR", None) 515 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 516 517 def GetPasswords(self, items): 518 """Get passwords corresponding to each string in 'items', 519 returning a dict. (The dict may have keys in addition to the 520 values in 'items'.) 521 522 Uses the passwords in $ANDROID_PW_FILE if available, letting the 523 user edit that file to add more needed passwords. If no editor is 524 available, or $ANDROID_PW_FILE isn't define, prompts the user 525 interactively in the ordinary way. 526 """ 527 528 current = self.ReadFile() 529 530 first = True 531 while True: 532 missing = [] 533 for i in items: 534 if i not in current or not current[i]: 535 missing.append(i) 536 # Are all the passwords already in the file? 537 if not missing: return current 538 539 for i in missing: 540 current[i] = "" 541 542 if not first: 543 print "key file %s still missing some passwords." % (self.pwfile,) 544 answer = raw_input("try to edit again? [y]> ").strip() 545 if answer and answer[0] not in 'yY': 546 raise RuntimeError("key passwords unavailable") 547 first = False 548 549 current = self.UpdateAndReadFile(current) 550 551 def PromptResult(self, current): 552 """Prompt the user to enter a value (password) for each key in 553 'current' whose value is fales. Returns a new dict with all the 554 values. 555 """ 556 result = {} 557 for k, v in sorted(current.iteritems()): 558 if v: 559 result[k] = v 560 else: 561 while True: 562 result[k] = getpass.getpass("Enter password for %s key> " 563 % (k,)).strip() 564 if result[k]: break 565 return result 566 567 def UpdateAndReadFile(self, current): 568 if not self.editor or not self.pwfile: 569 return self.PromptResult(current) 570 571 f = open(self.pwfile, "w") 572 os.chmod(self.pwfile, 0600) 573 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 574 f.write("# (Additional spaces are harmless.)\n\n") 575 576 first_line = None 577 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 578 sorted.sort() 579 for i, (_, k, v) in enumerate(sorted): 580 f.write("[[[ %s ]]] %s\n" % (v, k)) 581 if not v and first_line is None: 582 # position cursor on first line with no password. 583 first_line = i + 4 584 f.close() 585 586 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 587 _, _ = p.communicate() 588 589 return self.ReadFile() 590 591 def ReadFile(self): 592 result = {} 593 if self.pwfile is None: return result 594 try: 595 f = open(self.pwfile, "r") 596 for line in f: 597 line = line.strip() 598 if not line or line[0] == '#': continue 599 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 600 if not m: 601 print "failed to parse password file: ", line 602 else: 603 result[m.group(2)] = m.group(1) 604 f.close() 605 except IOError, e: 606 if e.errno != errno.ENOENT: 607 print "error reading password file: ", str(e) 608 return result 609 610 611 def ZipWriteStr(zip, filename, data, perms=0644): 612 # use a fixed timestamp so the output is repeatable. 613 zinfo = zipfile.ZipInfo(filename=filename, 614 date_time=(2009, 1, 1, 0, 0, 0)) 615 zinfo.compress_type = zip.compression 616 zinfo.external_attr = perms << 16 617 zip.writestr(zinfo, data) 618 619 620 class DeviceSpecificParams(object): 621 module = None 622 def __init__(self, **kwargs): 623 """Keyword arguments to the constructor become attributes of this 624 object, which is passed to all functions in the device-specific 625 module.""" 626 for k, v in kwargs.iteritems(): 627 setattr(self, k, v) 628 self.extras = OPTIONS.extras 629 630 if self.module is None: 631 path = OPTIONS.device_specific 632 if not path: return 633 try: 634 if os.path.isdir(path): 635 info = imp.find_module("releasetools", [path]) 636 else: 637 d, f = os.path.split(path) 638 b, x = os.path.splitext(f) 639 if x == ".py": 640 f = b 641 info = imp.find_module(f, [d]) 642 self.module = imp.load_module("device_specific", *info) 643 except ImportError: 644 print "unable to load device-specific module; assuming none" 645 646 def _DoCall(self, function_name, *args, **kwargs): 647 """Call the named function in the device-specific module, passing 648 the given args and kwargs. The first argument to the call will be 649 the DeviceSpecific object itself. If there is no module, or the 650 module does not define the function, return the value of the 651 'default' kwarg (which itself defaults to None).""" 652 if self.module is None or not hasattr(self.module, function_name): 653 return kwargs.get("default", None) 654 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 655 656 def FullOTA_Assertions(self): 657 """Called after emitting the block of assertions at the top of a 658 full OTA package. Implementations can add whatever additional 659 assertions they like.""" 660 return self._DoCall("FullOTA_Assertions") 661 662 def FullOTA_InstallEnd(self): 663 """Called at the end of full OTA installation; typically this is 664 used to install the image for the device's baseband processor.""" 665 return self._DoCall("FullOTA_InstallEnd") 666 667 def IncrementalOTA_Assertions(self): 668 """Called after emitting the block of assertions at the top of an 669 incremental OTA package. Implementations can add whatever 670 additional assertions they like.""" 671 return self._DoCall("IncrementalOTA_Assertions") 672 673 def IncrementalOTA_VerifyEnd(self): 674 """Called at the end of the verification phase of incremental OTA 675 installation; additional checks can be placed here to abort the 676 script before any changes are made.""" 677 return self._DoCall("IncrementalOTA_VerifyEnd") 678 679 def IncrementalOTA_InstallEnd(self): 680 """Called at the end of incremental OTA installation; typically 681 this is used to install the image for the device's baseband 682 processor.""" 683 return self._DoCall("IncrementalOTA_InstallEnd") 684 685 class File(object): 686 def __init__(self, name, data): 687 self.name = name 688 self.data = data 689 self.size = len(data) 690 self.sha1 = sha1(data).hexdigest() 691 692 @classmethod 693 def FromLocalFile(cls, name, diskname): 694 f = open(diskname, "rb") 695 data = f.read() 696 f.close() 697 return File(name, data) 698 699 def WriteToTemp(self): 700 t = tempfile.NamedTemporaryFile() 701 t.write(self.data) 702 t.flush() 703 return t 704 705 def AddToZip(self, z): 706 ZipWriteStr(z, self.name, self.data) 707 708 DIFF_PROGRAM_BY_EXT = { 709 ".gz" : "imgdiff", 710 ".zip" : ["imgdiff", "-z"], 711 ".jar" : ["imgdiff", "-z"], 712 ".apk" : ["imgdiff", "-z"], 713 ".img" : "imgdiff", 714 } 715 716 class Difference(object): 717 def __init__(self, tf, sf): 718 self.tf = tf 719 self.sf = sf 720 self.patch = None 721 722 def ComputePatch(self): 723 """Compute the patch (as a string of data) needed to turn sf into 724 tf. Returns the same tuple as GetPatch().""" 725 726 tf = self.tf 727 sf = self.sf 728 729 ext = os.path.splitext(tf.name)[1] 730 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 731 732 ttemp = tf.WriteToTemp() 733 stemp = sf.WriteToTemp() 734 735 ext = os.path.splitext(tf.name)[1] 736 737 try: 738 ptemp = tempfile.NamedTemporaryFile() 739 if isinstance(diff_program, list): 740 cmd = copy.copy(diff_program) 741 else: 742 cmd = [diff_program] 743 cmd.append(stemp.name) 744 cmd.append(ttemp.name) 745 cmd.append(ptemp.name) 746 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 747 _, err = p.communicate() 748 if err or p.returncode != 0: 749 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 750 return None 751 diff = ptemp.read() 752 finally: 753 ptemp.close() 754 stemp.close() 755 ttemp.close() 756 757 self.patch = diff 758 return self.tf, self.sf, self.patch 759 760 761 def GetPatch(self): 762 """Return a tuple (target_file, source_file, patch_data). 763 patch_data may be None if ComputePatch hasn't been called, or if 764 computing the patch failed.""" 765 return self.tf, self.sf, self.patch 766 767 768 def ComputeDifferences(diffs): 769 """Call ComputePatch on all the Difference objects in 'diffs'.""" 770 print len(diffs), "diffs to compute" 771 772 # Do the largest files first, to try and reduce the long-pole effect. 773 by_size = [(i.tf.size, i) for i in diffs] 774 by_size.sort(reverse=True) 775 by_size = [i[1] for i in by_size] 776 777 lock = threading.Lock() 778 diff_iter = iter(by_size) # accessed under lock 779 780 def worker(): 781 try: 782 lock.acquire() 783 for d in diff_iter: 784 lock.release() 785 start = time.time() 786 d.ComputePatch() 787 dur = time.time() - start 788 lock.acquire() 789 790 tf, sf, patch = d.GetPatch() 791 if sf.name == tf.name: 792 name = tf.name 793 else: 794 name = "%s (%s)" % (tf.name, sf.name) 795 if patch is None: 796 print "patching failed! %s" % (name,) 797 else: 798 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 799 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 800 lock.release() 801 except Exception, e: 802 print e 803 raise 804 805 # start worker threads; wait for them all to finish. 806 threads = [threading.Thread(target=worker) 807 for i in range(OPTIONS.worker_threads)] 808 for th in threads: 809 th.start() 810 while threads: 811 threads.pop().join() 812 813 814 # map recovery.fstab's fs_types to mount/format "partition types" 815 PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 816 "ext4": "EMMC", "emmc": "EMMC" } 817 818 def GetTypeAndDevice(mount_point, info): 819 fstab = info["fstab"] 820 if fstab: 821 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 822 else: 823 return None 824