Home | History | Annotate | Download | only in releasetools
      1 # Copyright (C) 2008 The Android Open Source Project
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
      4 # you may not use this file except in compliance with the License.
      5 # You may obtain a copy of the License at
      6 #
      7 #      http://www.apache.org/licenses/LICENSE-2.0
      8 #
      9 # Unless required by applicable law or agreed to in writing, software
     10 # distributed under the License is distributed on an "AS IS" BASIS,
     11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 
     15 import copy
     16 import errno
     17 import getopt
     18 import getpass
     19 import imp
     20 import os
     21 import platform
     22 import re
     23 import shutil
     24 import subprocess
     25 import sys
     26 import tempfile
     27 import threading
     28 import time
     29 import zipfile
     30 
     31 try:
     32   from hashlib import sha1 as sha1
     33 except ImportError:
     34   from sha import sha as sha1
     35 
     36 # missing in Python 2.4 and before
     37 if not hasattr(os, "SEEK_SET"):
     38   os.SEEK_SET = 0
     39 
     40 class Options(object): pass
     41 OPTIONS = Options()
     42 OPTIONS.search_path = "out/host/linux-x86"
     43 OPTIONS.verbose = False
     44 OPTIONS.tempfiles = []
     45 OPTIONS.device_specific = None
     46 OPTIONS.extras = {}
     47 OPTIONS.info_dict = None
     48 
     49 
     50 # Values for "certificate" in apkcerts that mean special things.
     51 SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
     52 
     53 
     54 class ExternalError(RuntimeError): pass
     55 
     56 
     57 def Run(args, **kwargs):
     58   """Create and return a subprocess.Popen object, printing the command
     59   line on the terminal if -v was specified."""
     60   if OPTIONS.verbose:
     61     print "  running: ", " ".join(args)
     62   return subprocess.Popen(args, **kwargs)
     63 
     64 
     65 def CloseInheritedPipes():
     66   """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
     67   before doing other work."""
     68   if platform.system() != "Darwin":
     69     return
     70   for d in range(3, 1025):
     71     try:
     72       stat = os.fstat(d)
     73       if stat is not None:
     74         pipebit = stat[0] & 0x1000
     75         if pipebit != 0:
     76           os.close(d)
     77     except OSError:
     78       pass
     79 
     80 
     81 def LoadInfoDict(zip):
     82   """Read and parse the META/misc_info.txt key/value pairs from the
     83   input target files and return a dict."""
     84 
     85   d = {}
     86   try:
     87     for line in zip.read("META/misc_info.txt").split("\n"):
     88       line = line.strip()
     89       if not line or line.startswith("#"): continue
     90       k, v = line.split("=", 1)
     91       d[k] = v
     92   except KeyError:
     93     # ok if misc_info.txt doesn't exist
     94     pass
     95 
     96   # backwards compatibility: These values used to be in their own
     97   # files.  Look for them, in case we're processing an old
     98   # target_files zip.
     99 
    100   if "mkyaffs2_extra_flags" not in d:
    101     try:
    102       d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
    103     except KeyError:
    104       # ok if flags don't exist
    105       pass
    106 
    107   if "recovery_api_version" not in d:
    108     try:
    109       d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
    110     except KeyError:
    111       raise ValueError("can't find recovery API version in input target-files")
    112 
    113   if "tool_extensions" not in d:
    114     try:
    115       d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
    116     except KeyError:
    117       # ok if extensions don't exist
    118       pass
    119 
    120   try:
    121     data = zip.read("META/imagesizes.txt")
    122     for line in data.split("\n"):
    123       if not line: continue
    124       name, value = line.split(" ", 1)
    125       if not value: continue
    126       if name == "blocksize":
    127         d[name] = value
    128       else:
    129         d[name + "_size"] = value
    130   except KeyError:
    131     pass
    132 
    133   def makeint(key):
    134     if key in d:
    135       d[key] = int(d[key], 0)
    136 
    137   makeint("recovery_api_version")
    138   makeint("blocksize")
    139   makeint("system_size")
    140   makeint("userdata_size")
    141   makeint("recovery_size")
    142   makeint("boot_size")
    143 
    144   d["fstab"] = LoadRecoveryFSTab(zip)
    145   return d
    146 
    147 def LoadRecoveryFSTab(zip):
    148   class Partition(object):
    149     pass
    150 
    151   try:
    152     data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
    153   except KeyError:
    154     print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip
    155     data = ""
    156 
    157   d = {}
    158   for line in data.split("\n"):
    159     line = line.strip()
    160     if not line or line.startswith("#"): continue
    161     pieces = line.split()
    162     if not (3 <= len(pieces) <= 4):
    163       raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
    164 
    165     p = Partition()
    166     p.mount_point = pieces[0]
    167     p.fs_type = pieces[1]
    168     p.device = pieces[2]
    169     p.length = 0
    170     options = None
    171     if len(pieces) >= 4:
    172       if pieces[3].startswith("/"):
    173         p.device2 = pieces[3]
    174         if len(pieces) >= 5:
    175           options = pieces[4]
    176       else:
    177         p.device2 = None
    178         options = pieces[3]
    179     else:
    180       p.device2 = None
    181 
    182     if options:
    183       options = options.split(",")
    184       for i in options:
    185         if i.startswith("length="):
    186           p.length = int(i[7:])
    187         else:
    188           print "%s: unknown option \"%s\"" % (p.mount_point, i)
    189 
    190     d[p.mount_point] = p
    191   return d
    192 
    193 
    194 def DumpInfoDict(d):
    195   for k, v in sorted(d.items()):
    196     print "%-25s = (%s) %s" % (k, type(v).__name__, v)
    197 
    198 def BuildBootableImage(sourcedir):
    199   """Take a kernel, cmdline, and ramdisk directory from the input (in
    200   'sourcedir'), and turn them into a boot image.  Return the image
    201   data, or None if sourcedir does not appear to contains files for
    202   building the requested image."""
    203 
    204   if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
    205       not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
    206     return None
    207 
    208   ramdisk_img = tempfile.NamedTemporaryFile()
    209   img = tempfile.NamedTemporaryFile()
    210 
    211   p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
    212            stdout=subprocess.PIPE)
    213   p2 = Run(["minigzip"],
    214            stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
    215 
    216   p2.wait()
    217   p1.wait()
    218   assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
    219   assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
    220 
    221   cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
    222 
    223   fn = os.path.join(sourcedir, "cmdline")
    224   if os.access(fn, os.F_OK):
    225     cmd.append("--cmdline")
    226     cmd.append(open(fn).read().rstrip("\n"))
    227 
    228   fn = os.path.join(sourcedir, "base")
    229   if os.access(fn, os.F_OK):
    230     cmd.append("--base")
    231     cmd.append(open(fn).read().rstrip("\n"))
    232 
    233   fn = os.path.join(sourcedir, "pagesize")
    234   if os.access(fn, os.F_OK):
    235     cmd.append("--pagesize")
    236     cmd.append(open(fn).read().rstrip("\n"))
    237 
    238   cmd.extend(["--ramdisk", ramdisk_img.name,
    239               "--output", img.name])
    240 
    241   p = Run(cmd, stdout=subprocess.PIPE)
    242   p.communicate()
    243   assert p.returncode == 0, "mkbootimg of %s image failed" % (
    244       os.path.basename(sourcedir),)
    245 
    246   img.seek(os.SEEK_SET, 0)
    247   data = img.read()
    248 
    249   ramdisk_img.close()
    250   img.close()
    251 
    252   return data
    253 
    254 
    255 def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
    256   """Return a File object (with name 'name') with the desired bootable
    257   image.  Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
    258   'prebuilt_name', otherwise construct it from the source files in
    259   'unpack_dir'/'tree_subdir'."""
    260 
    261   prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
    262   if os.path.exists(prebuilt_path):
    263     print "using prebuilt %s..." % (prebuilt_name,)
    264     return File.FromLocalFile(name, prebuilt_path)
    265   else:
    266     print "building image from target_files %s..." % (tree_subdir,)
    267     return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
    268 
    269 
    270 def UnzipTemp(filename, pattern=None):
    271   """Unzip the given archive into a temporary directory and return the name.
    272 
    273   If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
    274   temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
    275 
    276   Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
    277   main file), open for reading.
    278   """
    279 
    280   tmp = tempfile.mkdtemp(prefix="targetfiles-")
    281   OPTIONS.tempfiles.append(tmp)
    282 
    283   def unzip_to_dir(filename, dirname):
    284     cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
    285     if pattern is not None:
    286       cmd.append(pattern)
    287     p = Run(cmd, stdout=subprocess.PIPE)
    288     p.communicate()
    289     if p.returncode != 0:
    290       raise ExternalError("failed to unzip input target-files \"%s\"" %
    291                           (filename,))
    292 
    293   m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
    294   if m:
    295     unzip_to_dir(m.group(1), tmp)
    296     unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
    297     filename = m.group(1)
    298   else:
    299     unzip_to_dir(filename, tmp)
    300 
    301   return tmp, zipfile.ZipFile(filename, "r")
    302 
    303 
    304 def GetKeyPasswords(keylist):
    305   """Given a list of keys, prompt the user to enter passwords for
    306   those which require them.  Return a {key: password} dict.  password
    307   will be None if the key has no password."""
    308 
    309   no_passwords = []
    310   need_passwords = []
    311   devnull = open("/dev/null", "w+b")
    312   for k in sorted(keylist):
    313     # We don't need a password for things that aren't really keys.
    314     if k in SPECIAL_CERT_STRINGS:
    315       no_passwords.append(k)
    316       continue
    317 
    318     p = Run(["openssl", "pkcs8", "-in", k+".pk8",
    319              "-inform", "DER", "-nocrypt"],
    320             stdin=devnull.fileno(),
    321             stdout=devnull.fileno(),
    322             stderr=subprocess.STDOUT)
    323     p.communicate()
    324     if p.returncode == 0:
    325       no_passwords.append(k)
    326     else:
    327       need_passwords.append(k)
    328   devnull.close()
    329 
    330   key_passwords = PasswordManager().GetPasswords(need_passwords)
    331   key_passwords.update(dict.fromkeys(no_passwords, None))
    332   return key_passwords
    333 
    334 
    335 def SignFile(input_name, output_name, key, password, align=None,
    336              whole_file=False):
    337   """Sign the input_name zip/jar/apk, producing output_name.  Use the
    338   given key and password (the latter may be None if the key does not
    339   have a password.
    340 
    341   If align is an integer > 1, zipalign is run to align stored files in
    342   the output zip on 'align'-byte boundaries.
    343 
    344   If whole_file is true, use the "-w" option to SignApk to embed a
    345   signature that covers the whole file in the archive comment of the
    346   zip file.
    347   """
    348 
    349   if align == 0 or align == 1:
    350     align = None
    351 
    352   if align:
    353     temp = tempfile.NamedTemporaryFile()
    354     sign_name = temp.name
    355   else:
    356     sign_name = output_name
    357 
    358   cmd = ["java", "-Xmx2048m", "-jar",
    359            os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
    360   if whole_file:
    361     cmd.append("-w")
    362   cmd.extend([key + ".x509.pem", key + ".pk8",
    363               input_name, sign_name])
    364 
    365   p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    366   if password is not None:
    367     password += "\n"
    368   p.communicate(password)
    369   if p.returncode != 0:
    370     raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
    371 
    372   if align:
    373     p = Run(["zipalign", "-f", str(align), sign_name, output_name])
    374     p.communicate()
    375     if p.returncode != 0:
    376       raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
    377     temp.close()
    378 
    379 
    380 def CheckSize(data, target, info_dict):
    381   """Check the data string passed against the max size limit, if
    382   any, for the given target.  Raise exception if the data is too big.
    383   Print a warning if the data is nearing the maximum size."""
    384 
    385   if target.endswith(".img"): target = target[:-4]
    386   mount_point = "/" + target
    387 
    388   if info_dict["fstab"]:
    389     if mount_point == "/userdata": mount_point = "/data"
    390     p = info_dict["fstab"][mount_point]
    391     fs_type = p.fs_type
    392     limit = info_dict.get(p.device + "_size", None)
    393   if not fs_type or not limit: return
    394 
    395   if fs_type == "yaffs2":
    396     # image size should be increased by 1/64th to account for the
    397     # spare area (64 bytes per 2k page)
    398     limit = limit / 2048 * (2048+64)
    399     size = len(data)
    400     pct = float(size) * 100.0 / limit
    401     msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
    402     if pct >= 99.0:
    403       raise ExternalError(msg)
    404     elif pct >= 95.0:
    405       print
    406       print "  WARNING: ", msg
    407       print
    408     elif OPTIONS.verbose:
    409       print "  ", msg
    410 
    411 
    412 def ReadApkCerts(tf_zip):
    413   """Given a target_files ZipFile, parse the META/apkcerts.txt file
    414   and return a {package: cert} dict."""
    415   certmap = {}
    416   for line in tf_zip.read("META/apkcerts.txt").split("\n"):
    417     line = line.strip()
    418     if not line: continue
    419     m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
    420                  r'private_key="(.*)"$', line)
    421     if m:
    422       name, cert, privkey = m.groups()
    423       if cert in SPECIAL_CERT_STRINGS and not privkey:
    424         certmap[name] = cert
    425       elif (cert.endswith(".x509.pem") and
    426             privkey.endswith(".pk8") and
    427             cert[:-9] == privkey[:-4]):
    428         certmap[name] = cert[:-9]
    429       else:
    430         raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
    431   return certmap
    432 
    433 
    434 COMMON_DOCSTRING = """
    435   -p  (--path)  <dir>
    436       Prepend <dir>/bin to the list of places to search for binaries
    437       run by this script, and expect to find jars in <dir>/framework.
    438 
    439   -s  (--device_specific) <file>
    440       Path to the python module containing device-specific
    441       releasetools code.
    442 
    443   -x  (--extra)  <key=value>
    444       Add a key/value pair to the 'extras' dict, which device-specific
    445       extension code may look at.
    446 
    447   -v  (--verbose)
    448       Show command lines being executed.
    449 
    450   -h  (--help)
    451       Display this usage message and exit.
    452 """
    453 
    454 def Usage(docstring):
    455   print docstring.rstrip("\n")
    456   print COMMON_DOCSTRING
    457 
    458 
    459 def ParseOptions(argv,
    460                  docstring,
    461                  extra_opts="", extra_long_opts=(),
    462                  extra_option_handler=None):
    463   """Parse the options in argv and return any arguments that aren't
    464   flags.  docstring is the calling module's docstring, to be displayed
    465   for errors and -h.  extra_opts and extra_long_opts are for flags
    466   defined by the caller, which are processed by passing them to
    467   extra_option_handler."""
    468 
    469   try:
    470     opts, args = getopt.getopt(
    471         argv, "hvp:s:x:" + extra_opts,
    472         ["help", "verbose", "path=", "device_specific=", "extra="] +
    473           list(extra_long_opts))
    474   except getopt.GetoptError, err:
    475     Usage(docstring)
    476     print "**", str(err), "**"
    477     sys.exit(2)
    478 
    479   path_specified = False
    480 
    481   for o, a in opts:
    482     if o in ("-h", "--help"):
    483       Usage(docstring)
    484       sys.exit()
    485     elif o in ("-v", "--verbose"):
    486       OPTIONS.verbose = True
    487     elif o in ("-p", "--path"):
    488       OPTIONS.search_path = a
    489     elif o in ("-s", "--device_specific"):
    490       OPTIONS.device_specific = a
    491     elif o in ("-x", "--extra"):
    492       key, value = a.split("=", 1)
    493       OPTIONS.extras[key] = value
    494     else:
    495       if extra_option_handler is None or not extra_option_handler(o, a):
    496         assert False, "unknown option \"%s\"" % (o,)
    497 
    498   os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
    499                         os.pathsep + os.environ["PATH"])
    500 
    501   return args
    502 
    503 
    504 def Cleanup():
    505   for i in OPTIONS.tempfiles:
    506     if os.path.isdir(i):
    507       shutil.rmtree(i)
    508     else:
    509       os.remove(i)
    510 
    511 
    512 class PasswordManager(object):
    513   def __init__(self):
    514     self.editor = os.getenv("EDITOR", None)
    515     self.pwfile = os.getenv("ANDROID_PW_FILE", None)
    516 
    517   def GetPasswords(self, items):
    518     """Get passwords corresponding to each string in 'items',
    519     returning a dict.  (The dict may have keys in addition to the
    520     values in 'items'.)
    521 
    522     Uses the passwords in $ANDROID_PW_FILE if available, letting the
    523     user edit that file to add more needed passwords.  If no editor is
    524     available, or $ANDROID_PW_FILE isn't define, prompts the user
    525     interactively in the ordinary way.
    526     """
    527 
    528     current = self.ReadFile()
    529 
    530     first = True
    531     while True:
    532       missing = []
    533       for i in items:
    534         if i not in current or not current[i]:
    535           missing.append(i)
    536       # Are all the passwords already in the file?
    537       if not missing: return current
    538 
    539       for i in missing:
    540         current[i] = ""
    541 
    542       if not first:
    543         print "key file %s still missing some passwords." % (self.pwfile,)
    544         answer = raw_input("try to edit again? [y]> ").strip()
    545         if answer and answer[0] not in 'yY':
    546           raise RuntimeError("key passwords unavailable")
    547       first = False
    548 
    549       current = self.UpdateAndReadFile(current)
    550 
    551   def PromptResult(self, current):
    552     """Prompt the user to enter a value (password) for each key in
    553     'current' whose value is fales.  Returns a new dict with all the
    554     values.
    555     """
    556     result = {}
    557     for k, v in sorted(current.iteritems()):
    558       if v:
    559         result[k] = v
    560       else:
    561         while True:
    562           result[k] = getpass.getpass("Enter password for %s key> "
    563                                       % (k,)).strip()
    564           if result[k]: break
    565     return result
    566 
    567   def UpdateAndReadFile(self, current):
    568     if not self.editor or not self.pwfile:
    569       return self.PromptResult(current)
    570 
    571     f = open(self.pwfile, "w")
    572     os.chmod(self.pwfile, 0600)
    573     f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
    574     f.write("# (Additional spaces are harmless.)\n\n")
    575 
    576     first_line = None
    577     sorted = [(not v, k, v) for (k, v) in current.iteritems()]
    578     sorted.sort()
    579     for i, (_, k, v) in enumerate(sorted):
    580       f.write("[[[  %s  ]]] %s\n" % (v, k))
    581       if not v and first_line is None:
    582         # position cursor on first line with no password.
    583         first_line = i + 4
    584     f.close()
    585 
    586     p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
    587     _, _ = p.communicate()
    588 
    589     return self.ReadFile()
    590 
    591   def ReadFile(self):
    592     result = {}
    593     if self.pwfile is None: return result
    594     try:
    595       f = open(self.pwfile, "r")
    596       for line in f:
    597         line = line.strip()
    598         if not line or line[0] == '#': continue
    599         m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
    600         if not m:
    601           print "failed to parse password file: ", line
    602         else:
    603           result[m.group(2)] = m.group(1)
    604       f.close()
    605     except IOError, e:
    606       if e.errno != errno.ENOENT:
    607         print "error reading password file: ", str(e)
    608     return result
    609 
    610 
    611 def ZipWriteStr(zip, filename, data, perms=0644):
    612   # use a fixed timestamp so the output is repeatable.
    613   zinfo = zipfile.ZipInfo(filename=filename,
    614                           date_time=(2009, 1, 1, 0, 0, 0))
    615   zinfo.compress_type = zip.compression
    616   zinfo.external_attr = perms << 16
    617   zip.writestr(zinfo, data)
    618 
    619 
    620 class DeviceSpecificParams(object):
    621   module = None
    622   def __init__(self, **kwargs):
    623     """Keyword arguments to the constructor become attributes of this
    624     object, which is passed to all functions in the device-specific
    625     module."""
    626     for k, v in kwargs.iteritems():
    627       setattr(self, k, v)
    628     self.extras = OPTIONS.extras
    629 
    630     if self.module is None:
    631       path = OPTIONS.device_specific
    632       if not path: return
    633       try:
    634         if os.path.isdir(path):
    635           info = imp.find_module("releasetools", [path])
    636         else:
    637           d, f = os.path.split(path)
    638           b, x = os.path.splitext(f)
    639           if x == ".py":
    640             f = b
    641           info = imp.find_module(f, [d])
    642         self.module = imp.load_module("device_specific", *info)
    643       except ImportError:
    644         print "unable to load device-specific module; assuming none"
    645 
    646   def _DoCall(self, function_name, *args, **kwargs):
    647     """Call the named function in the device-specific module, passing
    648     the given args and kwargs.  The first argument to the call will be
    649     the DeviceSpecific object itself.  If there is no module, or the
    650     module does not define the function, return the value of the
    651     'default' kwarg (which itself defaults to None)."""
    652     if self.module is None or not hasattr(self.module, function_name):
    653       return kwargs.get("default", None)
    654     return getattr(self.module, function_name)(*((self,) + args), **kwargs)
    655 
    656   def FullOTA_Assertions(self):
    657     """Called after emitting the block of assertions at the top of a
    658     full OTA package.  Implementations can add whatever additional
    659     assertions they like."""
    660     return self._DoCall("FullOTA_Assertions")
    661 
    662   def FullOTA_InstallEnd(self):
    663     """Called at the end of full OTA installation; typically this is
    664     used to install the image for the device's baseband processor."""
    665     return self._DoCall("FullOTA_InstallEnd")
    666 
    667   def IncrementalOTA_Assertions(self):
    668     """Called after emitting the block of assertions at the top of an
    669     incremental OTA package.  Implementations can add whatever
    670     additional assertions they like."""
    671     return self._DoCall("IncrementalOTA_Assertions")
    672 
    673   def IncrementalOTA_VerifyEnd(self):
    674     """Called at the end of the verification phase of incremental OTA
    675     installation; additional checks can be placed here to abort the
    676     script before any changes are made."""
    677     return self._DoCall("IncrementalOTA_VerifyEnd")
    678 
    679   def IncrementalOTA_InstallEnd(self):
    680     """Called at the end of incremental OTA installation; typically
    681     this is used to install the image for the device's baseband
    682     processor."""
    683     return self._DoCall("IncrementalOTA_InstallEnd")
    684 
    685 class File(object):
    686   def __init__(self, name, data):
    687     self.name = name
    688     self.data = data
    689     self.size = len(data)
    690     self.sha1 = sha1(data).hexdigest()
    691 
    692   @classmethod
    693   def FromLocalFile(cls, name, diskname):
    694     f = open(diskname, "rb")
    695     data = f.read()
    696     f.close()
    697     return File(name, data)
    698 
    699   def WriteToTemp(self):
    700     t = tempfile.NamedTemporaryFile()
    701     t.write(self.data)
    702     t.flush()
    703     return t
    704 
    705   def AddToZip(self, z):
    706     ZipWriteStr(z, self.name, self.data)
    707 
    708 DIFF_PROGRAM_BY_EXT = {
    709     ".gz" : "imgdiff",
    710     ".zip" : ["imgdiff", "-z"],
    711     ".jar" : ["imgdiff", "-z"],
    712     ".apk" : ["imgdiff", "-z"],
    713     ".img" : "imgdiff",
    714     }
    715 
    716 class Difference(object):
    717   def __init__(self, tf, sf):
    718     self.tf = tf
    719     self.sf = sf
    720     self.patch = None
    721 
    722   def ComputePatch(self):
    723     """Compute the patch (as a string of data) needed to turn sf into
    724     tf.  Returns the same tuple as GetPatch()."""
    725 
    726     tf = self.tf
    727     sf = self.sf
    728 
    729     ext = os.path.splitext(tf.name)[1]
    730     diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
    731 
    732     ttemp = tf.WriteToTemp()
    733     stemp = sf.WriteToTemp()
    734 
    735     ext = os.path.splitext(tf.name)[1]
    736 
    737     try:
    738       ptemp = tempfile.NamedTemporaryFile()
    739       if isinstance(diff_program, list):
    740         cmd = copy.copy(diff_program)
    741       else:
    742         cmd = [diff_program]
    743       cmd.append(stemp.name)
    744       cmd.append(ttemp.name)
    745       cmd.append(ptemp.name)
    746       p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    747       _, err = p.communicate()
    748       if err or p.returncode != 0:
    749         print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
    750         return None
    751       diff = ptemp.read()
    752     finally:
    753       ptemp.close()
    754       stemp.close()
    755       ttemp.close()
    756 
    757     self.patch = diff
    758     return self.tf, self.sf, self.patch
    759 
    760 
    761   def GetPatch(self):
    762     """Return a tuple (target_file, source_file, patch_data).
    763     patch_data may be None if ComputePatch hasn't been called, or if
    764     computing the patch failed."""
    765     return self.tf, self.sf, self.patch
    766 
    767 
    768 def ComputeDifferences(diffs):
    769   """Call ComputePatch on all the Difference objects in 'diffs'."""
    770   print len(diffs), "diffs to compute"
    771 
    772   # Do the largest files first, to try and reduce the long-pole effect.
    773   by_size = [(i.tf.size, i) for i in diffs]
    774   by_size.sort(reverse=True)
    775   by_size = [i[1] for i in by_size]
    776 
    777   lock = threading.Lock()
    778   diff_iter = iter(by_size)   # accessed under lock
    779 
    780   def worker():
    781     try:
    782       lock.acquire()
    783       for d in diff_iter:
    784         lock.release()
    785         start = time.time()
    786         d.ComputePatch()
    787         dur = time.time() - start
    788         lock.acquire()
    789 
    790         tf, sf, patch = d.GetPatch()
    791         if sf.name == tf.name:
    792           name = tf.name
    793         else:
    794           name = "%s (%s)" % (tf.name, sf.name)
    795         if patch is None:
    796           print "patching failed!                                  %s" % (name,)
    797         else:
    798           print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
    799               dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
    800       lock.release()
    801     except Exception, e:
    802       print e
    803       raise
    804 
    805   # start worker threads; wait for them all to finish.
    806   threads = [threading.Thread(target=worker)
    807              for i in range(OPTIONS.worker_threads)]
    808   for th in threads:
    809     th.start()
    810   while threads:
    811     threads.pop().join()
    812 
    813 
    814 # map recovery.fstab's fs_types to mount/format "partition types"
    815 PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
    816                     "ext4": "EMMC", "emmc": "EMMC" }
    817 
    818 def GetTypeAndDevice(mount_point, info):
    819   fstab = info["fstab"]
    820   if fstab:
    821     return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
    822   else:
    823     return None
    824