Home | History | Annotate | Download | only in overlaytests
      1 #!/usr/bin/python
      2 import hashlib
      3 import optparse
      4 import os
      5 import re
      6 import shlex
      7 import subprocess
      8 import sys
      9 import threading
     10 import time
     11 
     12 TASK_COMPILATION = 'compile'
     13 TASK_DISABLE_OVERLAYS = 'disable overlays'
     14 TASK_ENABLE_MULTIPLE_OVERLAYS = 'enable multiple overlays'
     15 TASK_ENABLE_SINGLE_OVERLAY = 'enable single overlay'
     16 TASK_FILE_EXISTS_TEST = 'test (file exists)'
     17 TASK_GREP_IDMAP_TEST = 'test (grep idmap)'
     18 TASK_MD5_TEST = 'test (md5)'
     19 TASK_IDMAP_PATH = 'idmap --path'
     20 TASK_IDMAP_SCAN = 'idmap --scan'
     21 TASK_INSTRUMENTATION = 'instrumentation'
     22 TASK_INSTRUMENTATION_TEST = 'test (instrumentation)'
     23 TASK_MKDIR = 'mkdir'
     24 TASK_PUSH = 'push'
     25 TASK_ROOT = 'root'
     26 TASK_REMOUNT = 'remount'
     27 TASK_RM = 'rm'
     28 TASK_SETUP_IDMAP_PATH = 'setup idmap --path'
     29 TASK_SETUP_IDMAP_SCAN = 'setup idmap --scan'
     30 TASK_START = 'start'
     31 TASK_STOP = 'stop'
     32 
     33 adb = 'adb'
     34 
     35 def _adb_shell(cmd):
     36     argv = shlex.split(adb + " shell '" + cmd + "; echo $?'")
     37     proc = subprocess.Popen(argv, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     38     (stdout, stderr) = proc.communicate()
     39     (stdout, stderr) = (stdout.replace('\r', ''), stderr.replace('\r', ''))
     40     tmp = stdout.rsplit('\n', 2)
     41     if len(tmp) == 2:
     42         stdout == ''
     43         returncode = int(tmp[0])
     44     else:
     45         stdout = tmp[0] + '\n'
     46         returncode = int(tmp[1])
     47     return returncode, stdout, stderr
     48 
     49 class VerbosePrinter:
     50     class Ticker(threading.Thread):
     51         def _print(self):
     52             s = '\r' + self.text + '[' + '.' * self.i + ' ' * (4 - self.i) + ']'
     53             sys.stdout.write(s)
     54             sys.stdout.flush()
     55             self.i = (self.i + 1) % 5
     56 
     57         def __init__(self, cond_var, text):
     58             threading.Thread.__init__(self)
     59             self.text = text
     60             self.setDaemon(True)
     61             self.cond_var = cond_var
     62             self.running = False
     63             self.i = 0
     64             self._print()
     65             self.running = True
     66 
     67         def run(self):
     68             self.cond_var.acquire()
     69             while True:
     70                 self.cond_var.wait(0.25)
     71                 running = self.running
     72                 if not running:
     73                     break
     74                 self._print()
     75             self.cond_var.release()
     76 
     77         def stop(self):
     78             self.cond_var.acquire()
     79             self.running = False
     80             self.cond_var.notify_all()
     81             self.cond_var.release()
     82 
     83     def _start_ticker(self):
     84         self.ticker = VerbosePrinter.Ticker(self.cond_var, self.text)
     85         self.ticker.start()
     86 
     87     def _stop_ticker(self):
     88         self.ticker.stop()
     89         self.ticker.join()
     90         self.ticker = None
     91 
     92     def _format_begin(self, type, name):
     93         N = self.width - len(type) - len(' [    ] ')
     94         fmt = '%%s %%-%ds ' % N
     95         return fmt % (type, name)
     96 
     97     def __init__(self, use_color):
     98         self.cond_var = threading.Condition()
     99         self.ticker = None
    100         if use_color:
    101             self.color_RED = '\033[1;31m'
    102             self.color_red = '\033[0;31m'
    103             self.color_reset = '\033[0;37m'
    104         else:
    105             self.color_RED = ''
    106             self.color_red = ''
    107             self.color_reset = ''
    108 
    109         argv = shlex.split('stty size') # get terminal width
    110         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    111         (stdout, stderr) = proc.communicate()
    112         if proc.returncode == 0:
    113             (h, w) = stdout.split()
    114             self.width = int(w)
    115         else:
    116             self.width = 72 # conservative guesstimate
    117 
    118     def begin(self, type, name):
    119         self.text = self._format_begin(type, name)
    120         sys.stdout.write(self.text + '[    ]')
    121         sys.stdout.flush()
    122         self._start_ticker()
    123 
    124     def end_pass(self, type, name):
    125         self._stop_ticker()
    126         sys.stdout.write('\r' + self.text + '[ OK ]\n')
    127         sys.stdout.flush()
    128 
    129     def end_fail(self, type, name, msg):
    130         self._stop_ticker()
    131         sys.stdout.write('\r' + self.color_RED + self.text + '[FAIL]\n')
    132         sys.stdout.write(self.color_red)
    133         sys.stdout.write(msg)
    134         sys.stdout.write(self.color_reset)
    135         sys.stdout.flush()
    136 
    137 class QuietPrinter:
    138     def begin(self, type, name):
    139         pass
    140 
    141     def end_pass(self, type, name):
    142         sys.stdout.write('PASS ' + type + ' ' + name + '\n')
    143         sys.stdout.flush()
    144 
    145     def end_fail(self, type, name, msg):
    146         sys.stdout.write('FAIL ' + type + ' ' + name + '\n')
    147         sys.stdout.flush()
    148 
    149 class CompilationTask:
    150     def __init__(self, makefile):
    151         self.makefile = makefile
    152 
    153     def get_type(self):
    154         return TASK_COMPILATION
    155 
    156     def get_name(self):
    157         return self.makefile
    158 
    159     def execute(self):
    160         os.putenv('ONE_SHOT_MAKEFILE', os.getcwd() + "/" + self.makefile)
    161         argv = shlex.split('make -C "../../../../../" files')
    162         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    163         (stdout, stderr) = proc.communicate()
    164         return proc.returncode, stdout, stderr
    165 
    166 class InstrumentationTask:
    167     def __init__(self, instrumentation_class):
    168         self.instrumentation_class = instrumentation_class
    169 
    170     def get_type(self):
    171         return TASK_INSTRUMENTATION
    172 
    173     def get_name(self):
    174         return self.instrumentation_class
    175 
    176     def execute(self):
    177         return _adb_shell('am instrument -r -w -e class %s com.android.overlaytest/android.test.InstrumentationTestRunner' % self.instrumentation_class)
    178 
    179 class PushTask:
    180     def __init__(self, src, dest):
    181         self.src = src
    182         self.dest = dest
    183 
    184     def get_type(self):
    185         return TASK_PUSH
    186 
    187     def get_name(self):
    188         return "%s -> %s" % (self.src, self.dest)
    189 
    190     def execute(self):
    191         src = os.getenv('OUT') + "/" + self.src
    192         argv = shlex.split(adb + ' push %s %s' % (src, self.dest))
    193         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    194         (stdout, stderr) = proc.communicate()
    195         return proc.returncode, stdout, stderr
    196 
    197 class MkdirTask:
    198     def __init__(self, path):
    199         self.path = path
    200 
    201     def get_type(self):
    202         return TASK_MKDIR
    203 
    204     def get_name(self):
    205         return self.path
    206 
    207     def execute(self):
    208         return _adb_shell('mkdir -p %s' % self.path)
    209 
    210 class RmTask:
    211     def __init__(self, path):
    212         self.path = path
    213 
    214     def get_type(self):
    215         return TASK_RM
    216 
    217     def get_name(self):
    218         return self.path
    219 
    220     def execute(self):
    221         returncode, stdout, stderr = _adb_shell('ls %s' % self.path)
    222         if returncode != 0 and stdout.endswith(': No such file or directory\n'):
    223             return 0, "", ""
    224         return _adb_shell('rm -r %s' % self.path)
    225 
    226 class IdmapPathTask:
    227     def __init__(self, path_target_apk, path_overlay_apk, path_idmap):
    228         self.path_target_apk = path_target_apk
    229         self.path_overlay_apk = path_overlay_apk
    230         self.path_idmap = path_idmap
    231 
    232     def get_type(self):
    233         return TASK_IDMAP_PATH
    234 
    235     def get_name(self):
    236         return self.path_idmap
    237 
    238     def execute(self):
    239         return _adb_shell('su system idmap --path "%s" "%s" "%s"' % (self.path_target_apk, self.path_overlay_apk, self.path_idmap))
    240 
    241 class IdmapScanTask:
    242     def __init__(self, overlay_dir, target_pkg_name, target_pkg, idmap_dir, symlink_dir):
    243         self.overlay_dir = overlay_dir
    244         self.target_pkg_name = target_pkg_name
    245         self.target_pkg = target_pkg
    246         self.idmap_dir = idmap_dir
    247         self.symlink_dir = symlink_dir
    248 
    249     def get_type(self):
    250         return TASK_IDMAP_SCAN
    251 
    252     def get_name(self):
    253         return self.target_pkg_name
    254 
    255     def execute(self):
    256         return _adb_shell('su system idmap --scan "%s" "%s" "%s" "%s"' % (self.overlay_dir, self.target_pkg_name, self.target_pkg, self.idmap_dir))
    257 
    258 class FileExistsTest:
    259     def __init__(self, path):
    260         self.path = path
    261 
    262     def get_type(self):
    263         return TASK_FILE_EXISTS_TEST
    264 
    265     def get_name(self):
    266         return self.path
    267 
    268     def execute(self):
    269         return _adb_shell('ls %s' % self.path)
    270 
    271 class GrepIdmapTest:
    272     def __init__(self, path_idmap, pattern, expected_n):
    273         self.path_idmap = path_idmap
    274         self.pattern = pattern
    275         self.expected_n = expected_n
    276 
    277     def get_type(self):
    278         return TASK_GREP_IDMAP_TEST
    279 
    280     def get_name(self):
    281         return self.pattern
    282 
    283     def execute(self):
    284         returncode, stdout, stderr = _adb_shell('idmap --inspect %s' % self.path_idmap)
    285         if returncode != 0:
    286             return returncode, stdout, stderr
    287         all_matches = re.findall('\s' + self.pattern + '$', stdout, flags=re.MULTILINE)
    288         if len(all_matches) != self.expected_n:
    289             return 1, 'pattern=%s idmap=%s expected=%d found=%d\n' % (self.pattern, self.path_idmap, self.expected_n, len(all_matches)), ''
    290         return 0, "", ""
    291 
    292 class Md5Test:
    293     def __init__(self, path, expected_content):
    294         self.path = path
    295         self.expected_md5 = hashlib.md5(expected_content).hexdigest()
    296 
    297     def get_type(self):
    298         return TASK_MD5_TEST
    299 
    300     def get_name(self):
    301         return self.path
    302 
    303     def execute(self):
    304         returncode, stdout, stderr = _adb_shell('md5 %s' % self.path)
    305         if returncode != 0:
    306             return returncode, stdout, stderr
    307         actual_md5 = stdout.split()[0]
    308         if actual_md5 != self.expected_md5:
    309             return 1, 'expected %s, got %s\n' % (self.expected_md5, actual_md5), ''
    310         return 0, "", ""
    311 
    312 class StartTask:
    313     def get_type(self):
    314         return TASK_START
    315 
    316     def get_name(self):
    317         return ""
    318 
    319     def execute(self):
    320         (returncode, stdout, stderr) = _adb_shell('start')
    321         if returncode != 0:
    322             return returncode, stdout, stderr
    323 
    324         while True:
    325             (returncode, stdout, stderr) = _adb_shell('getprop dev.bootcomplete')
    326             if returncode != 0:
    327                 return returncode, stdout, stderr
    328             if stdout.strip() == "1":
    329                 break
    330             time.sleep(0.5)
    331 
    332         return 0, "", ""
    333 
    334 class StopTask:
    335     def get_type(self):
    336         return TASK_STOP
    337 
    338     def get_name(self):
    339         return ""
    340 
    341     def execute(self):
    342         (returncode, stdout, stderr) = _adb_shell('stop')
    343         if returncode != 0:
    344             return returncode, stdout, stderr
    345         return _adb_shell('setprop dev.bootcomplete 0')
    346 
    347 class RootTask:
    348     def get_type(self):
    349         return TASK_ROOT
    350 
    351     def get_name(self):
    352         return ""
    353 
    354     def execute(self):
    355         (returncode, stdout, stderr) = _adb_shell('getprop service.adb.root 0')
    356         if returncode != 0:
    357             return returncode, stdout, stderr
    358         if stdout.strip() == '1': # already root
    359             return 0, "", ""
    360 
    361         argv = shlex.split(adb + ' root')
    362         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    363         (stdout, stderr) = proc.communicate()
    364         if proc.returncode != 0:
    365             return proc.returncode, stdout, stderr
    366 
    367         argv = shlex.split(adb + ' wait-for-device')
    368         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    369         (stdout, stderr) = proc.communicate()
    370         return proc.returncode, stdout, stderr
    371 
    372 class RemountTask:
    373     def get_type(self):
    374         return TASK_REMOUNT
    375 
    376     def get_name(self):
    377         return ""
    378 
    379     def execute(self):
    380         argv = shlex.split(adb + ' remount')
    381         proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    382         (stdout, stderr) = proc.communicate()
    383         # adb remount returns 0 even if the operation failed, so check stdout
    384         if stdout.startswith('remount failed:'):
    385             return 1, stdout, stderr
    386         return proc.returncode, stdout, stderr
    387 
    388 class CompoundTask:
    389     def __init__(self, type, tasks):
    390         self.type = type
    391         self.tasks = tasks
    392 
    393     def get_type(self):
    394         return self.type
    395 
    396     def get_name(self):
    397         return ""
    398 
    399     def execute(self):
    400         for t in self.tasks:
    401             (returncode, stdout, stderr) = t.execute()
    402             if returncode != 0:
    403                 return returncode, stdout, stderr
    404         return 0, "", ""
    405 
    406 def _create_disable_overlays_task():
    407     tasks = [
    408         RmTask("/vendor/overlay/framework_a.apk"),
    409         RmTask("/vendor/overlay/framework_b.apk"),
    410         RmTask("/data/resource-cache/vendor@overlay (at] framework_a.apk@idmap"),
    411         RmTask("/data/resource-cache/vendor@overlay (at] framework_b.apk@idmap"),
    412         RmTask("/vendor/overlay/app_a.apk"),
    413         RmTask("/vendor/overlay/app_b.apk"),
    414         RmTask("/data/resource-cache/vendor@overlay (at] app_a.apk@idmap"),
    415         RmTask("/data/resource-cache/vendor@overlay (at] app_b.apk@idmap"),
    416     ]
    417     return CompoundTask(TASK_DISABLE_OVERLAYS, tasks)
    418 
    419 def _create_enable_single_overlay_task():
    420     tasks = [
    421         _create_disable_overlays_task(),
    422         MkdirTask('/system/vendor'),
    423         MkdirTask('/vendor/overlay'),
    424         PushTask('/data/app/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_a.apk'),
    425         PushTask('/data/app/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
    426     ]
    427     return CompoundTask(TASK_ENABLE_SINGLE_OVERLAY, tasks)
    428 
    429 def _create_enable_multiple_overlays_task():
    430     tasks = [
    431         _create_disable_overlays_task(),
    432         MkdirTask('/system/vendor'),
    433         MkdirTask('/vendor/overlay'),
    434 
    435         PushTask('/data/app/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_b.apk'),
    436         PushTask('/data/app/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
    437         PushTask('/data/app/com.android.overlaytest.second_app_overlay.apk', '/vendor/overlay/app_b.apk'),
    438     ]
    439     return CompoundTask(TASK_ENABLE_MULTIPLE_OVERLAYS, tasks)
    440 
    441 def _create_setup_idmap_path_task(idmaps, symlinks):
    442     tasks = [
    443         _create_enable_single_overlay_task(),
    444         RmTask(symlinks),
    445         RmTask(idmaps),
    446         MkdirTask(idmaps),
    447         MkdirTask(symlinks),
    448     ]
    449     return CompoundTask(TASK_SETUP_IDMAP_PATH, tasks)
    450 
    451 def _create_setup_idmap_scan_task(idmaps, symlinks):
    452     tasks = [
    453         _create_enable_single_overlay_task(),
    454         RmTask(symlinks),
    455         RmTask(idmaps),
    456         MkdirTask(idmaps),
    457         MkdirTask(symlinks),
    458         _create_enable_multiple_overlays_task(),
    459     ]
    460     return CompoundTask(TASK_SETUP_IDMAP_SCAN, tasks)
    461 
    462 def _handle_instrumentation_task_output(stdout, printer):
    463     regex_status_code = re.compile(r'^INSTRUMENTATION_STATUS_CODE: -?(\d+)')
    464     regex_name = re.compile(r'^INSTRUMENTATION_STATUS: test=(.*)')
    465     regex_begin_stack = re.compile(r'^INSTRUMENTATION_STATUS: stack=(.*)')
    466     regex_end_stack = re.compile(r'^$')
    467 
    468     failed_tests = 0
    469     current_test = None
    470     current_stack = []
    471     mode_stack = False
    472     for line in stdout.split("\n"):
    473         line = line.rstrip() # strip \r from adb output
    474         m = regex_status_code.match(line)
    475         if m:
    476             c = int(m.group(1))
    477             if c == 1:
    478                 printer.begin(TASK_INSTRUMENTATION_TEST, current_test)
    479             elif c == 0:
    480                 printer.end_pass(TASK_INSTRUMENTATION_TEST, current_test)
    481             else:
    482                 failed_tests += 1
    483                 current_stack.append("\n")
    484                 msg = "\n".join(current_stack)
    485                 printer.end_fail(TASK_INSTRUMENTATION_TEST, current_test, msg.rstrip() + '\n')
    486             continue
    487 
    488         m = regex_name.match(line)
    489         if m:
    490             current_test = m.group(1)
    491             continue
    492 
    493         m = regex_begin_stack.match(line)
    494         if m:
    495             mode_stack = True
    496             current_stack = []
    497             current_stack.append("  " + m.group(1))
    498             continue
    499 
    500         m = regex_end_stack.match(line)
    501         if m:
    502             mode_stack = False
    503             continue
    504 
    505         if mode_stack:
    506             current_stack.append("    " + line.strip())
    507 
    508     return failed_tests
    509 
    510 def _set_adb_device(option, opt, value, parser):
    511     global adb
    512     if opt == '-d' or opt == '--device':
    513         adb = 'adb -d'
    514     if opt == '-e' or opt == '--emulator':
    515         adb = 'adb -e'
    516     if opt == '-s' or opt == '--serial':
    517         adb = 'adb -s ' + value
    518 
    519 def _create_opt_parser():
    520     parser = optparse.OptionParser()
    521     parser.add_option('-d', '--device', action='callback', callback=_set_adb_device,
    522             help='pass -d to adb')
    523     parser.add_option('-e', '--emulator', action='callback', callback=_set_adb_device,
    524             help='pass -e to adb')
    525     parser.add_option('-s', '--serial', type="str", action='callback', callback=_set_adb_device,
    526             help='pass -s <serical> to adb')
    527     parser.add_option('-C', '--no-color', action='store_false',
    528             dest='use_color', default=True,
    529             help='disable color escape sequences in output')
    530     parser.add_option('-q', '--quiet', action='store_true',
    531             dest='quiet_mode', default=False,
    532             help='quiet mode, output only results')
    533     parser.add_option('-b', '--no-build', action='store_false',
    534             dest='do_build', default=True,
    535             help='do not rebuild test projects')
    536     parser.add_option('-k', '--continue', action='store_true',
    537             dest='do_continue', default=False,
    538             help='do not rebuild test projects')
    539     parser.add_option('-i', '--test-idmap', action='store_true',
    540             dest='test_idmap', default=False,
    541             help='run tests for single overlay')
    542     parser.add_option('-0', '--test-no-overlay', action='store_true',
    543             dest='test_no_overlay', default=False,
    544             help='run tests without any overlay')
    545     parser.add_option('-1', '--test-single-overlay', action='store_true',
    546             dest='test_single_overlay', default=False,
    547             help='run tests for single overlay')
    548     parser.add_option('-2', '--test-multiple-overlays', action='store_true',
    549             dest='test_multiple_overlays', default=False,
    550             help='run tests for multiple overlays')
    551     return parser
    552 
    553 if __name__ == '__main__':
    554     opt_parser = _create_opt_parser()
    555     opts, args = opt_parser.parse_args(sys.argv[1:])
    556     if not opts.test_idmap and not opts.test_no_overlay and not opts.test_single_overlay and not opts.test_multiple_overlays:
    557         opts.test_idmap = True
    558         opts.test_no_overlay = True
    559         opts.test_single_overlay = True
    560         opts.test_multiple_overlays = True
    561     if len(args) > 0:
    562         opt_parser.error("unexpected arguments: %s" % " ".join(args))
    563         # will never reach this: opt_parser.error will call sys.exit
    564 
    565     if opts.quiet_mode:
    566         printer = QuietPrinter()
    567     else:
    568         printer = VerbosePrinter(opts.use_color)
    569     tasks = []
    570 
    571     # must be in the same directory as this script for compilation tasks to work
    572     script = sys.argv[0]
    573     dirname = os.path.dirname(script)
    574     wd = os.path.realpath(dirname)
    575     os.chdir(wd)
    576 
    577     # build test cases
    578     if opts.do_build:
    579         tasks.append(CompilationTask('OverlayTest/Android.mk'))
    580         tasks.append(CompilationTask('OverlayTestOverlay/Android.mk'))
    581         tasks.append(CompilationTask('OverlayAppFirst/Android.mk'))
    582         tasks.append(CompilationTask('OverlayAppSecond/Android.mk'))
    583 
    584     # remount filesystem, install test project
    585     tasks.append(RootTask())
    586     tasks.append(RemountTask())
    587     tasks.append(PushTask('/system/app/OverlayTest.apk', '/system/app/OverlayTest.apk'))
    588 
    589     # test idmap
    590     if opts.test_idmap:
    591         idmaps='/data/local/tmp/idmaps'
    592         symlinks='/data/local/tmp/symlinks'
    593 
    594         # idmap --path
    595         tasks.append(StopTask())
    596         tasks.append(_create_setup_idmap_path_task(idmaps, symlinks))
    597         tasks.append(StartTask())
    598         tasks.append(IdmapPathTask('/vendor/overlay/framework_a.apk', '/system/framework/framework-res.apk', idmaps + '/a.idmap'))
    599         tasks.append(FileExistsTest(idmaps + '/a.idmap'))
    600         tasks.append(GrepIdmapTest(idmaps + '/a.idmap', 'bool/config_annoy_dianne', 1))
    601 
    602         # idmap --scan
    603         idmap = idmaps + '/vendor@overlay (at] framework_b.apk@idmap'
    604         tasks.append(StopTask())
    605         tasks.append(_create_setup_idmap_scan_task(idmaps, symlinks))
    606         tasks.append(StartTask())
    607         tasks.append(IdmapScanTask('/vendor/overlay', 'android', '/system/framework/framework-res.apk', idmaps, symlinks))
    608         tasks.append(FileExistsTest(idmap))
    609         tasks.append(GrepIdmapTest(idmap, 'bool/config_annoy_dianne', 1))
    610 
    611         # overlays.list
    612         overlays_list_path = '/data/resource-cache/overlays.list'
    613         expected_content = '''\
    614 /vendor/overlay/framework_b.apk /data/resource-cache/vendor@overlay (at] framework_b.apk@idmap
    615 '''
    616         tasks.append(FileExistsTest(overlays_list_path))
    617         tasks.append(Md5Test(overlays_list_path, expected_content))
    618 
    619         # idmap cleanup
    620         tasks.append(RmTask(symlinks))
    621         tasks.append(RmTask(idmaps))
    622 
    623     # test no overlay
    624     if opts.test_no_overlay:
    625         tasks.append(StopTask())
    626         tasks.append(_create_disable_overlays_task())
    627         tasks.append(StartTask())
    628         tasks.append(InstrumentationTask('com.android.overlaytest.WithoutOverlayTest'))
    629 
    630     # test single overlay
    631     if opts.test_single_overlay:
    632         tasks.append(StopTask())
    633         tasks.append(_create_enable_single_overlay_task())
    634         tasks.append(StartTask())
    635         tasks.append(InstrumentationTask('com.android.overlaytest.WithOverlayTest'))
    636 
    637     # test multiple overlays
    638     if opts.test_multiple_overlays:
    639         tasks.append(StopTask())
    640         tasks.append(_create_enable_multiple_overlays_task())
    641         tasks.append(StartTask())
    642         tasks.append(InstrumentationTask('com.android.overlaytest.WithMultipleOverlaysTest'))
    643 
    644     ignored_errors = 0
    645     for t in tasks:
    646         type = t.get_type()
    647         name = t.get_name()
    648         if type == TASK_INSTRUMENTATION:
    649             # InstrumentationTask will run several tests, but we want it
    650             # to appear as if each test was run individually. Calling
    651             # "am instrument" with a single test method is prohibitively
    652             # expensive, so let's instead post-process the output to
    653             # emulate individual calls.
    654             retcode, stdout, stderr = t.execute()
    655             if retcode != 0:
    656                 printer.begin(TASK_INSTRUMENTATION, name)
    657                 printer.end_fail(TASK_INSTRUMENTATION, name, stderr)
    658                 sys.exit(retcode)
    659             retcode = _handle_instrumentation_task_output(stdout, printer)
    660             if retcode != 0:
    661                 if not opts.do_continue:
    662                     sys.exit(retcode)
    663                 else:
    664                     ignored_errors += retcode
    665         else:
    666             printer.begin(type, name)
    667             retcode, stdout, stderr = t.execute()
    668             if retcode == 0:
    669                 printer.end_pass(type, name)
    670             if retcode != 0:
    671                 if len(stderr) == 0:
    672                     # hope for output from stdout instead (true for eg adb shell rm)
    673                     stderr = stdout
    674                 printer.end_fail(type, name, stderr)
    675                 if not opts.do_continue:
    676                     sys.exit(retcode)
    677                 else:
    678                     ignored_errors += retcode
    679     sys.exit(ignored_errors)
    680