Home | History | Annotate | Download | only in host_driven 
      1 # Copyright 2013 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Base class for host-driven test cases.
      6 
      7 This test case is intended to serve as the base class for any host-driven
      8 test cases. It is similar to the Python unitttest module in that test cases
      9 inherit from this class and add methods which will be run as tests.
     10 
     11 When a HostDrivenTestCase object is instantiated, its purpose is to run only one
     12 test method in the derived class. The test runner gives it the name of the test
     13 method the instance will run. The test runner calls SetUp with the device ID
     14 which the test method will run against. The test runner runs the test method
     15 itself, collecting the result, and calls TearDown.
     16 
     17 Tests can perform arbitrary Python commands and asserts in test methods. Tests
     18 that run instrumentation tests can make use of the _RunJavaTestFilters helper
     19 function to trigger Java tests and convert results into a single host-driven
     20 test result.
     21 """
     22 
     23 import logging
     24 import os
     25 import time
     26 
     27 from pylib import constants
     28 from pylib import forwarder
     29 from pylib import valgrind_tools
     30 from pylib.base import base_test_result
     31 from pylib.device import device_utils
     32 from pylib.instrumentation import test_package
     33 from pylib.instrumentation import test_result
     34 from pylib.instrumentation import test_runner
     35 
     36 # aka the parent of com.google.android
     37 BASE_ROOT = 'src' + os.sep
     38 
     39 
     40 class HostDrivenTestCase(object):
     41   """Base class for host-driven test cases."""
     42 
     43   _HOST_DRIVEN_TAG = 'HostDriven'
     44 
     45   def __init__(self, test_name, instrumentation_options=None):
     46     """Create a test case initialized to run |test_name|.
     47 
     48     Args:
     49       test_name: The name of the method to run as the test.
     50       instrumentation_options: An InstrumentationOptions object.
     51     """
     52     class_name = self.__class__.__name__
     53     self.adb = None
     54     self.cleanup_test_files = False
     55     self.device = None
     56     self.device_id = ''
     57     self.has_forwarded_ports = False
     58     self.instrumentation_options = instrumentation_options
     59     self.ports_to_forward = []
     60     self.push_deps = False
     61     self.shard_index = 0
     62 
     63     # Use tagged_name when creating results, so that we can identify host-driven
     64     # tests in the overall results.
     65     self.test_name = test_name
     66     self.qualified_name = '%s.%s' % (class_name, self.test_name)
     67     self.tagged_name = '%s_%s' % (self._HOST_DRIVEN_TAG, self.qualified_name)
     68 
     69   # TODO(bulach): make ports_to_forward not optional and move the Forwarder
     70   # mapping here.
     71   def SetUp(self, device, shard_index, push_deps,
     72             cleanup_test_files, ports_to_forward=None):
     73     if not ports_to_forward:
     74       ports_to_forward = []
     75     self.device_id = device
     76     self.shard_index = shard_index
     77     self.device = device_utils.DeviceUtils(self.device_id)
     78     self.adb = self.device.old_interface
     79     self.push_deps = push_deps
     80     self.cleanup_test_files = cleanup_test_files
     81     if ports_to_forward:
     82       self.ports_to_forward = ports_to_forward
     83 
     84   def TearDown(self):
     85     pass
     86 
     87   # TODO(craigdh): Remove GetOutDir once references have been removed
     88   # downstream.
     89   @staticmethod
     90   def GetOutDir():
     91     return constants.GetOutDirectory()
     92 
     93   def Run(self):
     94     logging.info('Running host-driven test: %s', self.tagged_name)
     95     # Get the test method on the derived class and execute it
     96     return getattr(self, self.test_name)()
     97 
     98   @staticmethod
     99   def __GetHostForwarderLog():
    100     return ('-- Begin Full HostForwarder log\n'
    101             '%s\n'
    102             '--End Full HostForwarder log\n' % forwarder.Forwarder.GetHostLog())
    103 
    104   def __StartForwarder(self):
    105     logging.warning('Forwarding %s %s', self.ports_to_forward,
    106                     self.has_forwarded_ports)
    107     if self.ports_to_forward and not self.has_forwarded_ports:
    108       self.has_forwarded_ports = True
    109       tool = valgrind_tools.CreateTool(None, self.device)
    110       forwarder.Forwarder.Map([(port, port) for port in self.ports_to_forward],
    111                               self.device, tool)
    112 
    113   def __RunJavaTest(self, test, test_pkg, additional_flags=None):
    114     """Runs a single Java test in a Java TestRunner.
    115 
    116     Args:
    117       test: Fully qualified test name (ex. foo.bar.TestClass#testMethod)
    118       test_pkg: TestPackage object.
    119       additional_flags: A list of additional flags to add to the command line.
    120 
    121     Returns:
    122       TestRunResults object with a single test result.
    123     """
    124     # TODO(bulach): move this to SetUp() stage.
    125     self.__StartForwarder()
    126 
    127     java_test_runner = test_runner.TestRunner(self.instrumentation_options,
    128                                               self.device_id,
    129                                               self.shard_index, test_pkg,
    130                                               additional_flags=additional_flags)
    131     try:
    132       java_test_runner.SetUp()
    133       return java_test_runner.RunTest(test)[0]
    134     finally:
    135       java_test_runner.TearDown()
    136 
    137   def _RunJavaTestFilters(self, test_filters, additional_flags=None):
    138     """Calls a list of tests and stops at the first test failure.
    139 
    140     This method iterates until either it encounters a non-passing test or it
    141     exhausts the list of tests. Then it returns the appropriate overall result.
    142 
    143     Test cases may make use of this method internally to assist in running
    144     instrumentation tests. This function relies on instrumentation_options
    145     being defined.
    146 
    147     Args:
    148       test_filters: A list of Java test filters.
    149       additional_flags: A list of addition flags to add to the command line.
    150 
    151     Returns:
    152       A TestRunResults object containing an overall result for this set of Java
    153       tests. If any Java tests do not pass, this is a fail overall.
    154     """
    155     test_type = base_test_result.ResultType.PASS
    156     log = ''
    157 
    158     test_pkg = test_package.TestPackage(
    159         self.instrumentation_options.test_apk_path,
    160         self.instrumentation_options.test_apk_jar_path,
    161         self.instrumentation_options.test_support_apk_path)
    162 
    163     start_ms = int(time.time()) * 1000
    164     done = False
    165     for test_filter in test_filters:
    166       tests = test_pkg.GetAllMatchingTests(None, None, test_filter)
    167       # Filters should always result in >= 1 test.
    168       if len(tests) == 0:
    169         raise Exception('Java test filter "%s" returned no tests.'
    170                         % test_filter)
    171       for test in tests:
    172         # We're only running one test at a time, so this TestRunResults object
    173         # will hold only one result.
    174         java_result = self.__RunJavaTest(test, test_pkg, additional_flags)
    175         assert len(java_result.GetAll()) == 1
    176         if not java_result.DidRunPass():
    177           result = java_result.GetNotPass().pop()
    178           log = result.GetLog()
    179           log += self.__GetHostForwarderLog()
    180           test_type = result.GetType()
    181           done = True
    182           break
    183       if done:
    184         break
    185     duration_ms = int(time.time()) * 1000 - start_ms
    186 
    187     overall_result = base_test_result.TestRunResults()
    188     overall_result.AddResult(
    189         test_result.InstrumentationTestResult(
    190             self.tagged_name, test_type, start_ms, duration_ms, log=log))
    191     return overall_result
    192 
    193   def __str__(self):
    194     return self.tagged_name
    195 
    196   def __repr__(self):
    197     return self.tagged_name
    198