Home | History | Annotate | Download | only in gtest 
      1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import logging
      6 import os
      7 import re
      8 
      9 from pylib import android_commands
     10 from pylib import constants
     11 from pylib import pexpect
     12 from pylib.base import base_test_result
     13 from pylib.base import base_test_runner
     14 from pylib.perf import perf_control
     15 
     16 
     17 def _TestSuiteRequiresMockTestServer(suite_name):
     18   """Returns True if the test suite requires mock test server."""
     19   tests_require_net_test_server = ['unit_tests', 'net_unittests',
     20                                    'content_unittests',
     21                                    'content_browsertests']
     22   return (suite_name in
     23           tests_require_net_test_server)
     24 
     25 def _TestSuiteRequiresHighPerfMode(suite_name):
     26   """Returns True if the test suite requires high performance mode."""
     27   return 'perftests' in suite_name
     28 
     29 class TestRunner(base_test_runner.BaseTestRunner):
     30   def __init__(self, test_options, device, test_package):
     31     """Single test suite attached to a single device.
     32 
     33     Args:
     34       test_options: A GTestOptions object.
     35       device: Device to run the tests.
     36       test_package: An instance of TestPackage class.
     37     """
     38 
     39     super(TestRunner, self).__init__(device, test_options.tool,
     40                                      test_options.push_deps,
     41                                      test_options.cleanup_test_files)
     42 
     43     self.test_package = test_package
     44     self.test_package.tool = self.tool
     45     self._test_arguments = test_options.test_arguments
     46 
     47     timeout = test_options.timeout
     48     if timeout == 0:
     49       timeout = 60
     50     # On a VM (e.g. chromium buildbots), this timeout is way too small.
     51     if os.environ.get('BUILDBOT_SLAVENAME'):
     52       timeout = timeout * 2
     53 
     54     self._timeout = timeout * self.tool.GetTimeoutScale()
     55     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
     56       self._perf_controller = perf_control.PerfControl(self.adb)
     57 
     58   #override
     59   def InstallTestPackage(self):
     60     self.test_package.Install(self.adb)
     61 
     62   def GetAllTests(self):
     63     """Install test package and get a list of all tests."""
     64     self.test_package.Install(self.adb)
     65     return self.test_package.GetAllTests(self.adb)
     66 
     67   #override
     68   def PushDataDeps(self):
     69     self.adb.WaitForSdCardReady(20)
     70     self.tool.CopyFiles()
     71     if os.path.exists(constants.ISOLATE_DEPS_DIR):
     72       device_dir = self.adb.GetExternalStorage()
     73       # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir
     74       # as breakpad_unittests exe. Find a better way to do this.
     75       if self.test_package.suite_name == 'breakpad_unittests':
     76         device_dir = constants.TEST_EXECUTABLE_DIR
     77       for p in os.listdir(constants.ISOLATE_DEPS_DIR):
     78         self.adb.PushIfNeeded(
     79             os.path.join(constants.ISOLATE_DEPS_DIR, p),
     80             os.path.join(device_dir, p))
     81 
     82   def _ParseTestOutput(self, p):
     83     """Process the test output.
     84 
     85     Args:
     86       p: An instance of pexpect spawn class.
     87 
     88     Returns:
     89       A TestRunResults object.
     90     """
     91     results = base_test_result.TestRunResults()
     92 
     93     # Test case statuses.
     94     re_run = re.compile('\[ RUN      \] ?(.*)\r\n')
     95     re_fail = re.compile('\[  FAILED  \] ?(.*)\r\n')
     96     re_ok = re.compile('\[       OK \] ?(.*?) .*\r\n')
     97 
     98     # Test run statuses.
     99     re_passed = re.compile('\[  PASSED  \] ?(.*)\r\n')
    100     re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
    101     # Signal handlers are installed before starting tests
    102     # to output the CRASHED marker when a crash happens.
    103     re_crash = re.compile('\[ CRASHED      \](.*)\r\n')
    104 
    105     log = ''
    106     try:
    107       while True:
    108         full_test_name = None
    109         found = p.expect([re_run, re_passed, re_runner_fail],
    110                          timeout=self._timeout)
    111         if found == 1:  # re_passed
    112           break
    113         elif found == 2:  # re_runner_fail
    114           break
    115         else:  # re_run
    116           full_test_name = p.match.group(1).replace('\r', '')
    117           found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout)
    118           log = p.before.replace('\r', '')
    119           if found == 0:  # re_ok
    120             if full_test_name == p.match.group(1).replace('\r', ''):
    121               results.AddResult(base_test_result.BaseTestResult(
    122                   full_test_name, base_test_result.ResultType.PASS,
    123                   log=log))
    124           elif found == 2:  # re_crash
    125             results.AddResult(base_test_result.BaseTestResult(
    126                 full_test_name, base_test_result.ResultType.CRASH,
    127                 log=log))
    128             break
    129           else:  # re_fail
    130             results.AddResult(base_test_result.BaseTestResult(
    131                 full_test_name, base_test_result.ResultType.FAIL, log=log))
    132     except pexpect.EOF:
    133       logging.error('Test terminated - EOF')
    134       # We're here because either the device went offline, or the test harness
    135       # crashed without outputting the CRASHED marker (crbug.com/175538).
    136       if not self.adb.IsOnline():
    137         raise android_commands.errors.DeviceUnresponsiveError(
    138             'Device %s went offline.' % self.device)
    139       if full_test_name:
    140         results.AddResult(base_test_result.BaseTestResult(
    141             full_test_name, base_test_result.ResultType.CRASH,
    142             log=p.before.replace('\r', '')))
    143     except pexpect.TIMEOUT:
    144       logging.error('Test terminated after %d second timeout.',
    145                     self._timeout)
    146       if full_test_name:
    147         results.AddResult(base_test_result.BaseTestResult(
    148             full_test_name, base_test_result.ResultType.TIMEOUT,
    149             log=p.before.replace('\r', '')))
    150     finally:
    151       p.close()
    152 
    153     ret_code = self.test_package.GetGTestReturnCode(self.adb)
    154     if ret_code:
    155       logging.critical(
    156           'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
    157           ret_code, p.before, p.after)
    158 
    159     return results
    160 
    161   #override
    162   def RunTest(self, test):
    163     test_results = base_test_result.TestRunResults()
    164     if not test:
    165       return test_results, None
    166 
    167     try:
    168       self.test_package.ClearApplicationState(self.adb)
    169       self.test_package.CreateCommandLineFileOnDevice(
    170           self.adb, test, self._test_arguments)
    171       test_results = self._ParseTestOutput(
    172           self.test_package.SpawnTestProcess(self.adb))
    173     finally:
    174       self.CleanupSpawningServerState()
    175     # Calculate unknown test results.
    176     all_tests = set(test.split(':'))
    177     all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
    178     unknown_tests = all_tests - all_tests_ran
    179     test_results.AddResults(
    180         [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN)
    181          for t in unknown_tests])
    182     retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
    183     return test_results, retry
    184 
    185   #override
    186   def SetUp(self):
    187     """Sets up necessary test enviroment for the test suite."""
    188     super(TestRunner, self).SetUp()
    189     if _TestSuiteRequiresMockTestServer(self.test_package.suite_name):
    190       self.LaunchChromeTestServerSpawner()
    191     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
    192       self._perf_controller.SetHighPerfMode()
    193     self.tool.SetupEnvironment()
    194 
    195   #override
    196   def TearDown(self):
    197     """Cleans up the test enviroment for the test suite."""
    198     if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
    199       self._perf_controller.RestoreOriginalPerfMode()
    200     self.test_package.ClearApplicationState(self.adb)
    201     self.tool.CleanUpEnvironment()
    202     super(TestRunner, self).TearDown()
    203