1 # Copyright 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import unittest 7 8 from telemetry import decorators 9 from telemetry.core import browser_finder 10 from telemetry.core import browser_options 11 from telemetry.core import command_line 12 from telemetry.core import discover 13 from telemetry.unittest import json_results 14 from telemetry.unittest import progress_reporter 15 16 17 class Config(object): 18 def __init__(self, top_level_dir, test_dirs, progress_reporters): 19 self._top_level_dir = top_level_dir 20 self._test_dirs = tuple(test_dirs) 21 self._progress_reporters = tuple(progress_reporters) 22 23 @property 24 def top_level_dir(self): 25 return self._top_level_dir 26 27 @property 28 def test_dirs(self): 29 return self._test_dirs 30 31 @property 32 def progress_reporters(self): 33 return self._progress_reporters 34 35 36 def Discover(start_dir, top_level_dir=None, pattern='test*.py'): 37 loader = unittest.defaultTestLoader 38 loader.suiteClass = progress_reporter.TestSuite 39 40 test_suites = [] 41 modules = discover.DiscoverModules(start_dir, top_level_dir, pattern) 42 for module in modules: 43 if hasattr(module, 'suite'): 44 suite = module.suite() 45 else: 46 suite = loader.loadTestsFromModule(module) 47 if suite.countTestCases(): 48 test_suites.append(suite) 49 return test_suites 50 51 52 def FilterSuite(suite, predicate): 53 new_suite = suite.__class__() 54 for test in suite: 55 if isinstance(test, unittest.TestSuite): 56 subsuite = FilterSuite(test, predicate) 57 if subsuite.countTestCases(): 58 new_suite.addTest(subsuite) 59 else: 60 assert isinstance(test, unittest.TestCase) 61 if predicate(test): 62 new_suite.addTest(test) 63 64 return new_suite 65 66 67 def DiscoverTests(search_dirs, top_level_dir, possible_browser, 68 selected_tests=None, selected_tests_are_exact=False, 69 run_disabled_tests=False): 70 def IsTestSelected(test): 71 if selected_tests: 72 found = False 73 for name in selected_tests: 74 if selected_tests_are_exact: 75 if name == test.id(): 76 found = True 77 else: 78 if name in test.id(): 79 found = True 80 if not found: 81 return False 82 if run_disabled_tests: 83 return True 84 # pylint: disable=W0212 85 if not hasattr(test, '_testMethodName'): 86 return True 87 method = getattr(test, test._testMethodName) 88 return decorators.IsEnabled(method, possible_browser) 89 90 wrapper_suite = progress_reporter.TestSuite() 91 for search_dir in search_dirs: 92 wrapper_suite.addTests(Discover(search_dir, top_level_dir, '*_unittest.py')) 93 return FilterSuite(wrapper_suite, IsTestSelected) 94 95 96 def RestoreLoggingLevel(func): 97 def _LoggingRestoreWrapper(*args, **kwargs): 98 # Cache the current logging level, this needs to be done before calling 99 # parser.parse_args, which changes logging level based on verbosity 100 # setting. 101 logging_level = logging.getLogger().getEffectiveLevel() 102 try: 103 return func(*args, **kwargs) 104 finally: 105 # Restore logging level, which may be changed in parser.parse_args. 106 logging.getLogger().setLevel(logging_level) 107 108 return _LoggingRestoreWrapper 109 110 111 config = None 112 113 114 class RunTestsCommand(command_line.OptparseCommand): 115 """Run unit tests""" 116 117 usage = '[test_name ...] [<options>]' 118 119 @classmethod 120 def CreateParser(cls): 121 options = browser_options.BrowserFinderOptions() 122 options.browser_type = 'any' 123 parser = options.CreateParser('%%prog %s' % cls.usage) 124 return parser 125 126 @classmethod 127 def AddCommandLineArgs(cls, parser): 128 parser.add_option('--repeat-count', type='int', default=1, 129 help='Repeats each a provided number of times.') 130 parser.add_option('-d', '--also-run-disabled-tests', 131 dest='run_disabled_tests', 132 action='store_true', default=False, 133 help='Ignore @Disabled and @Enabled restrictions.') 134 parser.add_option('--retry-limit', type='int', 135 help='Retry each failure up to N times' 136 ' to de-flake things.') 137 parser.add_option('--exact-test-filter', action='store_true', default=False, 138 help='Treat test filter as exact matches (default is ' 139 'substring matches).') 140 json_results.AddOptions(parser) 141 142 @classmethod 143 def ProcessCommandLineArgs(cls, parser, args): 144 if args.verbosity == 0: 145 logging.getLogger().setLevel(logging.WARN) 146 147 # We retry failures by default unless we're running a list of tests 148 # explicitly. 149 if args.retry_limit is None and not args.positional_args: 150 args.retry_limit = 3 151 152 try: 153 possible_browser = browser_finder.FindBrowser(args) 154 except browser_finder.BrowserFinderException, ex: 155 parser.error(ex) 156 157 if not possible_browser: 158 parser.error('No browser found of type %s. Cannot run tests.\n' 159 'Re-run with --browser=list to see ' 160 'available browser types.' % args.browser_type) 161 162 json_results.ValidateArgs(parser, args) 163 164 def Run(self, args): 165 possible_browser = browser_finder.FindBrowser(args) 166 167 test_suite, result = self.RunOneSuite(possible_browser, args) 168 169 results = [result] 170 171 failed_tests = json_results.FailedTestNames(test_suite, result) 172 retry_limit = args.retry_limit 173 174 while retry_limit and failed_tests: 175 args.positional_args = failed_tests 176 args.exact_test_filter = True 177 178 _, result = self.RunOneSuite(possible_browser, args) 179 results.append(result) 180 181 failed_tests = json_results.FailedTestNames(test_suite, result) 182 retry_limit -= 1 183 184 full_results = json_results.FullResults(args, test_suite, results) 185 json_results.WriteFullResultsIfNecessary(args, full_results) 186 187 err_occurred, err_str = json_results.UploadFullResultsIfNecessary( 188 args, full_results) 189 if err_occurred: 190 for line in err_str.splitlines(): 191 logging.error(line) 192 return 1 193 194 return json_results.ExitCodeFromFullResults(full_results) 195 196 def RunOneSuite(self, possible_browser, args): 197 test_suite = DiscoverTests(config.test_dirs, config.top_level_dir, 198 possible_browser, args.positional_args, 199 args.exact_test_filter, args.run_disabled_tests) 200 runner = progress_reporter.TestRunner() 201 result = runner.run(test_suite, config.progress_reporters, 202 args.repeat_count, args) 203 return test_suite, result 204 205 @classmethod 206 @RestoreLoggingLevel 207 def main(cls, args=None): 208 return super(RunTestsCommand, cls).main(args) 209
