1 # Copyright (C) 2012 Google, Inc. 2 # Copyright (C) 2010 Chris Jerdonek (cjerdonek (at] webkit.org) 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions 6 # are met: 7 # 1. Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # 2. Redistributions in binary form must reproduce the above copyright 10 # notice, this list of conditions and the following disclaimer in the 11 # documentation and/or other materials provided with the distribution. 12 # 13 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND 14 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR 17 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 24 """unit testing code for webkitpy.""" 25 26 import logging 27 import multiprocessing 28 import optparse 29 import os 30 import StringIO 31 import sys 32 import time 33 import traceback 34 import unittest 35 36 from webkitpy.common.webkit_finder import WebKitFinder 37 from webkitpy.common.system.filesystem import FileSystem 38 from webkitpy.test.finder import Finder 39 from webkitpy.test.printer import Printer 40 from webkitpy.test.runner import Runner, unit_test_name 41 42 _log = logging.getLogger(__name__) 43 44 45 up = os.path.dirname 46 webkit_root = up(up(up(up(up(os.path.abspath(__file__)))))) 47 48 49 def main(): 50 filesystem = FileSystem() 51 wkf = WebKitFinder(filesystem) 52 tester = Tester(filesystem, wkf) 53 tester.add_tree(wkf.path_from_webkit_base('Tools', 'Scripts'), 'webkitpy') 54 55 tester.skip(('webkitpy.common.checkout.scm.scm_unittest',), 'are really, really, slow', 31818) 56 if sys.platform == 'win32': 57 tester.skip(('webkitpy.common.checkout', 'webkitpy.common.config', 'webkitpy.tool', 'webkitpy.w3c', 'webkitpy.layout_tests.layout_package.bot_test_expectations'), 'fail horribly on win32', 54526) 58 59 # This only needs to run on Unix, so don't worry about win32 for now. 60 appengine_sdk_path = '/usr/local/google_appengine' 61 if os.path.exists(appengine_sdk_path): 62 if not appengine_sdk_path in sys.path: 63 sys.path.append(appengine_sdk_path) 64 import dev_appserver 65 from google.appengine.dist import use_library 66 use_library('django', '1.2') 67 dev_appserver.fix_sys_path() 68 tester.add_tree(wkf.path_from_webkit_base('Tools', 'TestResultServer')) 69 else: 70 _log.info('Skipping TestResultServer tests; the Google AppEngine Python SDK is not installed.') 71 72 return not tester.run() 73 74 75 class Tester(object): 76 def __init__(self, filesystem=None, webkit_finder=None): 77 self.filesystem = filesystem or FileSystem() 78 self.finder = Finder(self.filesystem) 79 self.printer = Printer(sys.stderr) 80 self.webkit_finder = webkit_finder or WebKitFinder(self.filesystem) 81 self._options = None 82 83 def add_tree(self, top_directory, starting_subdirectory=None): 84 self.finder.add_tree(top_directory, starting_subdirectory) 85 86 def skip(self, names, reason, bugid): 87 self.finder.skip(names, reason, bugid) 88 89 def _parse_args(self, argv=None): 90 parser = optparse.OptionParser(usage='usage: %prog [options] [args...]') 91 parser.add_option('-a', '--all', action='store_true', default=False, 92 help='run all the tests') 93 parser.add_option('-c', '--coverage', action='store_true', default=False, 94 help='generate code coverage info') 95 parser.add_option('-i', '--integration-tests', action='store_true', default=False, 96 help='run integration tests as well as unit tests'), 97 parser.add_option('-j', '--child-processes', action='store', type='int', default=(1 if sys.platform == 'win32' else multiprocessing.cpu_count()), 98 help='number of tests to run in parallel (default=%default)') 99 parser.add_option('-p', '--pass-through', action='store_true', default=False, 100 help='be debugger friendly by passing captured output through to the system') 101 parser.add_option('-q', '--quiet', action='store_true', default=False, 102 help='run quietly (errors, warnings, and progress only)') 103 parser.add_option('-t', '--timing', action='store_true', default=False, 104 help='display per-test execution time (implies --verbose)') 105 parser.add_option('-v', '--verbose', action='count', default=0, 106 help='verbose output (specify once for individual test results, twice for debug messages)') 107 108 parser.epilog = ('[args...] is an optional list of modules, test_classes, or individual tests. ' 109 'If no args are given, all the tests will be run.') 110 111 return parser.parse_args(argv) 112 113 def run(self): 114 self._options, args = self._parse_args() 115 self.printer.configure(self._options) 116 117 self.finder.clean_trees() 118 119 names = self.finder.find_names(args, self._options.all) 120 if not names: 121 _log.error('No tests to run') 122 return False 123 124 return self._run_tests(names) 125 126 def _run_tests(self, names): 127 # Make sure PYTHONPATH is set up properly. 128 sys.path = self.finder.additional_paths(sys.path) + sys.path 129 130 # FIXME: unittest2 and coverage need to be in sys.path for their internal imports to work. 131 thirdparty_path = self.webkit_finder.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'thirdparty') 132 if not thirdparty_path in sys.path: 133 sys.path.append(thirdparty_path) 134 135 if self._options.coverage: 136 _log.warning("Checking code coverage, so running things serially") 137 self._options.child_processes = 1 138 139 import coverage 140 cov = coverage.coverage(omit=["/usr/*", "*/webkitpy/thirdparty/*", "/Library/*"]) 141 cov.start() 142 143 self.printer.write_update("Checking imports ...") 144 if not self._check_imports(names): 145 return False 146 147 self.printer.write_update("Finding the individual test methods ...") 148 loader = _Loader() 149 parallel_tests, serial_tests = self._test_names(loader, names) 150 151 self.printer.write_update("Running the tests ...") 152 self.printer.num_tests = len(parallel_tests) + len(serial_tests) 153 start = time.time() 154 test_runner = Runner(self.printer, loader, self.webkit_finder) 155 test_runner.run(parallel_tests, self._options.child_processes) 156 test_runner.run(serial_tests, 1) 157 158 self.printer.print_result(time.time() - start) 159 160 if self._options.coverage: 161 cov.stop() 162 cov.save() 163 cov.report(show_missing=False) 164 165 return not self.printer.num_errors and not self.printer.num_failures 166 167 def _check_imports(self, names): 168 for name in names: 169 if self.finder.is_module(name): 170 # if we failed to load a name and it looks like a module, 171 # try importing it directly, because loadTestsFromName() 172 # produces lousy error messages for bad modules. 173 try: 174 __import__(name) 175 except ImportError: 176 _log.fatal('Failed to import %s:' % name) 177 self._log_exception() 178 return False 179 return True 180 181 def _test_names(self, loader, names): 182 parallel_test_method_prefixes = ['test_'] 183 serial_test_method_prefixes = ['serial_test_'] 184 if self._options.integration_tests: 185 parallel_test_method_prefixes.append('integration_test_') 186 serial_test_method_prefixes.append('serial_integration_test_') 187 188 parallel_tests = [] 189 loader.test_method_prefixes = parallel_test_method_prefixes 190 for name in names: 191 parallel_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) 192 193 serial_tests = [] 194 loader.test_method_prefixes = serial_test_method_prefixes 195 for name in names: 196 serial_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) 197 198 # loader.loadTestsFromName() will not verify that names begin with one of the test_method_prefixes 199 # if the names were explicitly provided (e.g., MainTest.test_basic), so this means that any individual 200 # tests will be included in both parallel_tests and serial_tests, and we need to de-dup them. 201 serial_tests = list(set(serial_tests).difference(set(parallel_tests))) 202 203 return (parallel_tests, serial_tests) 204 205 def _all_test_names(self, suite): 206 names = [] 207 if hasattr(suite, '_tests'): 208 for t in suite._tests: 209 names.extend(self._all_test_names(t)) 210 else: 211 names.append(unit_test_name(suite)) 212 return names 213 214 def _log_exception(self): 215 s = StringIO.StringIO() 216 traceback.print_exc(file=s) 217 for l in s.buflist: 218 _log.error(' ' + l.rstrip()) 219 220 221 class _Loader(unittest.TestLoader): 222 test_method_prefixes = [] 223 224 def getTestCaseNames(self, testCaseClass): 225 def isTestMethod(attrname, testCaseClass=testCaseClass): 226 if not hasattr(getattr(testCaseClass, attrname), '__call__'): 227 return False 228 return (any(attrname.startswith(prefix) for prefix in self.test_method_prefixes)) 229 testFnNames = filter(isTestMethod, dir(testCaseClass)) 230 testFnNames.sort() 231 return testFnNames 232 233 234 if __name__ == '__main__': 235 sys.exit(main()) 236
