1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 ''' Runs various chrome tests through valgrind_test.py.''' 7 8 import glob 9 import logging 10 import optparse 11 import os 12 import subprocess 13 import sys 14 15 import logging_utils 16 import path_utils 17 18 import common 19 import valgrind_test 20 21 class TestNotFound(Exception): pass 22 23 class MultipleGTestFiltersSpecified(Exception): pass 24 25 class BuildDirNotFound(Exception): pass 26 27 class BuildDirAmbiguous(Exception): pass 28 29 class ExecutableNotFound(Exception): pass 30 31 class BadBinary(Exception): pass 32 33 class ChromeTests: 34 SLOW_TOOLS = ["drmemory"] 35 36 def __init__(self, options, args, test): 37 if ':' in test: 38 (self._test, self._gtest_filter) = test.split(':', 1) 39 else: 40 self._test = test 41 self._gtest_filter = options.gtest_filter 42 43 if self._test not in self._test_list: 44 raise TestNotFound("Unknown test: %s" % test) 45 46 if options.gtest_filter and options.gtest_filter != self._gtest_filter: 47 raise MultipleGTestFiltersSpecified("Can not specify both --gtest_filter " 48 "and --test %s" % test) 49 50 self._options = options 51 self._args = args 52 53 # Compute the top of the tree (the "source dir") from the script dir 54 # (where this script lives). We assume that the script dir is in 55 # tools/drmemory/scripts relative to the top of the tree. 56 script_dir = os.path.dirname(path_utils.ScriptDir()) 57 self._source_dir = os.path.dirname(os.path.dirname(script_dir)) 58 # Setup Dr. Memory if it's not set up yet. 59 drmem_cmd = os.getenv("DRMEMORY_COMMAND") 60 if not drmem_cmd: 61 drmem_sfx = os.path.join(script_dir, "drmemory-windows-sfx.exe") 62 if not os.path.isfile(drmem_sfx): 63 raise RuntimeError, "Cannot find drmemory-windows-sfx.exe" 64 drmem_dir = os.path.join(script_dir, "unpacked") 65 subprocess.call([drmem_sfx, "-o" + drmem_dir, "-y"], 0) 66 drmem_cmd = os.path.join(drmem_dir, "bin", "drmemory.exe") 67 os.environ["DRMEMORY_COMMAND"] = drmem_cmd 68 # since this path is used for string matching, make sure it's always 69 # an absolute Unix-style path 70 self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/') 71 self._command_preamble = ["--source-dir=%s" % (self._source_dir)] 72 73 if not self._options.build_dir: 74 dirs = [ 75 os.path.join(self._source_dir, "xcodebuild", "Debug"), 76 os.path.join(self._source_dir, "out", "Debug"), 77 os.path.join(self._source_dir, "build", "Debug"), 78 ] 79 build_dir = [d for d in dirs if os.path.isdir(d)] 80 if len(build_dir) > 1: 81 raise BuildDirAmbiguous("Found more than one suitable build dir:\n" 82 "%s\nPlease specify just one " 83 "using --build-dir" % ", ".join(build_dir)) 84 elif build_dir: 85 self._options.build_dir = build_dir[0] 86 else: 87 self._options.build_dir = None 88 89 if self._options.build_dir: 90 build_dir = os.path.abspath(self._options.build_dir) 91 self._command_preamble += ["--build-dir=%s" % (self._options.build_dir)] 92 93 def _EnsureBuildDirFound(self): 94 if not self._options.build_dir: 95 raise BuildDirNotFound("Oops, couldn't find a build dir, please " 96 "specify it manually using --build-dir") 97 98 def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None): 99 '''Generates the default command array that most tests will use.''' 100 if exe and common.IsWindows(): 101 exe += '.exe' 102 103 cmd = list(self._command_preamble) 104 105 # Find all suppressions matching the following pattern: 106 # tools/valgrind/TOOL/suppressions[_PLATFORM].txt 107 # and list them with --suppressions= prefix. 108 script_dir = path_utils.ScriptDir() 109 suppression_file = os.path.join(script_dir, "..", "suppressions.txt") 110 if os.path.exists(suppression_file): 111 cmd.append("--suppressions=%s" % suppression_file) 112 # Platform-specific suppression 113 for platform in common.PlatformNames(): 114 platform_suppression_file = \ 115 os.path.join(script_dir, "..", 'suppressions_%s.txt' % platform) 116 if os.path.exists(platform_suppression_file): 117 cmd.append("--suppressions=%s" % platform_suppression_file) 118 119 if self._options.valgrind_tool_flags: 120 cmd += self._options.valgrind_tool_flags.split(" ") 121 if self._options.keep_logs: 122 cmd += ["--keep_logs"] 123 if valgrind_test_args != None: 124 for arg in valgrind_test_args: 125 cmd.append(arg) 126 if exe: 127 self._EnsureBuildDirFound() 128 exe_path = os.path.join(self._options.build_dir, exe) 129 if not os.path.exists(exe_path): 130 raise ExecutableNotFound("Couldn't find '%s'" % exe_path) 131 132 cmd.append(exe_path) 133 # Valgrind runs tests slowly, so slow tests hurt more; show elapased time 134 # so we can find the slowpokes. 135 cmd.append("--gtest_print_time") 136 # Built-in test launcher for gtest-based executables runs tests using 137 # multiple process by default. Force the single-process mode back. 138 cmd.append("--single-process-tests") 139 if self._options.gtest_repeat: 140 cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat) 141 if self._options.gtest_shuffle: 142 cmd.append("--gtest_shuffle") 143 if self._options.gtest_break_on_failure: 144 cmd.append("--gtest_break_on_failure") 145 if self._options.test_launcher_bot_mode: 146 cmd.append("--test-launcher-bot-mode") 147 if self._options.test_launcher_total_shards is not None: 148 cmd.append("--test-launcher-total-shards=%d" % self._options.test_launcher_total_shards) 149 if self._options.test_launcher_shard_index is not None: 150 cmd.append("--test-launcher-shard-index=%d" % self._options.test_launcher_shard_index) 151 return cmd 152 153 def Run(self): 154 ''' Runs the test specified by command-line argument --test ''' 155 logging.info("running test %s" % (self._test)) 156 return self._test_list[self._test](self) 157 158 def _AppendGtestFilter(self, tool, name, cmd): 159 '''Append an appropriate --gtest_filter flag to the googletest binary 160 invocation. 161 If the user passed his own filter mentioning only one test, just use it. 162 Othewise, filter out tests listed in the appropriate gtest_exclude files. 163 ''' 164 if (self._gtest_filter and 165 ":" not in self._gtest_filter and 166 "?" not in self._gtest_filter and 167 "*" not in self._gtest_filter): 168 cmd.append("--gtest_filter=%s" % self._gtest_filter) 169 return 170 171 filters = [] 172 gtest_files_dir = os.path.join(path_utils.ScriptDir(), "gtest_exclude") 173 174 gtest_filter_files = [ 175 os.path.join(gtest_files_dir, name + ".gtest-%s.txt" % tool.ToolName())] 176 # Use ".gtest.txt" files only for slow tools, as they now contain 177 # Valgrind- and Dr.Memory-specific filters. 178 # TODO(glider): rename the files to ".gtest_slow.txt" 179 if tool.ToolName() in ChromeTests.SLOW_TOOLS: 180 gtest_filter_files += [os.path.join(gtest_files_dir, name + ".gtest.txt")] 181 for platform_suffix in common.PlatformNames(): 182 gtest_filter_files += [ 183 os.path.join(gtest_files_dir, name + ".gtest_%s.txt" % platform_suffix), 184 os.path.join(gtest_files_dir, name + ".gtest-%s_%s.txt" % \ 185 (tool.ToolName(), platform_suffix))] 186 logging.info("Reading gtest exclude filter files:") 187 for filename in gtest_filter_files: 188 # strip the leading absolute path (may be very long on the bot) 189 # and the following / or \. 190 readable_filename = filename.replace("\\", "/") # '\' on Windows 191 readable_filename = readable_filename.replace(self._source_dir, "")[1:] 192 if not os.path.exists(filename): 193 logging.info(" \"%s\" - not found" % readable_filename) 194 continue 195 logging.info(" \"%s\" - OK" % readable_filename) 196 f = open(filename, 'r') 197 for line in f.readlines(): 198 if line.startswith("#") or line.startswith("//") or line.isspace(): 199 continue 200 line = line.rstrip() 201 test_prefixes = ["FLAKY", "FAILS"] 202 for p in test_prefixes: 203 # Strip prefixes from the test names. 204 line = line.replace(".%s_" % p, ".") 205 # Exclude the original test name. 206 filters.append(line) 207 if line[-2:] != ".*": 208 # List all possible prefixes if line doesn't end with ".*". 209 for p in test_prefixes: 210 filters.append(line.replace(".", ".%s_" % p)) 211 # Get rid of duplicates. 212 filters = set(filters) 213 gtest_filter = self._gtest_filter 214 if len(filters): 215 if gtest_filter: 216 gtest_filter += ":" 217 if gtest_filter.find("-") < 0: 218 gtest_filter += "-" 219 else: 220 gtest_filter = "-" 221 gtest_filter += ":".join(filters) 222 if gtest_filter: 223 cmd.append("--gtest_filter=%s" % gtest_filter) 224 225 @staticmethod 226 def ShowTests(): 227 test_to_names = {} 228 for name, test_function in ChromeTests._test_list.iteritems(): 229 test_to_names.setdefault(test_function, []).append(name) 230 231 name_to_aliases = {} 232 for names in test_to_names.itervalues(): 233 names.sort(key=lambda name: len(name)) 234 name_to_aliases[names[0]] = names[1:] 235 236 print 237 print "Available tests:" 238 print "----------------" 239 for name, aliases in sorted(name_to_aliases.iteritems()): 240 if aliases: 241 print " {} (aka {})".format(name, ', '.join(aliases)) 242 else: 243 print " {}".format(name) 244 245 def SetupLdPath(self, requires_build_dir): 246 if requires_build_dir: 247 self._EnsureBuildDirFound() 248 elif not self._options.build_dir: 249 return 250 251 # Append build_dir to LD_LIBRARY_PATH so external libraries can be loaded. 252 if (os.getenv("LD_LIBRARY_PATH")): 253 os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"), 254 self._options.build_dir)) 255 else: 256 os.putenv("LD_LIBRARY_PATH", self._options.build_dir) 257 258 def SimpleTest(self, module, name, valgrind_test_args=None, cmd_args=None): 259 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 260 cmd = self._DefaultCommand(tool, name, valgrind_test_args) 261 self._AppendGtestFilter(tool, name, cmd) 262 cmd.extend(['--test-tiny-timeout=1000']) 263 if cmd_args: 264 cmd.extend(cmd_args) 265 266 self.SetupLdPath(True) 267 return tool.Run(cmd, module) 268 269 def RunCmdLine(self): 270 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 271 cmd = self._DefaultCommand(tool, None, self._args) 272 self.SetupLdPath(False) 273 return tool.Run(cmd, None) 274 275 def TestPDFiumUnitTests(self): 276 return self.SimpleTest("pdfium_unittests", "pdfium_unittests") 277 278 def TestPDFiumEmbedderTests(self): 279 return self.SimpleTest("pdfium_embeddertests", "pdfium_embeddertests") 280 281 def TestPDFiumTest(self, script_name): 282 # Build the command line in 'cmd'. 283 # It's going to be roughly 284 # python valgrind_test.py ... 285 # but we'll use the --indirect_pdfium_test flag to valgrind_test.py 286 # to avoid valgrinding python. 287 288 # Start by building the valgrind_test.py commandline. 289 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 290 cmd = self._DefaultCommand(tool) 291 cmd.append("--trace_children") 292 cmd.append("--indirect_pdfium_test") 293 cmd.append("--ignore_exit_code") 294 # Now build script_cmd, the run_corpus_tests commandline. 295 script = os.path.join(self._source_dir, "testing", "tools", script_name) 296 script_cmd = ["python", script] 297 if self._options.build_dir: 298 script_cmd.extend(["--build-dir", self._options.build_dir]) 299 # TODO(zhaoqin): it only runs in single process mode now, 300 # need figure out why it does not work with test_one_file_parallel 301 # in run_corpus_tests.py. 302 if script_name == "run_corpus_tests.py": 303 script_cmd.extend(["-j", "1"]) 304 # Now run script_cmd with the wrapper in cmd 305 cmd.append("--") 306 cmd.extend(script_cmd) 307 308 ret = tool.Run(cmd, "layout", min_runtime_in_seconds=0) 309 return ret 310 311 def TestPDFiumJavascript(self): 312 return self.TestPDFiumTest("run_javascript_tests.py") 313 314 def TestPDFiumPixel(self): 315 return self.TestPDFiumTest("run_pixel_tests.py") 316 317 def TestPDFiumCorpus(self): 318 return self.TestPDFiumTest("run_corpus_tests.py") 319 320 # The known list of tests. 321 _test_list = { 322 "cmdline" : RunCmdLine, 323 "pdfium_corpus": TestPDFiumCorpus, 324 "pdfium_embeddertests": TestPDFiumEmbedderTests, 325 "pdfium_javascript": TestPDFiumJavascript, 326 "pdfium_pixel": TestPDFiumPixel, 327 "pdfium_unittests": TestPDFiumUnitTests, 328 } 329 330 331 def _main(): 332 parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> " 333 "[-t <test> ...]") 334 335 parser.add_option("--help-tests", dest="help_tests", action="store_true", 336 default=False, help="List all available tests") 337 parser.add_option("-b", "--build-dir", 338 help="the location of the compiler output") 339 parser.add_option("--target", help="Debug or Release") 340 parser.add_option("-t", "--test", action="append", default=[], 341 help="which test to run, supports test:gtest_filter format " 342 "as well.") 343 parser.add_option("--gtest_filter", 344 help="additional arguments to --gtest_filter") 345 parser.add_option("--gtest_repeat", help="argument for --gtest_repeat") 346 parser.add_option("--gtest_shuffle", action="store_true", default=False, 347 help="Randomize tests' orders on every iteration.") 348 parser.add_option("--gtest_break_on_failure", action="store_true", 349 default=False, 350 help="Drop in to debugger on assertion failure. Also " 351 "useful for forcing tests to exit with a stack dump " 352 "on the first assertion failure when running with " 353 "--gtest_repeat=-1") 354 parser.add_option("-v", "--verbose", action="store_true", default=False, 355 help="verbose output - enable debug log messages") 356 parser.add_option("--tool", dest="valgrind_tool", default="drmemory_full", 357 help="specify a valgrind tool to run the tests under") 358 parser.add_option("--tool_flags", dest="valgrind_tool_flags", default="", 359 help="specify custom flags for the selected valgrind tool") 360 parser.add_option("--keep_logs", action="store_true", default=False, 361 help="store memory tool logs in the <tool>.logs directory " 362 "instead of /tmp.\nThis can be useful for tool " 363 "developers/maintainers.\nPlease note that the <tool>" 364 ".logs directory will be clobbered on tool startup.") 365 parser.add_option("--test-launcher-bot-mode", action="store_true", 366 help="run the tests with --test-launcher-bot-mode") 367 parser.add_option("--test-launcher-total-shards", type=int, 368 help="run the tests with --test-launcher-total-shards") 369 parser.add_option("--test-launcher-shard-index", type=int, 370 help="run the tests with --test-launcher-shard-index") 371 372 options, args = parser.parse_args() 373 374 # Bake target into build_dir. 375 if options.target and options.build_dir: 376 assert (options.target != 377 os.path.basename(os.path.dirname(options.build_dir))) 378 options.build_dir = os.path.join(os.path.abspath(options.build_dir), 379 options.target) 380 381 if options.verbose: 382 logging_utils.config_root(logging.DEBUG) 383 else: 384 logging_utils.config_root() 385 386 if options.help_tests: 387 ChromeTests.ShowTests() 388 return 0 389 390 if not options.test: 391 parser.error("--test not specified") 392 393 if len(options.test) != 1 and options.gtest_filter: 394 parser.error("--gtest_filter and multiple tests don't make sense together") 395 396 for t in options.test: 397 tests = ChromeTests(options, args, t) 398 ret = tests.Run() 399 if ret: return ret 400 return 0 401 402 403 if __name__ == "__main__": 404 sys.exit(_main()) 405
