1 #!/usr/bin/env python 2 # 3 # Copyright 2005, Google Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are 8 # met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following disclaimer 14 # in the documentation and/or other materials provided with the 15 # distribution. 16 # * Neither the name of Google Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived from 18 # this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32 """Unit test for Google Test test filters. 33 34 A user can specify which test(s) in a Google Test program to run via either 35 the GTEST_FILTER environment variable or the --gtest_filter flag. 36 This script tests such functionality by invoking 37 gtest_filter_unittest_ (a program written with Google Test) with different 38 environments and command line flags. 39 40 Note that test sharding may also influence which tests are filtered. Therefore, 41 we test that here also. 42 """ 43 44 __author__ = 'wan (at] google.com (Zhanyong Wan)' 45 46 import os 47 import re 48 import sets 49 import tempfile 50 import unittest 51 import gtest_test_utils 52 53 # Constants. 54 55 # The environment variable for specifying the test filters. 56 FILTER_ENV_VAR = 'GTEST_FILTER' 57 58 # The environment variables for test sharding. 59 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 60 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 61 SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE' 62 63 # The command line flag for specifying the test filters. 64 FILTER_FLAG = 'gtest_filter' 65 66 # The command line flag for including disabled tests. 67 ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests' 68 69 # Command to run the gtest_filter_unittest_ program. 70 COMMAND = os.path.join(gtest_test_utils.GetBuildDir(), 71 'gtest_filter_unittest_') 72 73 # Regex for determining whether parameterized tests are enabled in the binary. 74 PARAM_TEST_REGEX = re.compile(r'/ParamTest') 75 76 # Regex for parsing test case names from Google Test's output. 77 TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)') 78 79 # Regex for parsing test names from Google Test's output. 80 TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)') 81 82 # Full names of all tests in gtest_filter_unittests_. 83 PARAM_TESTS = [ 84 'SeqP/ParamTest.TestX/0', 85 'SeqP/ParamTest.TestX/1', 86 'SeqP/ParamTest.TestY/0', 87 'SeqP/ParamTest.TestY/1', 88 'SeqQ/ParamTest.TestX/0', 89 'SeqQ/ParamTest.TestX/1', 90 'SeqQ/ParamTest.TestY/0', 91 'SeqQ/ParamTest.TestY/1', 92 ] 93 94 DISABLED_TESTS = [ 95 'BarTest.DISABLED_TestFour', 96 'BarTest.DISABLED_TestFive', 97 'BazTest.DISABLED_TestC', 98 'DISABLED_FoobarTest.Test1', 99 'DISABLED_FoobarTest.DISABLED_Test2', 100 'DISABLED_FoobarbazTest.TestA', 101 ] 102 103 # All the non-disabled tests. 104 ACTIVE_TESTS = [ 105 'FooTest.Abc', 106 'FooTest.Xyz', 107 108 'BarTest.TestOne', 109 'BarTest.TestTwo', 110 'BarTest.TestThree', 111 112 'BazTest.TestOne', 113 'BazTest.TestA', 114 'BazTest.TestB', 115 116 'HasDeathTest.Test1', 117 'HasDeathTest.Test2', 118 ] + PARAM_TESTS 119 120 param_tests_present = None 121 122 # Utilities. 123 124 125 def SetEnvVar(env_var, value): 126 """Sets the env variable to 'value'; unsets it when 'value' is None.""" 127 128 if value is not None: 129 os.environ[env_var] = value 130 elif env_var in os.environ: 131 del os.environ[env_var] 132 133 134 def Run(command): 135 """Runs a Google Test program and returns a list of full names of the 136 tests that were run along with the test exit code. 137 """ 138 139 stdout_file = os.popen(command, 'r') 140 tests_run = [] 141 test_case = '' 142 test = '' 143 for line in stdout_file: 144 match = TEST_CASE_REGEX.match(line) 145 if match is not None: 146 test_case = match.group(1) 147 else: 148 match = TEST_REGEX.match(line) 149 if match is not None: 150 test = match.group(1) 151 tests_run += [test_case + '.' + test] 152 exit_code = stdout_file.close() 153 return (tests_run, exit_code) 154 155 156 def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs): 157 """Runs the given function and arguments in a modified environment.""" 158 try: 159 original_env = os.environ.copy() 160 os.environ.update(extra_env) 161 return function(*args, **kwargs) 162 finally: 163 for key in extra_env.iterkeys(): 164 if key in original_env: 165 os.environ[key] = original_env[key] 166 else: 167 del os.environ[key] 168 169 170 def RunWithSharding(total_shards, shard_index, command): 171 """Runs the Google Test program shard and returns a list of full names of the 172 tests that were run along with the exit code. 173 """ 174 175 extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index), 176 TOTAL_SHARDS_ENV_VAR: str(total_shards)} 177 return InvokeWithModifiedEnv(extra_env, Run, command) 178 179 # The unit test. 180 181 182 class GTestFilterUnitTest(unittest.TestCase): 183 """Tests using the GTEST_FILTER environment variable or the 184 --gtest_filter flag to filter tests. 185 """ 186 187 # Utilities. 188 189 def AssertSetEqual(self, lhs, rhs): 190 """Asserts that two sets are equal.""" 191 192 for elem in lhs: 193 self.assert_(elem in rhs, '%s in %s' % (elem, rhs)) 194 195 for elem in rhs: 196 self.assert_(elem in lhs, '%s in %s' % (elem, lhs)) 197 198 def AssertPartitionIsValid(self, set_var, list_of_sets): 199 """Asserts that list_of_sets is a valid partition of set_var.""" 200 201 full_partition = [] 202 for slice_var in list_of_sets: 203 full_partition.extend(slice_var) 204 self.assertEqual(len(set_var), len(full_partition)) 205 self.assertEqual(sets.Set(set_var), sets.Set(full_partition)) 206 207 def RunAndVerify(self, gtest_filter, tests_to_run): 208 """Runs gtest_flag_unittest_ with the given filter, and verifies 209 that the right set of tests were run. 210 """ 211 # Adjust tests_to_run in case value parameterized tests are disabled 212 # in the binary. 213 global param_tests_present 214 if not param_tests_present: 215 tests_to_run = list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS)) 216 217 # First, tests using GTEST_FILTER. 218 219 SetEnvVar(FILTER_ENV_VAR, gtest_filter) 220 tests_run = Run(COMMAND)[0] 221 SetEnvVar(FILTER_ENV_VAR, None) 222 223 self.AssertSetEqual(tests_run, tests_to_run) 224 225 # Next, tests using --gtest_filter. 226 227 if gtest_filter is None: 228 command = COMMAND 229 else: 230 command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, gtest_filter) 231 232 tests_run = Run(command)[0] 233 self.AssertSetEqual(tests_run, tests_to_run) 234 235 def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run, 236 command=COMMAND, check_exit_0=False): 237 """Runs all shards of gtest_flag_unittest_ with the given filter, and 238 verifies that the right set of tests were run. The union of tests run 239 on each shard should be identical to tests_to_run, without duplicates. 240 If check_exit_0, make sure that all shards returned 0. 241 """ 242 SetEnvVar(FILTER_ENV_VAR, gtest_filter) 243 partition = [] 244 for i in range(0, total_shards): 245 (tests_run, exit_code) = RunWithSharding(total_shards, i, command) 246 if check_exit_0: 247 self.assert_(exit_code is None) 248 partition.append(tests_run) 249 250 self.AssertPartitionIsValid(tests_to_run, partition) 251 SetEnvVar(FILTER_ENV_VAR, None) 252 253 def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run): 254 """Runs gtest_flag_unittest_ with the given filter, and enables 255 disabled tests. Verifies that the right set of tests were run. 256 """ 257 # Construct the command line. 258 command = '%s --%s' % (COMMAND, ALSO_RUN_DISABED_TESTS_FLAG) 259 if gtest_filter is not None: 260 command = '%s --%s=%s' % (command, FILTER_FLAG, gtest_filter) 261 262 tests_run = Run(command)[0] 263 self.AssertSetEqual(tests_run, tests_to_run) 264 265 def setUp(self): 266 """Sets up test case. Determines whether value-parameterized tests are 267 enabled in the binary and sets flags accordingly. 268 """ 269 global param_tests_present 270 if param_tests_present is None: 271 param_tests_present = PARAM_TEST_REGEX.search( 272 '\n'.join(os.popen(COMMAND, 'r').readlines())) is not None 273 274 def testDefaultBehavior(self): 275 """Tests the behavior of not specifying the filter.""" 276 277 self.RunAndVerify(None, ACTIVE_TESTS) 278 279 def testDefaultBehaviorWithShards(self): 280 """Tests the behavior of not specifying the filter, with sharding 281 enabled. 282 """ 283 self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS) 284 self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS) 285 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS) 286 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS) 287 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS) 288 289 def testEmptyFilter(self): 290 """Tests an empty filter.""" 291 292 self.RunAndVerify('', []) 293 self.RunAndVerifyWithSharding('', 1, []) 294 self.RunAndVerifyWithSharding('', 2, []) 295 296 def testBadFilter(self): 297 """Tests a filter that matches nothing.""" 298 299 self.RunAndVerify('BadFilter', []) 300 self.RunAndVerifyAllowingDisabled('BadFilter', []) 301 302 def testFullName(self): 303 """Tests filtering by full name.""" 304 305 self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz']) 306 self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz']) 307 self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz']) 308 309 def testUniversalFilters(self): 310 """Tests filters that match everything.""" 311 312 self.RunAndVerify('*', ACTIVE_TESTS) 313 self.RunAndVerify('*.*', ACTIVE_TESTS) 314 self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS) 315 self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS) 316 self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS) 317 318 def testFilterByTestCase(self): 319 """Tests filtering by test case name.""" 320 321 self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz']) 322 323 BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB'] 324 self.RunAndVerify('BazTest.*', BAZ_TESTS) 325 self.RunAndVerifyAllowingDisabled('BazTest.*', 326 BAZ_TESTS + ['BazTest.DISABLED_TestC']) 327 328 def testFilterByTest(self): 329 """Tests filtering by test name.""" 330 331 self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne']) 332 333 def testFilterDisabledTests(self): 334 """Select only the disabled tests to run.""" 335 336 self.RunAndVerify('DISABLED_FoobarTest.Test1', []) 337 self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1', 338 ['DISABLED_FoobarTest.Test1']) 339 340 self.RunAndVerify('*DISABLED_*', []) 341 self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS) 342 343 self.RunAndVerify('*.DISABLED_*', []) 344 self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [ 345 'BarTest.DISABLED_TestFour', 346 'BarTest.DISABLED_TestFive', 347 'BazTest.DISABLED_TestC', 348 'DISABLED_FoobarTest.DISABLED_Test2', 349 ]) 350 351 self.RunAndVerify('DISABLED_*', []) 352 self.RunAndVerifyAllowingDisabled('DISABLED_*', [ 353 'DISABLED_FoobarTest.Test1', 354 'DISABLED_FoobarTest.DISABLED_Test2', 355 'DISABLED_FoobarbazTest.TestA', 356 ]) 357 358 def testWildcardInTestCaseName(self): 359 """Tests using wildcard in the test case name.""" 360 361 self.RunAndVerify('*a*.*', [ 362 'BarTest.TestOne', 363 'BarTest.TestTwo', 364 'BarTest.TestThree', 365 366 'BazTest.TestOne', 367 'BazTest.TestA', 368 'BazTest.TestB', 369 370 'HasDeathTest.Test1', 371 'HasDeathTest.Test2', ] + PARAM_TESTS) 372 373 def testWildcardInTestName(self): 374 """Tests using wildcard in the test name.""" 375 376 self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA']) 377 378 def testFilterWithoutDot(self): 379 """Tests a filter that has no '.' in it.""" 380 381 self.RunAndVerify('*z*', [ 382 'FooTest.Xyz', 383 384 'BazTest.TestOne', 385 'BazTest.TestA', 386 'BazTest.TestB', 387 ]) 388 389 def testTwoPatterns(self): 390 """Tests filters that consist of two patterns.""" 391 392 self.RunAndVerify('Foo*.*:*A*', [ 393 'FooTest.Abc', 394 'FooTest.Xyz', 395 396 'BazTest.TestA', 397 ]) 398 399 # An empty pattern + a non-empty one 400 self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA']) 401 402 def testThreePatterns(self): 403 """Tests filters that consist of three patterns.""" 404 405 self.RunAndVerify('*oo*:*A*:*One', [ 406 'FooTest.Abc', 407 'FooTest.Xyz', 408 409 'BarTest.TestOne', 410 411 'BazTest.TestOne', 412 'BazTest.TestA', 413 ]) 414 415 # The 2nd pattern is empty. 416 self.RunAndVerify('*oo*::*One', [ 417 'FooTest.Abc', 418 'FooTest.Xyz', 419 420 'BarTest.TestOne', 421 422 'BazTest.TestOne', 423 ]) 424 425 # The last 2 patterns are empty. 426 self.RunAndVerify('*oo*::', [ 427 'FooTest.Abc', 428 'FooTest.Xyz', 429 ]) 430 431 def testNegativeFilters(self): 432 self.RunAndVerify('*-HasDeathTest.Test1', [ 433 'FooTest.Abc', 434 'FooTest.Xyz', 435 436 'BarTest.TestOne', 437 'BarTest.TestTwo', 438 'BarTest.TestThree', 439 440 'BazTest.TestOne', 441 'BazTest.TestA', 442 'BazTest.TestB', 443 444 'HasDeathTest.Test2', 445 ] + PARAM_TESTS) 446 447 self.RunAndVerify('*-FooTest.Abc:HasDeathTest.*', [ 448 'FooTest.Xyz', 449 450 'BarTest.TestOne', 451 'BarTest.TestTwo', 452 'BarTest.TestThree', 453 454 'BazTest.TestOne', 455 'BazTest.TestA', 456 'BazTest.TestB', 457 ] + PARAM_TESTS) 458 459 self.RunAndVerify('BarTest.*-BarTest.TestOne', [ 460 'BarTest.TestTwo', 461 'BarTest.TestThree', 462 ]) 463 464 # Tests without leading '*'. 465 self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:HasDeathTest.*', [ 466 'BarTest.TestOne', 467 'BarTest.TestTwo', 468 'BarTest.TestThree', 469 470 'BazTest.TestOne', 471 'BazTest.TestA', 472 'BazTest.TestB', 473 ] + PARAM_TESTS) 474 475 # Value parameterized tests. 476 self.RunAndVerify('*/*', PARAM_TESTS) 477 478 # Value parameterized tests filtering by the sequence name. 479 self.RunAndVerify('SeqP/*', [ 480 'SeqP/ParamTest.TestX/0', 481 'SeqP/ParamTest.TestX/1', 482 'SeqP/ParamTest.TestY/0', 483 'SeqP/ParamTest.TestY/1', 484 ]) 485 486 # Value parameterized tests filtering by the test name. 487 self.RunAndVerify('*/0', [ 488 'SeqP/ParamTest.TestX/0', 489 'SeqP/ParamTest.TestY/0', 490 'SeqQ/ParamTest.TestX/0', 491 'SeqQ/ParamTest.TestY/0', 492 ]) 493 494 def testFlagOverridesEnvVar(self): 495 """Tests that the --gtest_filter flag overrides the GTEST_FILTER 496 environment variable. 497 """ 498 499 SetEnvVar(FILTER_ENV_VAR, 'Foo*') 500 command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, '*One') 501 tests_run = Run(command)[0] 502 SetEnvVar(FILTER_ENV_VAR, None) 503 504 self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne']) 505 506 def testShardStatusFileIsCreated(self): 507 """Tests that the shard file is created if specified in the environment.""" 508 509 test_tmpdir = tempfile.mkdtemp() 510 shard_status_file = os.path.join(test_tmpdir, 'shard_status_file') 511 self.assert_(not os.path.exists(shard_status_file)) 512 513 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} 514 stdout_file = InvokeWithModifiedEnv(extra_env, os.popen, COMMAND, 'r') 515 try: 516 stdout_file.readlines() 517 finally: 518 stdout_file.close() 519 self.assert_(os.path.exists(shard_status_file)) 520 os.remove(shard_status_file) 521 os.removedirs(test_tmpdir) 522 523 def testShardStatusFileIsCreatedWithListTests(self): 524 """Tests that the shard file is created with --gtest_list_tests.""" 525 526 test_tmpdir = tempfile.mkdtemp() 527 shard_status_file = os.path.join(test_tmpdir, 'shard_status_file2') 528 self.assert_(not os.path.exists(shard_status_file)) 529 530 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} 531 stdout_file = InvokeWithModifiedEnv(extra_env, os.popen, 532 '%s --gtest_list_tests' % COMMAND, 'r') 533 try: 534 stdout_file.readlines() 535 finally: 536 stdout_file.close() 537 self.assert_(os.path.exists(shard_status_file)) 538 os.remove(shard_status_file) 539 os.removedirs(test_tmpdir) 540 541 def testShardingWorksWithDeathTests(self): 542 """Tests integration with death tests and sharding.""" 543 gtest_filter = 'HasDeathTest.*:SeqP/*' 544 expected_tests = [ 545 'HasDeathTest.Test1', 546 'HasDeathTest.Test2', 547 548 'SeqP/ParamTest.TestX/0', 549 'SeqP/ParamTest.TestX/1', 550 'SeqP/ParamTest.TestY/0', 551 'SeqP/ParamTest.TestY/1', 552 ] 553 554 for command in (COMMAND + ' --gtest_death_test_style=threadsafe', 555 COMMAND + ' --gtest_death_test_style=fast'): 556 self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests, 557 check_exit_0=True, command=command) 558 self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests, 559 check_exit_0=True, command=command) 560 561 if __name__ == '__main__': 562 gtest_test_utils.Main() 563
