1 #!/usr/bin/env python 2 # 3 # Copyright 2005 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Unit test for Google Test test filters. 32 33 A user can specify which test(s) in a Google Test program to run via either 34 the GTEST_FILTER environment variable or the --gtest_filter flag. 35 This script tests such functionality by invoking 36 gtest_filter_unittest_ (a program written with Google Test) with different 37 environments and command line flags. 38 39 Note that test sharding may also influence which tests are filtered. Therefore, 40 we test that here also. 41 """ 42 43 __author__ = 'wan (at] google.com (Zhanyong Wan)' 44 45 import os 46 import re 47 import sets 48 import gtest_test_utils 49 50 # Constants. 51 52 IS_WINDOWS = os.name == 'nt' 53 54 # The environment variable for specifying the test filters. 55 FILTER_ENV_VAR = 'GTEST_FILTER' 56 57 # The environment variables for test sharding. 58 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 59 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 60 SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE' 61 62 # The command line flag for specifying the test filters. 63 FILTER_FLAG = 'gtest_filter' 64 65 # The command line flag for including disabled tests. 66 ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests' 67 68 # Command to run the gtest_filter_unittest_ program. 69 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_filter_unittest_') 70 71 # Regex for determining whether parameterized tests are enabled in the binary. 72 PARAM_TEST_REGEX = re.compile(r'/ParamTest') 73 74 # Regex for parsing test case names from Google Test's output. 75 TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)') 76 77 # Regex for parsing test names from Google Test's output. 78 TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)') 79 80 # Full names of all tests in gtest_filter_unittests_. 81 PARAM_TESTS = [ 82 'SeqP/ParamTest.TestX/0', 83 'SeqP/ParamTest.TestX/1', 84 'SeqP/ParamTest.TestY/0', 85 'SeqP/ParamTest.TestY/1', 86 'SeqQ/ParamTest.TestX/0', 87 'SeqQ/ParamTest.TestX/1', 88 'SeqQ/ParamTest.TestY/0', 89 'SeqQ/ParamTest.TestY/1', 90 ] 91 92 DISABLED_TESTS = [ 93 'BarTest.DISABLED_TestFour', 94 'BarTest.DISABLED_TestFive', 95 'BazTest.DISABLED_TestC', 96 'DISABLED_FoobarTest.Test1', 97 'DISABLED_FoobarTest.DISABLED_Test2', 98 'DISABLED_FoobarbazTest.TestA', 99 ] 100 101 # All the non-disabled tests. 102 ACTIVE_TESTS = [ 103 'FooTest.Abc', 104 'FooTest.Xyz', 105 106 'BarTest.TestOne', 107 'BarTest.TestTwo', 108 'BarTest.TestThree', 109 110 'BazTest.TestOne', 111 'BazTest.TestA', 112 'BazTest.TestB', 113 114 'HasDeathTest.Test1', 115 'HasDeathTest.Test2', 116 ] + PARAM_TESTS 117 118 param_tests_present = None 119 120 # Utilities. 121 122 123 def SetEnvVar(env_var, value): 124 """Sets the env variable to 'value'; unsets it when 'value' is None.""" 125 126 if value is not None: 127 os.environ[env_var] = value 128 elif env_var in os.environ: 129 del os.environ[env_var] 130 131 132 def RunAndReturnOutput(args = None): 133 """Runs the test program and returns its output.""" 134 135 return gtest_test_utils.Subprocess([COMMAND] + (args or [])).output 136 137 138 def RunAndExtractTestList(args = None): 139 """Runs the test program and returns its exit code and a list of tests run.""" 140 141 p = gtest_test_utils.Subprocess([COMMAND] + (args or [])) 142 tests_run = [] 143 test_case = '' 144 test = '' 145 for line in p.output.split('\n'): 146 match = TEST_CASE_REGEX.match(line) 147 if match is not None: 148 test_case = match.group(1) 149 else: 150 match = TEST_REGEX.match(line) 151 if match is not None: 152 test = match.group(1) 153 tests_run.append(test_case + '.' + test) 154 return (tests_run, p.exit_code) 155 156 157 def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs): 158 """Runs the given function and arguments in a modified environment.""" 159 try: 160 original_env = os.environ.copy() 161 os.environ.update(extra_env) 162 return function(*args, **kwargs) 163 finally: 164 for key in extra_env.iterkeys(): 165 if key in original_env: 166 os.environ[key] = original_env[key] 167 else: 168 del os.environ[key] 169 170 171 def RunWithSharding(total_shards, shard_index, command): 172 """Runs a test program shard and returns exit code and a list of tests run.""" 173 174 extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index), 175 TOTAL_SHARDS_ENV_VAR: str(total_shards)} 176 return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command) 177 178 # The unit test. 179 180 181 class GTestFilterUnitTest(gtest_test_utils.TestCase): 182 """Tests GTEST_FILTER env variable or --gtest_filter flag to filter tests.""" 183 184 # Utilities. 185 186 def AssertSetEqual(self, lhs, rhs): 187 """Asserts that two sets are equal.""" 188 189 for elem in lhs: 190 self.assert_(elem in rhs, '%s in %s' % (elem, rhs)) 191 192 for elem in rhs: 193 self.assert_(elem in lhs, '%s in %s' % (elem, lhs)) 194 195 def AssertPartitionIsValid(self, set_var, list_of_sets): 196 """Asserts that list_of_sets is a valid partition of set_var.""" 197 198 full_partition = [] 199 for slice_var in list_of_sets: 200 full_partition.extend(slice_var) 201 self.assertEqual(len(set_var), len(full_partition)) 202 self.assertEqual(sets.Set(set_var), sets.Set(full_partition)) 203 204 def AdjustForParameterizedTests(self, tests_to_run): 205 """Adjust tests_to_run in case value parameterized tests are disabled.""" 206 207 global param_tests_present 208 if not param_tests_present: 209 return list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS)) 210 else: 211 return tests_to_run 212 213 def RunAndVerify(self, gtest_filter, tests_to_run): 214 """Checks that the binary runs correct set of tests for the given filter.""" 215 216 tests_to_run = self.AdjustForParameterizedTests(tests_to_run) 217 218 # First, tests using GTEST_FILTER. 219 220 # Windows removes empty variables from the environment when passing it 221 # to a new process. This means it is impossible to pass an empty filter 222 # into a process using the GTEST_FILTER environment variable. However, 223 # we can still test the case when the variable is not supplied (i.e., 224 # gtest_filter is None). 225 # pylint: disable-msg=C6403 226 if not IS_WINDOWS or gtest_filter != '': 227 SetEnvVar(FILTER_ENV_VAR, gtest_filter) 228 tests_run = RunAndExtractTestList()[0] 229 SetEnvVar(FILTER_ENV_VAR, None) 230 self.AssertSetEqual(tests_run, tests_to_run) 231 # pylint: enable-msg=C6403 232 233 # Next, tests using --gtest_filter. 234 235 if gtest_filter is None: 236 args = [] 237 else: 238 args = ['--%s=%s' % (FILTER_FLAG, gtest_filter)] 239 240 tests_run = RunAndExtractTestList(args)[0] 241 self.AssertSetEqual(tests_run, tests_to_run) 242 243 def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run, 244 args=None, check_exit_0=False): 245 """Checks that binary runs correct tests for the given filter and shard. 246 247 Runs all shards of gtest_filter_unittest_ with the given filter, and 248 verifies that the right set of tests were run. The union of tests run 249 on each shard should be identical to tests_to_run, without duplicates. 250 251 Args: 252 gtest_filter: A filter to apply to the tests. 253 total_shards: A total number of shards to split test run into. 254 tests_to_run: A set of tests expected to run. 255 args : Arguments to pass to the to the test binary. 256 check_exit_0: When set to a true value, make sure that all shards 257 return 0. 258 """ 259 260 tests_to_run = self.AdjustForParameterizedTests(tests_to_run) 261 262 # Windows removes empty variables from the environment when passing it 263 # to a new process. This means it is impossible to pass an empty filter 264 # into a process using the GTEST_FILTER environment variable. However, 265 # we can still test the case when the variable is not supplied (i.e., 266 # gtest_filter is None). 267 # pylint: disable-msg=C6403 268 if not IS_WINDOWS or gtest_filter != '': 269 SetEnvVar(FILTER_ENV_VAR, gtest_filter) 270 partition = [] 271 for i in range(0, total_shards): 272 (tests_run, exit_code) = RunWithSharding(total_shards, i, args) 273 if check_exit_0: 274 self.assertEqual(0, exit_code) 275 partition.append(tests_run) 276 277 self.AssertPartitionIsValid(tests_to_run, partition) 278 SetEnvVar(FILTER_ENV_VAR, None) 279 # pylint: enable-msg=C6403 280 281 def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run): 282 """Checks that the binary runs correct set of tests for the given filter. 283 284 Runs gtest_filter_unittest_ with the given filter, and enables 285 disabled tests. Verifies that the right set of tests were run. 286 287 Args: 288 gtest_filter: A filter to apply to the tests. 289 tests_to_run: A set of tests expected to run. 290 """ 291 292 tests_to_run = self.AdjustForParameterizedTests(tests_to_run) 293 294 # Construct the command line. 295 args = ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG] 296 if gtest_filter is not None: 297 args.append('--%s=%s' % (FILTER_FLAG, gtest_filter)) 298 299 tests_run = RunAndExtractTestList(args)[0] 300 self.AssertSetEqual(tests_run, tests_to_run) 301 302 def setUp(self): 303 """Sets up test case. 304 305 Determines whether value-parameterized tests are enabled in the binary and 306 sets the flags accordingly. 307 """ 308 309 global param_tests_present 310 if param_tests_present is None: 311 param_tests_present = PARAM_TEST_REGEX.search( 312 RunAndReturnOutput()) is not None 313 314 def testDefaultBehavior(self): 315 """Tests the behavior of not specifying the filter.""" 316 317 self.RunAndVerify(None, ACTIVE_TESTS) 318 319 def testDefaultBehaviorWithShards(self): 320 """Tests the behavior without the filter, with sharding enabled.""" 321 322 self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS) 323 self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS) 324 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS) 325 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS) 326 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS) 327 328 def testEmptyFilter(self): 329 """Tests an empty filter.""" 330 331 self.RunAndVerify('', []) 332 self.RunAndVerifyWithSharding('', 1, []) 333 self.RunAndVerifyWithSharding('', 2, []) 334 335 def testBadFilter(self): 336 """Tests a filter that matches nothing.""" 337 338 self.RunAndVerify('BadFilter', []) 339 self.RunAndVerifyAllowingDisabled('BadFilter', []) 340 341 def testFullName(self): 342 """Tests filtering by full name.""" 343 344 self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz']) 345 self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz']) 346 self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz']) 347 348 def testUniversalFilters(self): 349 """Tests filters that match everything.""" 350 351 self.RunAndVerify('*', ACTIVE_TESTS) 352 self.RunAndVerify('*.*', ACTIVE_TESTS) 353 self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS) 354 self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS) 355 self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS) 356 357 def testFilterByTestCase(self): 358 """Tests filtering by test case name.""" 359 360 self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz']) 361 362 BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB'] 363 self.RunAndVerify('BazTest.*', BAZ_TESTS) 364 self.RunAndVerifyAllowingDisabled('BazTest.*', 365 BAZ_TESTS + ['BazTest.DISABLED_TestC']) 366 367 def testFilterByTest(self): 368 """Tests filtering by test name.""" 369 370 self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne']) 371 372 def testFilterDisabledTests(self): 373 """Select only the disabled tests to run.""" 374 375 self.RunAndVerify('DISABLED_FoobarTest.Test1', []) 376 self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1', 377 ['DISABLED_FoobarTest.Test1']) 378 379 self.RunAndVerify('*DISABLED_*', []) 380 self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS) 381 382 self.RunAndVerify('*.DISABLED_*', []) 383 self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [ 384 'BarTest.DISABLED_TestFour', 385 'BarTest.DISABLED_TestFive', 386 'BazTest.DISABLED_TestC', 387 'DISABLED_FoobarTest.DISABLED_Test2', 388 ]) 389 390 self.RunAndVerify('DISABLED_*', []) 391 self.RunAndVerifyAllowingDisabled('DISABLED_*', [ 392 'DISABLED_FoobarTest.Test1', 393 'DISABLED_FoobarTest.DISABLED_Test2', 394 'DISABLED_FoobarbazTest.TestA', 395 ]) 396 397 def testWildcardInTestCaseName(self): 398 """Tests using wildcard in the test case name.""" 399 400 self.RunAndVerify('*a*.*', [ 401 'BarTest.TestOne', 402 'BarTest.TestTwo', 403 'BarTest.TestThree', 404 405 'BazTest.TestOne', 406 'BazTest.TestA', 407 'BazTest.TestB', 408 409 'HasDeathTest.Test1', 410 'HasDeathTest.Test2', ] + PARAM_TESTS) 411 412 def testWildcardInTestName(self): 413 """Tests using wildcard in the test name.""" 414 415 self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA']) 416 417 def testFilterWithoutDot(self): 418 """Tests a filter that has no '.' in it.""" 419 420 self.RunAndVerify('*z*', [ 421 'FooTest.Xyz', 422 423 'BazTest.TestOne', 424 'BazTest.TestA', 425 'BazTest.TestB', 426 ]) 427 428 def testTwoPatterns(self): 429 """Tests filters that consist of two patterns.""" 430 431 self.RunAndVerify('Foo*.*:*A*', [ 432 'FooTest.Abc', 433 'FooTest.Xyz', 434 435 'BazTest.TestA', 436 ]) 437 438 # An empty pattern + a non-empty one 439 self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA']) 440 441 def testThreePatterns(self): 442 """Tests filters that consist of three patterns.""" 443 444 self.RunAndVerify('*oo*:*A*:*One', [ 445 'FooTest.Abc', 446 'FooTest.Xyz', 447 448 'BarTest.TestOne', 449 450 'BazTest.TestOne', 451 'BazTest.TestA', 452 ]) 453 454 # The 2nd pattern is empty. 455 self.RunAndVerify('*oo*::*One', [ 456 'FooTest.Abc', 457 'FooTest.Xyz', 458 459 'BarTest.TestOne', 460 461 'BazTest.TestOne', 462 ]) 463 464 # The last 2 patterns are empty. 465 self.RunAndVerify('*oo*::', [ 466 'FooTest.Abc', 467 'FooTest.Xyz', 468 ]) 469 470 def testNegativeFilters(self): 471 self.RunAndVerify('*-HasDeathTest.Test1', [ 472 'FooTest.Abc', 473 'FooTest.Xyz', 474 475 'BarTest.TestOne', 476 'BarTest.TestTwo', 477 'BarTest.TestThree', 478 479 'BazTest.TestOne', 480 'BazTest.TestA', 481 'BazTest.TestB', 482 483 'HasDeathTest.Test2', 484 ] + PARAM_TESTS) 485 486 self.RunAndVerify('*-FooTest.Abc:HasDeathTest.*', [ 487 'FooTest.Xyz', 488 489 'BarTest.TestOne', 490 'BarTest.TestTwo', 491 'BarTest.TestThree', 492 493 'BazTest.TestOne', 494 'BazTest.TestA', 495 'BazTest.TestB', 496 ] + PARAM_TESTS) 497 498 self.RunAndVerify('BarTest.*-BarTest.TestOne', [ 499 'BarTest.TestTwo', 500 'BarTest.TestThree', 501 ]) 502 503 # Tests without leading '*'. 504 self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:HasDeathTest.*', [ 505 'BarTest.TestOne', 506 'BarTest.TestTwo', 507 'BarTest.TestThree', 508 509 'BazTest.TestOne', 510 'BazTest.TestA', 511 'BazTest.TestB', 512 ] + PARAM_TESTS) 513 514 # Value parameterized tests. 515 self.RunAndVerify('*/*', PARAM_TESTS) 516 517 # Value parameterized tests filtering by the sequence name. 518 self.RunAndVerify('SeqP/*', [ 519 'SeqP/ParamTest.TestX/0', 520 'SeqP/ParamTest.TestX/1', 521 'SeqP/ParamTest.TestY/0', 522 'SeqP/ParamTest.TestY/1', 523 ]) 524 525 # Value parameterized tests filtering by the test name. 526 self.RunAndVerify('*/0', [ 527 'SeqP/ParamTest.TestX/0', 528 'SeqP/ParamTest.TestY/0', 529 'SeqQ/ParamTest.TestX/0', 530 'SeqQ/ParamTest.TestY/0', 531 ]) 532 533 def testFlagOverridesEnvVar(self): 534 """Tests that the filter flag overrides the filtering env. variable.""" 535 536 SetEnvVar(FILTER_ENV_VAR, 'Foo*') 537 args = ['--%s=%s' % (FILTER_FLAG, '*One')] 538 tests_run = RunAndExtractTestList(args)[0] 539 SetEnvVar(FILTER_ENV_VAR, None) 540 541 self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne']) 542 543 def testShardStatusFileIsCreated(self): 544 """Tests that the shard file is created if specified in the environment.""" 545 546 shard_status_file = os.path.join(gtest_test_utils.GetTempDir(), 547 'shard_status_file') 548 self.assert_(not os.path.exists(shard_status_file)) 549 550 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} 551 try: 552 InvokeWithModifiedEnv(extra_env, RunAndReturnOutput) 553 finally: 554 self.assert_(os.path.exists(shard_status_file)) 555 os.remove(shard_status_file) 556 557 def testShardStatusFileIsCreatedWithListTests(self): 558 """Tests that the shard file is created with --gtest_list_tests.""" 559 560 shard_status_file = os.path.join(gtest_test_utils.GetTempDir(), 561 'shard_status_file2') 562 self.assert_(not os.path.exists(shard_status_file)) 563 564 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} 565 try: 566 InvokeWithModifiedEnv(extra_env, 567 RunAndReturnOutput, 568 ['--gtest_list_tests']) 569 finally: 570 self.assert_(os.path.exists(shard_status_file)) 571 os.remove(shard_status_file) 572 573 def testShardingWorksWithDeathTests(self): 574 """Tests integration with death tests and sharding.""" 575 gtest_filter = 'HasDeathTest.*:SeqP/*' 576 expected_tests = [ 577 'HasDeathTest.Test1', 578 'HasDeathTest.Test2', 579 580 'SeqP/ParamTest.TestX/0', 581 'SeqP/ParamTest.TestX/1', 582 'SeqP/ParamTest.TestY/0', 583 'SeqP/ParamTest.TestY/1', 584 ] 585 586 for flag in ['--gtest_death_test_style=threadsafe', 587 '--gtest_death_test_style=fast']: 588 self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests, 589 check_exit_0=True, args=[flag]) 590 self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests, 591 check_exit_0=True, args=[flag]) 592 593 if __name__ == '__main__': 594 gtest_test_utils.Main() 595
