1 #!/usr/bin/env python 2 # 3 # Copyright 2009 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Verifies that test shuffling works.""" 32 33 __author__ = 'wan (at] google.com (Zhanyong Wan)' 34 35 import os 36 import gtest_test_utils 37 38 # Command to run the gtest_shuffle_test_ program. 39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') 40 41 # The environment variables for test sharding. 42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 44 45 TEST_FILTER = 'A*.A:A*.B:C*' 46 47 ALL_TESTS = [] 48 ACTIVE_TESTS = [] 49 FILTERED_TESTS = [] 50 SHARDED_TESTS = [] 51 52 SHUFFLED_ALL_TESTS = [] 53 SHUFFLED_ACTIVE_TESTS = [] 54 SHUFFLED_FILTERED_TESTS = [] 55 SHUFFLED_SHARDED_TESTS = [] 56 57 58 def AlsoRunDisabledTestsFlag(): 59 return '--gtest_also_run_disabled_tests' 60 61 62 def FilterFlag(test_filter): 63 return '--gtest_filter=%s' % (test_filter,) 64 65 66 def RepeatFlag(n): 67 return '--gtest_repeat=%s' % (n,) 68 69 70 def ShuffleFlag(): 71 return '--gtest_shuffle' 72 73 74 def RandomSeedFlag(n): 75 return '--gtest_random_seed=%s' % (n,) 76 77 78 def RunAndReturnOutput(extra_env, args): 79 """Runs the test program and returns its output.""" 80 81 try: 82 original_env = os.environ.copy() 83 os.environ.update(extra_env) 84 return gtest_test_utils.Subprocess([COMMAND] + args).output 85 finally: 86 for key in extra_env.iterkeys(): 87 if key in original_env: 88 os.environ[key] = original_env[key] 89 else: 90 del os.environ[key] 91 92 93 def GetTestsForAllIterations(extra_env, args): 94 """Runs the test program and returns a list of test lists. 95 96 Args: 97 extra_env: a map from environment variables to their values 98 args: command line flags to pass to gtest_shuffle_test_ 99 100 Returns: 101 A list where the i-th element is the list of tests run in the i-th 102 test iteration. 103 """ 104 105 test_iterations = [] 106 for line in RunAndReturnOutput(extra_env, args).split('\n'): 107 if line.startswith('----'): 108 tests = [] 109 test_iterations.append(tests) 110 elif line.strip(): 111 tests.append(line.strip()) # 'TestCaseName.TestName' 112 113 return test_iterations 114 115 116 def GetTestCases(tests): 117 """Returns a list of test cases in the given full test names. 118 119 Args: 120 tests: a list of full test names 121 122 Returns: 123 A list of test cases from 'tests', in their original order. 124 Consecutive duplicates are removed. 125 """ 126 127 test_cases = [] 128 for test in tests: 129 test_case = test.split('.')[0] 130 if not test_case in test_cases: 131 test_cases.append(test_case) 132 133 return test_cases 134 135 136 def CalculateTestLists(): 137 """Calculates the list of tests run under different flags.""" 138 139 if not ALL_TESTS: 140 ALL_TESTS.extend( 141 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 142 143 if not ACTIVE_TESTS: 144 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 145 146 if not FILTERED_TESTS: 147 FILTERED_TESTS.extend( 148 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 149 150 if not SHARDED_TESTS: 151 SHARDED_TESTS.extend( 152 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 153 SHARD_INDEX_ENV_VAR: '1'}, 154 [])[0]) 155 156 if not SHUFFLED_ALL_TESTS: 157 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 158 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 159 160 if not SHUFFLED_ACTIVE_TESTS: 161 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 162 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 163 164 if not SHUFFLED_FILTERED_TESTS: 165 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 166 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 167 168 if not SHUFFLED_SHARDED_TESTS: 169 SHUFFLED_SHARDED_TESTS.extend( 170 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 171 SHARD_INDEX_ENV_VAR: '1'}, 172 [ShuffleFlag(), RandomSeedFlag(1)])[0]) 173 174 175 class GTestShuffleUnitTest(gtest_test_utils.TestCase): 176 """Tests test shuffling.""" 177 178 def setUp(self): 179 CalculateTestLists() 180 181 def testShufflePreservesNumberOfTests(self): 182 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 183 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 184 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 185 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 186 187 def testShuffleChangesTestOrder(self): 188 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 189 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 190 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 191 SHUFFLED_FILTERED_TESTS) 192 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 193 SHUFFLED_SHARDED_TESTS) 194 195 def testShuffleChangesTestCaseOrder(self): 196 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 197 GetTestCases(SHUFFLED_ALL_TESTS)) 198 self.assert_( 199 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 200 GetTestCases(SHUFFLED_ACTIVE_TESTS)) 201 self.assert_( 202 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 203 GetTestCases(SHUFFLED_FILTERED_TESTS)) 204 self.assert_( 205 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 206 GetTestCases(SHUFFLED_SHARDED_TESTS)) 207 208 def testShuffleDoesNotRepeatTest(self): 209 for test in SHUFFLED_ALL_TESTS: 210 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 211 '%s appears more than once' % (test,)) 212 for test in SHUFFLED_ACTIVE_TESTS: 213 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 214 '%s appears more than once' % (test,)) 215 for test in SHUFFLED_FILTERED_TESTS: 216 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 217 '%s appears more than once' % (test,)) 218 for test in SHUFFLED_SHARDED_TESTS: 219 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 220 '%s appears more than once' % (test,)) 221 222 def testShuffleDoesNotCreateNewTest(self): 223 for test in SHUFFLED_ALL_TESTS: 224 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 225 for test in SHUFFLED_ACTIVE_TESTS: 226 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 227 for test in SHUFFLED_FILTERED_TESTS: 228 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 229 for test in SHUFFLED_SHARDED_TESTS: 230 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 231 232 def testShuffleIncludesAllTests(self): 233 for test in ALL_TESTS: 234 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 235 for test in ACTIVE_TESTS: 236 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 237 for test in FILTERED_TESTS: 238 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 239 for test in SHARDED_TESTS: 240 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 241 242 def testShuffleLeavesDeathTestsAtFront(self): 243 non_death_test_found = False 244 for test in SHUFFLED_ACTIVE_TESTS: 245 if 'DeathTest.' in test: 246 self.assert_(not non_death_test_found, 247 '%s appears after a non-death test' % (test,)) 248 else: 249 non_death_test_found = True 250 251 def _VerifyTestCasesDoNotInterleave(self, tests): 252 test_cases = [] 253 for test in tests: 254 [test_case, _] = test.split('.') 255 if test_cases and test_cases[-1] != test_case: 256 test_cases.append(test_case) 257 self.assertEqual(1, test_cases.count(test_case), 258 'Test case %s is not grouped together in %s' % 259 (test_case, tests)) 260 261 def testShuffleDoesNotInterleaveTestCases(self): 262 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 263 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 264 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 265 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 266 267 def testShuffleRestoresOrderAfterEachIteration(self): 268 # Get the test lists in all 3 iterations, using random seed 1, 2, 269 # and 3 respectively. Google Test picks a different seed in each 270 # iteration, and this test depends on the current implementation 271 # picking successive numbers. This dependency is not ideal, but 272 # makes the test much easier to write. 273 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 274 GetTestsForAllIterations( 275 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 276 277 # Make sure running the tests with random seed 1 gets the same 278 # order as in iteration 1 above. 279 [tests_with_seed1] = GetTestsForAllIterations( 280 {}, [ShuffleFlag(), RandomSeedFlag(1)]) 281 self.assertEqual(tests_in_iteration1, tests_with_seed1) 282 283 # Make sure running the tests with random seed 2 gets the same 284 # order as in iteration 2 above. Success means that Google Test 285 # correctly restores the test order before re-shuffling at the 286 # beginning of iteration 2. 287 [tests_with_seed2] = GetTestsForAllIterations( 288 {}, [ShuffleFlag(), RandomSeedFlag(2)]) 289 self.assertEqual(tests_in_iteration2, tests_with_seed2) 290 291 # Make sure running the tests with random seed 3 gets the same 292 # order as in iteration 3 above. Success means that Google Test 293 # correctly restores the test order before re-shuffling at the 294 # beginning of iteration 3. 295 [tests_with_seed3] = GetTestsForAllIterations( 296 {}, [ShuffleFlag(), RandomSeedFlag(3)]) 297 self.assertEqual(tests_in_iteration3, tests_with_seed3) 298 299 def testShuffleGeneratesNewOrderInEachIteration(self): 300 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 301 GetTestsForAllIterations( 302 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 303 304 self.assert_(tests_in_iteration1 != tests_in_iteration2, 305 tests_in_iteration1) 306 self.assert_(tests_in_iteration1 != tests_in_iteration3, 307 tests_in_iteration1) 308 self.assert_(tests_in_iteration2 != tests_in_iteration3, 309 tests_in_iteration2) 310 311 def testShuffleShardedTestsPreservesPartition(self): 312 # If we run M tests on N shards, the same M tests should be run in 313 # total, regardless of the random seeds used by the shards. 314 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 315 SHARD_INDEX_ENV_VAR: '0'}, 316 [ShuffleFlag(), RandomSeedFlag(1)]) 317 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 318 SHARD_INDEX_ENV_VAR: '1'}, 319 [ShuffleFlag(), RandomSeedFlag(20)]) 320 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 321 SHARD_INDEX_ENV_VAR: '2'}, 322 [ShuffleFlag(), RandomSeedFlag(25)]) 323 sorted_sharded_tests = tests1 + tests2 + tests3 324 sorted_sharded_tests.sort() 325 sorted_active_tests = [] 326 sorted_active_tests.extend(ACTIVE_TESTS) 327 sorted_active_tests.sort() 328 self.assertEqual(sorted_active_tests, sorted_sharded_tests) 329 330 if __name__ == '__main__': 331 gtest_test_utils.Main() 332
