1 #!/usr/bin/env python 2 # 3 # Copyright 2009 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Verifies that test shuffling works.""" 32 33 __author__ = 'wan (at] google.com (Zhanyong Wan)' 34 35 import os 36 import gtest_test_utils 37 38 # Command to run the gtest_shuffle_test_ program. 39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') 40 41 # The environment variables for test sharding. 42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 44 45 TEST_FILTER = 'A*.A:A*.B:C*' 46 47 ALL_TESTS = [] 48 ACTIVE_TESTS = [] 49 FILTERED_TESTS = [] 50 SHARDED_TESTS = [] 51 52 SHUFFLED_ALL_TESTS = [] 53 SHUFFLED_ACTIVE_TESTS = [] 54 SHUFFLED_FILTERED_TESTS = [] 55 SHUFFLED_SHARDED_TESTS = [] 56 57 58 def AlsoRunDisabledTestsFlag(): 59 return '--gtest_also_run_disabled_tests' 60 61 62 def FilterFlag(test_filter): 63 return '--gtest_filter=%s' % (test_filter,) 64 65 66 def RepeatFlag(n): 67 return '--gtest_repeat=%s' % (n,) 68 69 70 def ShuffleFlag(): 71 return '--gtest_shuffle' 72 73 74 def RandomSeedFlag(n): 75 return '--gtest_random_seed=%s' % (n,) 76 77 78 def RunAndReturnOutput(extra_env, args): 79 """Runs the test program and returns its output.""" 80 81 environ_copy = os.environ.copy() 82 environ_copy.update(extra_env) 83 84 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy, 85 capture_stderr=False).output 86 87 88 def GetTestsForAllIterations(extra_env, args): 89 """Runs the test program and returns a list of test lists. 90 91 Args: 92 extra_env: a map from environment variables to their values 93 args: command line flags to pass to gtest_shuffle_test_ 94 95 Returns: 96 A list where the i-th element is the list of tests run in the i-th 97 test iteration. 98 """ 99 100 test_iterations = [] 101 for line in RunAndReturnOutput(extra_env, args).split('\n'): 102 if line.startswith('----'): 103 tests = [] 104 test_iterations.append(tests) 105 elif line.strip(): 106 tests.append(line.strip()) # 'TestCaseName.TestName' 107 108 return test_iterations 109 110 111 def GetTestCases(tests): 112 """Returns a list of test cases in the given full test names. 113 114 Args: 115 tests: a list of full test names 116 117 Returns: 118 A list of test cases from 'tests', in their original order. 119 Consecutive duplicates are removed. 120 """ 121 122 test_cases = [] 123 for test in tests: 124 test_case = test.split('.')[0] 125 if not test_case in test_cases: 126 test_cases.append(test_case) 127 128 return test_cases 129 130 131 def CalculateTestLists(): 132 """Calculates the list of tests run under different flags.""" 133 134 if not ALL_TESTS: 135 ALL_TESTS.extend( 136 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 137 138 if not ACTIVE_TESTS: 139 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 140 141 if not FILTERED_TESTS: 142 FILTERED_TESTS.extend( 143 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 144 145 if not SHARDED_TESTS: 146 SHARDED_TESTS.extend( 147 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 148 SHARD_INDEX_ENV_VAR: '1'}, 149 [])[0]) 150 151 if not SHUFFLED_ALL_TESTS: 152 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 153 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 154 155 if not SHUFFLED_ACTIVE_TESTS: 156 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 157 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 158 159 if not SHUFFLED_FILTERED_TESTS: 160 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 161 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 162 163 if not SHUFFLED_SHARDED_TESTS: 164 SHUFFLED_SHARDED_TESTS.extend( 165 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 166 SHARD_INDEX_ENV_VAR: '1'}, 167 [ShuffleFlag(), RandomSeedFlag(1)])[0]) 168 169 170 class GTestShuffleUnitTest(gtest_test_utils.TestCase): 171 """Tests test shuffling.""" 172 173 def setUp(self): 174 CalculateTestLists() 175 176 def testShufflePreservesNumberOfTests(self): 177 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 178 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 179 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 180 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 181 182 def testShuffleChangesTestOrder(self): 183 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 184 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 185 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 186 SHUFFLED_FILTERED_TESTS) 187 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 188 SHUFFLED_SHARDED_TESTS) 189 190 def testShuffleChangesTestCaseOrder(self): 191 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 192 GetTestCases(SHUFFLED_ALL_TESTS)) 193 self.assert_( 194 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 195 GetTestCases(SHUFFLED_ACTIVE_TESTS)) 196 self.assert_( 197 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 198 GetTestCases(SHUFFLED_FILTERED_TESTS)) 199 self.assert_( 200 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 201 GetTestCases(SHUFFLED_SHARDED_TESTS)) 202 203 def testShuffleDoesNotRepeatTest(self): 204 for test in SHUFFLED_ALL_TESTS: 205 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 206 '%s appears more than once' % (test,)) 207 for test in SHUFFLED_ACTIVE_TESTS: 208 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 209 '%s appears more than once' % (test,)) 210 for test in SHUFFLED_FILTERED_TESTS: 211 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 212 '%s appears more than once' % (test,)) 213 for test in SHUFFLED_SHARDED_TESTS: 214 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 215 '%s appears more than once' % (test,)) 216 217 def testShuffleDoesNotCreateNewTest(self): 218 for test in SHUFFLED_ALL_TESTS: 219 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 220 for test in SHUFFLED_ACTIVE_TESTS: 221 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 222 for test in SHUFFLED_FILTERED_TESTS: 223 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 224 for test in SHUFFLED_SHARDED_TESTS: 225 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 226 227 def testShuffleIncludesAllTests(self): 228 for test in ALL_TESTS: 229 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 230 for test in ACTIVE_TESTS: 231 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 232 for test in FILTERED_TESTS: 233 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 234 for test in SHARDED_TESTS: 235 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 236 237 def testShuffleLeavesDeathTestsAtFront(self): 238 non_death_test_found = False 239 for test in SHUFFLED_ACTIVE_TESTS: 240 if 'DeathTest.' in test: 241 self.assert_(not non_death_test_found, 242 '%s appears after a non-death test' % (test,)) 243 else: 244 non_death_test_found = True 245 246 def _VerifyTestCasesDoNotInterleave(self, tests): 247 test_cases = [] 248 for test in tests: 249 [test_case, _] = test.split('.') 250 if test_cases and test_cases[-1] != test_case: 251 test_cases.append(test_case) 252 self.assertEqual(1, test_cases.count(test_case), 253 'Test case %s is not grouped together in %s' % 254 (test_case, tests)) 255 256 def testShuffleDoesNotInterleaveTestCases(self): 257 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 258 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 259 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 260 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 261 262 def testShuffleRestoresOrderAfterEachIteration(self): 263 # Get the test lists in all 3 iterations, using random seed 1, 2, 264 # and 3 respectively. Google Test picks a different seed in each 265 # iteration, and this test depends on the current implementation 266 # picking successive numbers. This dependency is not ideal, but 267 # makes the test much easier to write. 268 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 269 GetTestsForAllIterations( 270 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 271 272 # Make sure running the tests with random seed 1 gets the same 273 # order as in iteration 1 above. 274 [tests_with_seed1] = GetTestsForAllIterations( 275 {}, [ShuffleFlag(), RandomSeedFlag(1)]) 276 self.assertEqual(tests_in_iteration1, tests_with_seed1) 277 278 # Make sure running the tests with random seed 2 gets the same 279 # order as in iteration 2 above. Success means that Google Test 280 # correctly restores the test order before re-shuffling at the 281 # beginning of iteration 2. 282 [tests_with_seed2] = GetTestsForAllIterations( 283 {}, [ShuffleFlag(), RandomSeedFlag(2)]) 284 self.assertEqual(tests_in_iteration2, tests_with_seed2) 285 286 # Make sure running the tests with random seed 3 gets the same 287 # order as in iteration 3 above. Success means that Google Test 288 # correctly restores the test order before re-shuffling at the 289 # beginning of iteration 3. 290 [tests_with_seed3] = GetTestsForAllIterations( 291 {}, [ShuffleFlag(), RandomSeedFlag(3)]) 292 self.assertEqual(tests_in_iteration3, tests_with_seed3) 293 294 def testShuffleGeneratesNewOrderInEachIteration(self): 295 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 296 GetTestsForAllIterations( 297 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 298 299 self.assert_(tests_in_iteration1 != tests_in_iteration2, 300 tests_in_iteration1) 301 self.assert_(tests_in_iteration1 != tests_in_iteration3, 302 tests_in_iteration1) 303 self.assert_(tests_in_iteration2 != tests_in_iteration3, 304 tests_in_iteration2) 305 306 def testShuffleShardedTestsPreservesPartition(self): 307 # If we run M tests on N shards, the same M tests should be run in 308 # total, regardless of the random seeds used by the shards. 309 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 310 SHARD_INDEX_ENV_VAR: '0'}, 311 [ShuffleFlag(), RandomSeedFlag(1)]) 312 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 313 SHARD_INDEX_ENV_VAR: '1'}, 314 [ShuffleFlag(), RandomSeedFlag(20)]) 315 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 316 SHARD_INDEX_ENV_VAR: '2'}, 317 [ShuffleFlag(), RandomSeedFlag(25)]) 318 sorted_sharded_tests = tests1 + tests2 + tests3 319 sorted_sharded_tests.sort() 320 sorted_active_tests = [] 321 sorted_active_tests.extend(ACTIVE_TESTS) 322 sorted_active_tests.sort() 323 self.assertEqual(sorted_active_tests, sorted_sharded_tests) 324 325 if __name__ == '__main__': 326 gtest_test_utils.Main() 327
