1 #!/usr/bin/env python 2 # 3 # Copyright 2009 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Verifies that test shuffling works.""" 32 33 __author__ = 'wan (at] google.com (Zhanyong Wan)' 34 35 import os 36 import gtest_test_utils 37 38 # Command to run the gtest_shuffle_test_ program. 39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') 40 41 # The environment variables for test sharding. 42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 44 45 TEST_FILTER = 'A*.A:A*.B:C*' 46 47 ALL_TESTS = [] 48 ACTIVE_TESTS = [] 49 FILTERED_TESTS = [] 50 SHARDED_TESTS = [] 51 52 SHUFFLED_ALL_TESTS = [] 53 SHUFFLED_ACTIVE_TESTS = [] 54 SHUFFLED_FILTERED_TESTS = [] 55 SHUFFLED_SHARDED_TESTS = [] 56 57 58 def AlsoRunDisabledTestsFlag(): 59 return '--gtest_also_run_disabled_tests' 60 61 62 def FilterFlag(test_filter): 63 return '--gtest_filter=%s' % (test_filter,) 64 65 66 def RepeatFlag(n): 67 return '--gtest_repeat=%s' % (n,) 68 69 70 def ShuffleFlag(): 71 return '--gtest_shuffle' 72 73 74 def RandomSeedFlag(n): 75 return '--gtest_random_seed=%s' % (n,) 76 77 78 def RunAndReturnOutput(extra_env, args): 79 """Runs the test program and returns its output.""" 80 81 environ_copy = os.environ.copy() 82 environ_copy.update(extra_env) 83 84 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 85 86 87 def GetTestsForAllIterations(extra_env, args): 88 """Runs the test program and returns a list of test lists. 89 90 Args: 91 extra_env: a map from environment variables to their values 92 args: command line flags to pass to gtest_shuffle_test_ 93 94 Returns: 95 A list where the i-th element is the list of tests run in the i-th 96 test iteration. 97 """ 98 99 test_iterations = [] 100 for line in RunAndReturnOutput(extra_env, args).split('\n'): 101 if line.startswith('----'): 102 tests = [] 103 test_iterations.append(tests) 104 elif line.strip(): 105 tests.append(line.strip()) # 'TestCaseName.TestName' 106 107 return test_iterations 108 109 110 def GetTestCases(tests): 111 """Returns a list of test cases in the given full test names. 112 113 Args: 114 tests: a list of full test names 115 116 Returns: 117 A list of test cases from 'tests', in their original order. 118 Consecutive duplicates are removed. 119 """ 120 121 test_cases = [] 122 for test in tests: 123 test_case = test.split('.')[0] 124 if not test_case in test_cases: 125 test_cases.append(test_case) 126 127 return test_cases 128 129 130 def CalculateTestLists(): 131 """Calculates the list of tests run under different flags.""" 132 133 if not ALL_TESTS: 134 ALL_TESTS.extend( 135 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 136 137 if not ACTIVE_TESTS: 138 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 139 140 if not FILTERED_TESTS: 141 FILTERED_TESTS.extend( 142 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 143 144 if not SHARDED_TESTS: 145 SHARDED_TESTS.extend( 146 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 147 SHARD_INDEX_ENV_VAR: '1'}, 148 [])[0]) 149 150 if not SHUFFLED_ALL_TESTS: 151 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 152 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 153 154 if not SHUFFLED_ACTIVE_TESTS: 155 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 156 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 157 158 if not SHUFFLED_FILTERED_TESTS: 159 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 160 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 161 162 if not SHUFFLED_SHARDED_TESTS: 163 SHUFFLED_SHARDED_TESTS.extend( 164 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 165 SHARD_INDEX_ENV_VAR: '1'}, 166 [ShuffleFlag(), RandomSeedFlag(1)])[0]) 167 168 169 class GTestShuffleUnitTest(gtest_test_utils.TestCase): 170 """Tests test shuffling.""" 171 172 def setUp(self): 173 CalculateTestLists() 174 175 def testShufflePreservesNumberOfTests(self): 176 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 177 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 178 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 179 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 180 181 def testShuffleChangesTestOrder(self): 182 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 183 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 184 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 185 SHUFFLED_FILTERED_TESTS) 186 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 187 SHUFFLED_SHARDED_TESTS) 188 189 def testShuffleChangesTestCaseOrder(self): 190 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 191 GetTestCases(SHUFFLED_ALL_TESTS)) 192 self.assert_( 193 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 194 GetTestCases(SHUFFLED_ACTIVE_TESTS)) 195 self.assert_( 196 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 197 GetTestCases(SHUFFLED_FILTERED_TESTS)) 198 self.assert_( 199 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 200 GetTestCases(SHUFFLED_SHARDED_TESTS)) 201 202 def testShuffleDoesNotRepeatTest(self): 203 for test in SHUFFLED_ALL_TESTS: 204 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 205 '%s appears more than once' % (test,)) 206 for test in SHUFFLED_ACTIVE_TESTS: 207 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 208 '%s appears more than once' % (test,)) 209 for test in SHUFFLED_FILTERED_TESTS: 210 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 211 '%s appears more than once' % (test,)) 212 for test in SHUFFLED_SHARDED_TESTS: 213 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 214 '%s appears more than once' % (test,)) 215 216 def testShuffleDoesNotCreateNewTest(self): 217 for test in SHUFFLED_ALL_TESTS: 218 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 219 for test in SHUFFLED_ACTIVE_TESTS: 220 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 221 for test in SHUFFLED_FILTERED_TESTS: 222 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 223 for test in SHUFFLED_SHARDED_TESTS: 224 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 225 226 def testShuffleIncludesAllTests(self): 227 for test in ALL_TESTS: 228 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 229 for test in ACTIVE_TESTS: 230 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 231 for test in FILTERED_TESTS: 232 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 233 for test in SHARDED_TESTS: 234 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 235 236 def testShuffleLeavesDeathTestsAtFront(self): 237 non_death_test_found = False 238 for test in SHUFFLED_ACTIVE_TESTS: 239 if 'DeathTest.' in test: 240 self.assert_(not non_death_test_found, 241 '%s appears after a non-death test' % (test,)) 242 else: 243 non_death_test_found = True 244 245 def _VerifyTestCasesDoNotInterleave(self, tests): 246 test_cases = [] 247 for test in tests: 248 [test_case, _] = test.split('.') 249 if test_cases and test_cases[-1] != test_case: 250 test_cases.append(test_case) 251 self.assertEqual(1, test_cases.count(test_case), 252 'Test case %s is not grouped together in %s' % 253 (test_case, tests)) 254 255 def testShuffleDoesNotInterleaveTestCases(self): 256 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 257 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 258 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 259 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 260 261 def testShuffleRestoresOrderAfterEachIteration(self): 262 # Get the test lists in all 3 iterations, using random seed 1, 2, 263 # and 3 respectively. Google Test picks a different seed in each 264 # iteration, and this test depends on the current implementation 265 # picking successive numbers. This dependency is not ideal, but 266 # makes the test much easier to write. 267 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 268 GetTestsForAllIterations( 269 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 270 271 # Make sure running the tests with random seed 1 gets the same 272 # order as in iteration 1 above. 273 [tests_with_seed1] = GetTestsForAllIterations( 274 {}, [ShuffleFlag(), RandomSeedFlag(1)]) 275 self.assertEqual(tests_in_iteration1, tests_with_seed1) 276 277 # Make sure running the tests with random seed 2 gets the same 278 # order as in iteration 2 above. Success means that Google Test 279 # correctly restores the test order before re-shuffling at the 280 # beginning of iteration 2. 281 [tests_with_seed2] = GetTestsForAllIterations( 282 {}, [ShuffleFlag(), RandomSeedFlag(2)]) 283 self.assertEqual(tests_in_iteration2, tests_with_seed2) 284 285 # Make sure running the tests with random seed 3 gets the same 286 # order as in iteration 3 above. Success means that Google Test 287 # correctly restores the test order before re-shuffling at the 288 # beginning of iteration 3. 289 [tests_with_seed3] = GetTestsForAllIterations( 290 {}, [ShuffleFlag(), RandomSeedFlag(3)]) 291 self.assertEqual(tests_in_iteration3, tests_with_seed3) 292 293 def testShuffleGeneratesNewOrderInEachIteration(self): 294 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 295 GetTestsForAllIterations( 296 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 297 298 self.assert_(tests_in_iteration1 != tests_in_iteration2, 299 tests_in_iteration1) 300 self.assert_(tests_in_iteration1 != tests_in_iteration3, 301 tests_in_iteration1) 302 self.assert_(tests_in_iteration2 != tests_in_iteration3, 303 tests_in_iteration2) 304 305 def testShuffleShardedTestsPreservesPartition(self): 306 # If we run M tests on N shards, the same M tests should be run in 307 # total, regardless of the random seeds used by the shards. 308 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 309 SHARD_INDEX_ENV_VAR: '0'}, 310 [ShuffleFlag(), RandomSeedFlag(1)]) 311 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 312 SHARD_INDEX_ENV_VAR: '1'}, 313 [ShuffleFlag(), RandomSeedFlag(20)]) 314 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 315 SHARD_INDEX_ENV_VAR: '2'}, 316 [ShuffleFlag(), RandomSeedFlag(25)]) 317 sorted_sharded_tests = tests1 + tests2 + tests3 318 sorted_sharded_tests.sort() 319 sorted_active_tests = [] 320 sorted_active_tests.extend(ACTIVE_TESTS) 321 sorted_active_tests.sort() 322 self.assertEqual(sorted_active_tests, sorted_sharded_tests) 323 324 if __name__ == '__main__': 325 gtest_test_utils.Main() 326
