1 #!/usr/bin/python2 2 3 # Copyright 2012 Google Inc. All Rights Reserved. 4 """Tests for bisecting tool.""" 5 6 from __future__ import print_function 7 8 __author__ = 'shenhan (at] google.com (Han Shen)' 9 10 import os 11 import random 12 import sys 13 import unittest 14 15 from cros_utils import command_executer 16 from binary_search_tool import binary_search_state 17 from binary_search_tool import bisect 18 19 import common 20 import gen_obj 21 22 23 def GenObj(): 24 obj_num = random.randint(100, 1000) 25 bad_obj_num = random.randint(obj_num / 100, obj_num / 20) 26 if bad_obj_num == 0: 27 bad_obj_num = 1 28 gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)]) 29 30 31 def CleanObj(): 32 os.remove(common.OBJECTS_FILE) 33 os.remove(common.WORKING_SET_FILE) 34 print('Deleted "{0}" and "{1}"'.format(common.OBJECTS_FILE, 35 common.WORKING_SET_FILE)) 36 37 38 class BisectTest(unittest.TestCase): 39 """Tests for bisect.py""" 40 41 def setUp(self): 42 with open('./is_setup', 'w'): 43 pass 44 45 try: 46 os.remove(binary_search_state.STATE_FILE) 47 except OSError: 48 pass 49 50 def tearDown(self): 51 try: 52 os.remove('./is_setup') 53 os.remove(os.readlink(binary_search_state.STATE_FILE)) 54 os.remove(binary_search_state.STATE_FILE) 55 except OSError: 56 pass 57 58 class FullBisector(bisect.Bisector): 59 """Test bisector to test bisect.py with""" 60 61 def __init__(self, options, overrides): 62 super(BisectTest.FullBisector, self).__init__(options, overrides) 63 64 def PreRun(self): 65 GenObj() 66 return 0 67 68 def Run(self): 69 return binary_search_state.Run(get_initial_items='./gen_init_list.py', 70 switch_to_good='./switch_to_good.py', 71 switch_to_bad='./switch_to_bad.py', 72 test_script='./is_good.py', 73 prune=True, 74 file_args=True) 75 76 def PostRun(self): 77 CleanObj() 78 return 0 79 80 def test_full_bisector(self): 81 ret = bisect.Run(self.FullBisector({}, {})) 82 self.assertEquals(ret, 0) 83 self.assertFalse(os.path.exists(common.OBJECTS_FILE)) 84 self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) 85 86 def check_output(self): 87 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 88 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 89 'tail -n1')) 90 ls = out.splitlines() 91 self.assertEqual(len(ls), 1) 92 line = ls[0] 93 94 _, _, bad_ones = line.partition('Bad items are: ') 95 bad_ones = bad_ones.split() 96 expected_result = common.ReadObjectsFile() 97 98 # Reconstruct objects file from bad_ones and compare 99 actual_result = [0] * len(expected_result) 100 for bad_obj in bad_ones: 101 actual_result[int(bad_obj)] = 1 102 103 self.assertEqual(actual_result, expected_result) 104 105 106 class BisectingUtilsTest(unittest.TestCase): 107 """Tests for bisecting tool.""" 108 109 def setUp(self): 110 """Generate [100-1000] object files, and 1-5% of which are bad ones.""" 111 GenObj() 112 113 with open('./is_setup', 'w'): 114 pass 115 116 try: 117 os.remove(binary_search_state.STATE_FILE) 118 except OSError: 119 pass 120 121 def tearDown(self): 122 """Cleanup temp files.""" 123 CleanObj() 124 125 try: 126 os.remove(os.readlink(binary_search_state.STATE_FILE)) 127 except OSError: 128 pass 129 130 cleanup_list = ['./is_setup', binary_search_state.STATE_FILE, 131 'noinc_prune_bad', 'noinc_prune_good'] 132 for f in cleanup_list: 133 if os.path.exists(f): 134 os.remove(f) 135 136 def runTest(self): 137 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 138 switch_to_good='./switch_to_good.py', 139 switch_to_bad='./switch_to_bad.py', 140 test_script='./is_good.py', 141 prune=True, 142 file_args=True) 143 self.assertEquals(ret, 0) 144 self.check_output() 145 146 def test_arg_parse(self): 147 args = ['--get_initial_items', './gen_init_list.py', '--switch_to_good', 148 './switch_to_good.py', '--switch_to_bad', './switch_to_bad.py', 149 '--test_script', './is_good.py', '--prune', '--file_args'] 150 ret = binary_search_state.Main(args) 151 self.assertEquals(ret, 0) 152 self.check_output() 153 154 def test_test_setup_script(self): 155 os.remove('./is_setup') 156 with self.assertRaises(AssertionError): 157 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 158 switch_to_good='./switch_to_good.py', 159 switch_to_bad='./switch_to_bad.py', 160 test_script='./is_good.py', 161 prune=True, 162 file_args=True) 163 164 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 165 switch_to_good='./switch_to_good.py', 166 switch_to_bad='./switch_to_bad.py', 167 test_script='./is_good.py', 168 test_setup_script='./test_setup.py', 169 prune=True, 170 file_args=True) 171 self.assertEquals(ret, 0) 172 self.check_output() 173 174 def test_bad_test_setup_script(self): 175 with self.assertRaises(AssertionError): 176 binary_search_state.Run(get_initial_items='./gen_init_list.py', 177 switch_to_good='./switch_to_good.py', 178 switch_to_bad='./switch_to_bad.py', 179 test_script='./is_good.py', 180 test_setup_script='./test_setup_bad.py', 181 prune=True, 182 file_args=True) 183 184 def test_bad_save_state(self): 185 state_file = binary_search_state.STATE_FILE 186 hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE) 187 188 with open(state_file, 'w') as f: 189 f.write('test123') 190 191 bss = binary_search_state.MockBinarySearchState() 192 with self.assertRaises(binary_search_state.Error): 193 bss.SaveState() 194 195 with open(state_file, 'r') as f: 196 self.assertEquals(f.read(), 'test123') 197 198 os.remove(state_file) 199 200 # Cleanup generated save state that has no symlink 201 files = os.listdir(os.getcwd()) 202 save_states = [x for x in files if x.startswith(hidden_state_file)] 203 _ = [os.remove(x) for x in save_states] 204 205 def test_save_state(self): 206 state_file = binary_search_state.STATE_FILE 207 208 bss = binary_search_state.MockBinarySearchState() 209 bss.SaveState() 210 self.assertTrue(os.path.exists(state_file)) 211 first_state = os.readlink(state_file) 212 213 bss.SaveState() 214 second_state = os.readlink(state_file) 215 self.assertTrue(os.path.exists(state_file)) 216 self.assertTrue(second_state != first_state) 217 self.assertFalse(os.path.exists(first_state)) 218 219 bss.RemoveState() 220 self.assertFalse(os.path.islink(state_file)) 221 self.assertFalse(os.path.exists(second_state)) 222 223 def test_load_state(self): 224 test_items = [1, 2, 3, 4, 5] 225 226 bss = binary_search_state.MockBinarySearchState() 227 bss.all_items = test_items 228 bss.currently_good_items = set([1, 2, 3]) 229 bss.currently_bad_items = set([4, 5]) 230 bss.SaveState() 231 232 bss = None 233 234 bss2 = binary_search_state.MockBinarySearchState.LoadState() 235 self.assertEquals(bss2.all_items, test_items) 236 self.assertEquals(bss2.currently_good_items, set([])) 237 self.assertEquals(bss2.currently_bad_items, set([])) 238 239 def test_tmp_cleanup(self): 240 bss = binary_search_state.MockBinarySearchState( 241 get_initial_items='echo "0\n1\n2\n3"', 242 switch_to_good='./switch_tmp.py', 243 file_args=True) 244 bss.SwitchToGood(['0', '1', '2', '3']) 245 246 tmp_file = None 247 with open('tmp_file', 'r') as f: 248 tmp_file = f.read() 249 os.remove('tmp_file') 250 251 self.assertFalse(os.path.exists(tmp_file)) 252 ws = common.ReadWorkingSet() 253 for i in range(3): 254 self.assertEquals(ws[i], 42) 255 256 def test_verify_fail(self): 257 bss = binary_search_state.MockBinarySearchState( 258 get_initial_items='./gen_init_list.py', 259 switch_to_good='./switch_to_bad.py', 260 switch_to_bad='./switch_to_good.py', 261 test_script='./is_good.py', 262 prune=True, 263 file_args=True, 264 verify=True) 265 with self.assertRaises(AssertionError): 266 bss.DoVerify() 267 268 def test_early_terminate(self): 269 bss = binary_search_state.MockBinarySearchState( 270 get_initial_items='./gen_init_list.py', 271 switch_to_good='./switch_to_good.py', 272 switch_to_bad='./switch_to_bad.py', 273 test_script='./is_good.py', 274 prune=True, 275 file_args=True, 276 iterations=1) 277 bss.DoSearch() 278 self.assertFalse(bss.found_items) 279 280 def test_no_prune(self): 281 bss = binary_search_state.MockBinarySearchState( 282 get_initial_items='./gen_init_list.py', 283 switch_to_good='./switch_to_good.py', 284 switch_to_bad='./switch_to_bad.py', 285 test_script='./is_good.py', 286 test_setup_script='./test_setup.py', 287 prune=False, 288 file_args=True) 289 bss.DoSearch() 290 self.assertEquals(len(bss.found_items), 1) 291 292 bad_objs = common.ReadObjectsFile() 293 found_obj = int(bss.found_items.pop()) 294 self.assertEquals(bad_objs[found_obj], 1) 295 296 def test_set_file(self): 297 binary_search_state.Run(get_initial_items='./gen_init_list.py', 298 switch_to_good='./switch_to_good_set_file.py', 299 switch_to_bad='./switch_to_bad_set_file.py', 300 test_script='./is_good.py', 301 prune=True, 302 file_args=True, 303 verify=True) 304 self.check_output() 305 306 def test_noincremental_prune(self): 307 ret = binary_search_state.Run( 308 get_initial_items='./gen_init_list.py', 309 switch_to_good='./switch_to_good_noinc_prune.py', 310 switch_to_bad='./switch_to_bad_noinc_prune.py', 311 test_script='./is_good_noinc_prune.py', 312 test_setup_script='./test_setup.py', 313 prune=True, 314 noincremental=True, 315 file_args=True, 316 verify=False) 317 self.assertEquals(ret, 0) 318 self.check_output() 319 320 def check_output(self): 321 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 322 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 323 'tail -n1')) 324 ls = out.splitlines() 325 self.assertEqual(len(ls), 1) 326 line = ls[0] 327 328 _, _, bad_ones = line.partition('Bad items are: ') 329 bad_ones = bad_ones.split() 330 expected_result = common.ReadObjectsFile() 331 332 # Reconstruct objects file from bad_ones and compare 333 actual_result = [0] * len(expected_result) 334 for bad_obj in bad_ones: 335 actual_result[int(bad_obj)] = 1 336 337 self.assertEqual(actual_result, expected_result) 338 339 340 class BisectStressTest(unittest.TestCase): 341 """Stress tests for bisecting tool.""" 342 343 def test_every_obj_bad(self): 344 amt = 25 345 gen_obj.Main(['--obj_num', str(amt), '--bad_obj_num', str(amt)]) 346 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 347 switch_to_good='./switch_to_good.py', 348 switch_to_bad='./switch_to_bad.py', 349 test_script='./is_good.py', 350 prune=True, 351 file_args=True, 352 verify=False) 353 self.assertEquals(ret, 0) 354 self.check_output() 355 356 def test_every_index_is_bad(self): 357 amt = 25 358 for i in range(amt): 359 obj_list = ['0'] * amt 360 obj_list[i] = '1' 361 obj_list = ','.join(obj_list) 362 gen_obj.Main(['--obj_list', obj_list]) 363 ret = binary_search_state.Run(get_initial_items='./gen_init_list.py', 364 switch_to_good='./switch_to_good.py', 365 switch_to_bad='./switch_to_bad.py', 366 test_setup_script='./test_setup.py', 367 test_script='./is_good.py', 368 prune=True, 369 file_args=True) 370 self.assertEquals(ret, 0) 371 self.check_output() 372 373 def check_output(self): 374 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 375 ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | ' 376 'tail -n1')) 377 ls = out.splitlines() 378 self.assertEqual(len(ls), 1) 379 line = ls[0] 380 381 _, _, bad_ones = line.partition('Bad items are: ') 382 bad_ones = bad_ones.split() 383 expected_result = common.ReadObjectsFile() 384 385 # Reconstruct objects file from bad_ones and compare 386 actual_result = [0] * len(expected_result) 387 for bad_obj in bad_ones: 388 actual_result[int(bad_obj)] = 1 389 390 self.assertEqual(actual_result, expected_result) 391 392 393 def Main(argv): 394 num_tests = 2 395 if len(argv) > 1: 396 num_tests = int(argv[1]) 397 398 suite = unittest.TestSuite() 399 for _ in range(0, num_tests): 400 suite.addTest(BisectingUtilsTest()) 401 suite.addTest(BisectingUtilsTest('test_arg_parse')) 402 suite.addTest(BisectingUtilsTest('test_test_setup_script')) 403 suite.addTest(BisectingUtilsTest('test_bad_test_setup_script')) 404 suite.addTest(BisectingUtilsTest('test_bad_save_state')) 405 suite.addTest(BisectingUtilsTest('test_save_state')) 406 suite.addTest(BisectingUtilsTest('test_load_state')) 407 suite.addTest(BisectingUtilsTest('test_tmp_cleanup')) 408 suite.addTest(BisectingUtilsTest('test_verify_fail')) 409 suite.addTest(BisectingUtilsTest('test_early_terminate')) 410 suite.addTest(BisectingUtilsTest('test_no_prune')) 411 suite.addTest(BisectingUtilsTest('test_set_file')) 412 suite.addTest(BisectingUtilsTest('test_noincremental_prune')) 413 suite.addTest(BisectTest('test_full_bisector')) 414 suite.addTest(BisectStressTest('test_every_obj_bad')) 415 suite.addTest(BisectStressTest('test_every_index_is_bad')) 416 runner = unittest.TextTestRunner() 417 runner.run(suite) 418 419 420 if __name__ == '__main__': 421 Main(sys.argv) 422
