1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10 import collections 11 from collections import OrderedDict 12 import os 13 import shutil 14 import tempfile 15 import unittest 16 17 import mock 18 import mox 19 20 import common 21 22 from autotest_lib.client.common_lib import base_job 23 from autotest_lib.client.common_lib import control_data 24 from autotest_lib.client.common_lib import error 25 from autotest_lib.client.common_lib import priorities 26 from autotest_lib.client.common_lib import utils 27 from autotest_lib.client.common_lib.cros import dev_server 28 from autotest_lib.server import frontend 29 from autotest_lib.server.cros import provision 30 from autotest_lib.server.cros.dynamic_suite import control_file_getter 31 from autotest_lib.server.cros.dynamic_suite import constants 32 from autotest_lib.server.cros.dynamic_suite import job_status 33 from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34 from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 35 from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 36 from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 37 from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 38 from autotest_lib.server.cros.dynamic_suite.suite import Suite 39 40 41 class SuiteTest(mox.MoxTestBase): 42 """Unit tests for dynamic_suite Suite class. 43 44 @var _BUILDS: fake build 45 @var _TAG: fake suite tag 46 """ 47 48 _BOARD = 'board:board' 49 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 50 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 51 _TAG = 'au' 52 _ATTR = {'attr:attr'} 53 _DEVSERVER_HOST = 'http://dontcare:8080' 54 _FAKE_JOB_ID = 10 55 56 57 def setUp(self): 58 """Setup.""" 59 super(SuiteTest, self).setUp() 60 self.maxDiff = None 61 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 62 SuiteBase.ENABLE_CONTROLS_IN_BATCH = False 63 self.afe = self.mox.CreateMock(frontend.AFE) 64 self.tko = self.mox.CreateMock(frontend.TKO) 65 66 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 67 68 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 69 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 70 71 self.files = OrderedDict( 72 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 73 'FAST', job_retries=None)), 74 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 75 'SHORT', dependencies=['feta'])), 76 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 77 'MEDIUM')), 78 ('four', FakeControlData('other', self._ATTR, 'data_four', 79 'LONG', dependencies=['arugula'])), 80 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 81 'LONG', dependencies=['arugula', 82 'caligula'])), 83 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 84 'LENGTHY')), 85 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 86 'FAST', job_retries=1))]) 87 88 self.files_to_filter = { 89 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 90 'gets filtered'), 91 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 92 'gets filtered')} 93 94 95 def tearDown(self): 96 """Teardown.""" 97 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 98 super(SuiteTest, self).tearDown() 99 shutil.rmtree(self.tmpdir, ignore_errors=True) 100 101 102 def expect_control_file_parsing(self, suite_name=_TAG): 103 """Expect an attempt to parse the 'control files' in |self.files|. 104 105 @param suite_name: The suite name to parse control files for. 106 """ 107 all_files = self.files.keys() + self.files_to_filter.keys() 108 self._set_control_file_parsing_expectations(False, all_files, 109 self.files, suite_name) 110 111 112 def _set_control_file_parsing_expectations(self, already_stubbed, 113 file_list, files_to_parse, 114 suite_name): 115 """Expect an attempt to parse the 'control files' in |files|. 116 117 @param already_stubbed: parse_control_string already stubbed out. 118 @param file_list: the files the dev server returns 119 @param files_to_parse: the {'name': FakeControlData} dict of files we 120 expect to get parsed. 121 """ 122 if not already_stubbed: 123 self.mox.StubOutWithMock(control_data, 'parse_control_string') 124 125 self.getter.get_control_file_list( 126 suite_name=suite_name).AndReturn(file_list) 127 for file, data in files_to_parse.iteritems(): 128 self.getter.get_control_file_contents( 129 file).InAnyOrder().AndReturn(data.string) 130 control_data.parse_control_string( 131 data.string, 132 raise_warnings=True, 133 path=file).InAnyOrder().AndReturn(data) 134 135 136 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 137 """Expect an attempt to parse the contents of all control files in 138 |self.files| and |self.files_to_filter|, form them to a dict. 139 140 @param suite_name: The suite name to parse control files for. 141 """ 142 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 143 self.mox.StubOutWithMock(control_data, 'parse_control_string') 144 suite_info = {} 145 for k, v in self.files.iteritems(): 146 suite_info[k] = v.string 147 control_data.parse_control_string( 148 v.string, 149 raise_warnings=True, 150 path=k).InAnyOrder().AndReturn(v) 151 for k, v in self.files_to_filter.iteritems(): 152 suite_info[k] = v.string 153 self.getter._dev_server = self._DEVSERVER_HOST 154 self.getter.get_suite_info( 155 suite_name=suite_name).AndReturn(suite_info) 156 157 158 def testFindAllTestInBatch(self): 159 """Test switch on enable_getting_controls_in_batch for function 160 find_all_test.""" 161 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 162 self.expect_control_file_parsing_in_batch() 163 SuiteBase.ENABLE_CONTROLS_IN_BATCH = True 164 165 self.mox.ReplayAll() 166 167 predicate = lambda d: d.suite == self._TAG 168 tests = SuiteBase.find_and_parse_tests(self.getter, 169 predicate, 170 self._TAG) 171 self.assertEquals(len(tests), 6) 172 self.assertTrue(self.files['one'] in tests) 173 self.assertTrue(self.files['two'] in tests) 174 self.assertTrue(self.files['three'] in tests) 175 self.assertTrue(self.files['five'] in tests) 176 self.assertTrue(self.files['six'] in tests) 177 self.assertTrue(self.files['seven'] in tests) 178 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 179 180 181 def testFindAndParseStableTests(self): 182 """Should find only tests that match a predicate.""" 183 self.expect_control_file_parsing() 184 self.mox.ReplayAll() 185 186 predicate = lambda d: d.text == self.files['two'].string 187 tests = SuiteBase.find_and_parse_tests(self.getter, 188 predicate, 189 self._TAG) 190 self.assertEquals(len(tests), 1) 191 self.assertEquals(tests[0], self.files['two']) 192 193 194 def testFindSuiteSyntaxErrors(self): 195 """Check all control files for syntax errors. 196 197 This test actually parses all control files in the autotest directory 198 for syntax errors, by using the un-forgiving parser and pretending to 199 look for all control files with the suite attribute. 200 """ 201 autodir = os.path.abspath( 202 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 203 fs_getter = SuiteBase.create_fs_getter(autodir) 204 predicate = lambda t: hasattr(t, 'suite') 205 SuiteBase.find_and_parse_tests(fs_getter, predicate, 206 forgiving_parser=False) 207 208 209 def testFindAndParseTestsSuite(self): 210 """Should find all tests that match a predicate.""" 211 self.expect_control_file_parsing() 212 self.mox.ReplayAll() 213 214 predicate = lambda d: d.suite == self._TAG 215 tests = SuiteBase.find_and_parse_tests(self.getter, 216 predicate, 217 self._TAG) 218 self.assertEquals(len(tests), 6) 219 self.assertTrue(self.files['one'] in tests) 220 self.assertTrue(self.files['two'] in tests) 221 self.assertTrue(self.files['three'] in tests) 222 self.assertTrue(self.files['five'] in tests) 223 self.assertTrue(self.files['six'] in tests) 224 self.assertTrue(self.files['seven'] in tests) 225 226 227 def testFindAndParseTestsAttr(self): 228 """Should find all tests that match a predicate.""" 229 self.expect_control_file_parsing() 230 self.mox.ReplayAll() 231 232 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 233 tests = SuiteBase.find_and_parse_tests(self.getter, 234 predicate, 235 self._TAG) 236 self.assertEquals(len(tests), 6) 237 self.assertTrue(self.files['one'] in tests) 238 self.assertTrue(self.files['two'] in tests) 239 self.assertTrue(self.files['three'] in tests) 240 self.assertTrue(self.files['four'] in tests) 241 self.assertTrue(self.files['six'] in tests) 242 self.assertTrue(self.files['seven'] in tests) 243 244 245 def testAdHocSuiteCreation(self): 246 """Should be able to schedule an ad-hoc suite by specifying 247 a single test name.""" 248 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 249 self.mox.ReplayAll() 250 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 251 suite = Suite.create_from_predicates([predicate], self._BUILDS, 252 self._BOARD, devserver=None, 253 cf_getter=self.getter, 254 afe=self.afe, tko=self.tko) 255 256 self.assertFalse(self.files['one'] in suite.tests) 257 self.assertFalse(self.files['two'] in suite.tests) 258 self.assertFalse(self.files['four'] in suite.tests) 259 self.assertTrue(self.files['five'] in suite.tests) 260 261 262 def mock_control_file_parsing(self): 263 """Fake out find_and_parse_tests(), returning content from |self.files|. 264 """ 265 for test in self.files.values(): 266 test.text = test.string # mimic parsing. 267 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 268 SuiteBase.find_and_parse_tests( 269 mox.IgnoreArg(), 270 mox.IgnoreArg(), 271 mox.IgnoreArg(), 272 forgiving_parser=True, 273 run_prod_code=False, 274 test_args=None).AndReturn(self.files.values()) 275 276 277 def expect_job_scheduling(self, recorder, 278 tests_to_skip=[], ignore_deps=False, 279 raises=False, suite_deps=[], suite=None, 280 extra_keyvals={}): 281 """Expect jobs to be scheduled for 'tests' in |self.files|. 282 283 @param recorder: object with a record_entry to be used to record test 284 results. 285 @param tests_to_skip: [list, of, test, names] that we expect to skip. 286 @param ignore_deps: If true, ignore tests' dependencies. 287 @param raises: If True, expect exceptions. 288 @param suite_deps: If True, add suite level dependencies. 289 @param extra_keyvals: Extra keyvals set to tests. 290 """ 291 record_job_id = suite and suite._results_dir 292 if record_job_id: 293 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 294 recorder.record_entry( 295 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 296 log_in_subdir=False) 297 tests = self.files.values() 298 n = 1 299 for test in tests: 300 if test.name in tests_to_skip: 301 continue 302 dependencies = [] 303 if not ignore_deps: 304 dependencies.extend(test.dependencies) 305 if suite_deps: 306 dependencies.extend(suite_deps) 307 dependencies.append(self._BOARD) 308 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 309 keyvals = { 310 'build': build, 311 'suite': self._TAG, 312 'builds': SuiteTest._BUILDS, 313 'experimental':test.experimental, 314 } 315 keyvals.update(extra_keyvals) 316 job_mock = self.afe.create_job( 317 control_file=test.text, 318 name=mox.And(mox.StrContains(build), 319 mox.StrContains(test.name)), 320 control_type=mox.IgnoreArg(), 321 meta_hosts=[self._BOARD], 322 dependencies=dependencies, 323 keyvals=keyvals, 324 max_runtime_mins=24*60, 325 timeout_mins=1440, 326 parent_job_id=None, 327 test_retry=0, 328 reboot_before=mox.IgnoreArg(), 329 run_reset=mox.IgnoreArg(), 330 priority=priorities.Priority.DEFAULT, 331 synch_count=test.sync_count, 332 require_ssp=test.require_ssp 333 ) 334 if raises: 335 job_mock.AndRaise(error.NoEligibleHostException()) 336 recorder.record_entry( 337 StatusContains.CreateFromStrings('START', test.name), 338 log_in_subdir=False) 339 recorder.record_entry( 340 StatusContains.CreateFromStrings('TEST_NA', test.name), 341 log_in_subdir=False) 342 recorder.record_entry( 343 StatusContains.CreateFromStrings('END', test.name), 344 log_in_subdir=False) 345 else: 346 fake_job = FakeJob(id=n) 347 job_mock.AndReturn(fake_job) 348 if record_job_id: 349 suite._remember_job_keyval(fake_job) 350 n += 1 351 352 353 def testScheduleTestsAndRecord(self): 354 """Should schedule stable and experimental tests with the AFE.""" 355 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 356 'name-data_four', 'name-data_five', 'name-data_six', 357 'name-data_seven'] 358 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 359 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 360 361 self.mock_control_file_parsing() 362 self.mox.ReplayAll() 363 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 364 self.devserver, 365 afe=self.afe, tko=self.tko, 366 results_dir=self.tmpdir) 367 self.mox.ResetAll() 368 recorder = self.mox.CreateMock(base_job.base_job) 369 self.expect_job_scheduling(recorder, suite=suite) 370 371 self.mox.StubOutWithMock(utils, 'write_keyval') 372 utils.write_keyval(self.tmpdir, keyval_dict) 373 self.mox.ReplayAll() 374 suite.schedule(recorder.record_entry) 375 for job in suite._jobs: 376 self.assertTrue(hasattr(job, 'test_name')) 377 378 379 def testScheduleTests(self): 380 """Should schedule tests with the AFE.""" 381 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 382 'name-data_four', 'name-data_five', 'name-data_six', 383 'name-data_seven'] 384 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 385 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 386 387 self.mock_control_file_parsing() 388 recorder = self.mox.CreateMock(base_job.base_job) 389 self.expect_job_scheduling(recorder) 390 self.mox.StubOutWithMock(utils, 'write_keyval') 391 utils.write_keyval(None, keyval_dict) 392 393 self.mox.ReplayAll() 394 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 395 self.devserver, 396 afe=self.afe, tko=self.tko) 397 suite.schedule(recorder.record_entry) 398 399 400 def testScheduleTestsIgnoreDeps(self): 401 """Test scheduling tests ignoring deps.""" 402 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 403 'name-data_four', 'name-data_five', 'name-data_six', 404 'name-data_seven'] 405 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 406 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 407 408 self.mock_control_file_parsing() 409 recorder = self.mox.CreateMock(base_job.base_job) 410 self.expect_job_scheduling(recorder, ignore_deps=True) 411 self.mox.StubOutWithMock(utils, 'write_keyval') 412 utils.write_keyval(None, keyval_dict) 413 414 self.mox.ReplayAll() 415 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 416 self.devserver, 417 afe=self.afe, tko=self.tko, 418 ignore_deps=True) 419 suite.schedule(recorder.record_entry) 420 421 422 def testScheduleUnrunnableTestsTESTNA(self): 423 """Tests which fail to schedule should be TEST_NA.""" 424 # Since all tests will be fail to schedule, the num of scheduled tests 425 # will be zero. 426 name_list = [] 427 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 428 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 429 430 self.mock_control_file_parsing() 431 recorder = self.mox.CreateMock(base_job.base_job) 432 self.expect_job_scheduling(recorder, raises=True) 433 self.mox.StubOutWithMock(utils, 'write_keyval') 434 utils.write_keyval(None, keyval_dict) 435 self.mox.ReplayAll() 436 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 437 self.devserver, 438 afe=self.afe, tko=self.tko) 439 suite.schedule(recorder.record_entry) 440 441 442 def testRetryMapAfterScheduling(self): 443 """Test job-test and test-job mapping are correctly updated.""" 444 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 445 'name-data_four', 'name-data_five', 'name-data_six', 446 'name-data_seven'] 447 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 448 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 449 450 self.mock_control_file_parsing() 451 recorder = self.mox.CreateMock(base_job.base_job) 452 self.expect_job_scheduling(recorder) 453 self.mox.StubOutWithMock(utils, 'write_keyval') 454 utils.write_keyval(None, keyval_dict) 455 456 all_files = self.files.items() 457 # Sort tests in self.files so that they are in the same 458 # order as they are scheduled. 459 expected_retry_map = {} 460 for n in range(len(all_files)): 461 test = all_files[n][1] 462 job_id = n + 1 463 job_retries = 1 if test.job_retries is None else test.job_retries 464 if job_retries > 0: 465 expected_retry_map[job_id] = { 466 'state': RetryHandler.States.NOT_ATTEMPTED, 467 'retry_max': job_retries} 468 469 self.mox.ReplayAll() 470 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 471 self.devserver, 472 afe=self.afe, tko=self.tko, 473 job_retry=True) 474 suite.schedule(recorder.record_entry) 475 476 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 477 478 479 def testSuiteMaxRetries(self): 480 """Test suite max retries.""" 481 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 482 'name-data_four', 'name-data_five', 483 'name-data_six', 'name-data_seven'] 484 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 485 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 486 487 self.mock_control_file_parsing() 488 recorder = self.mox.CreateMock(base_job.base_job) 489 self.expect_job_scheduling(recorder) 490 self.mox.StubOutWithMock(utils, 'write_keyval') 491 utils.write_keyval(None, keyval_dict) 492 self.mox.ReplayAll() 493 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 494 self.devserver, 495 afe=self.afe, tko=self.tko, 496 job_retry=True, max_retries=1) 497 suite.schedule(recorder.record_entry) 498 self.assertEqual(suite._retry_handler._max_retries, 1) 499 # Find the job_id of the test that allows retry 500 job_id = suite._retry_handler._retry_map.iterkeys().next() 501 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 502 self.assertEqual(suite._retry_handler._max_retries, 0) 503 504 505 def testSuiteDependencies(self): 506 """Should add suite dependencies to tests scheduled.""" 507 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 508 'name-data_four', 'name-data_five', 'name-data_six', 509 'name-data_seven'] 510 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 511 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 512 513 self.mock_control_file_parsing() 514 recorder = self.mox.CreateMock(base_job.base_job) 515 self.expect_job_scheduling(recorder, suite_deps=['extra']) 516 self.mox.StubOutWithMock(utils, 'write_keyval') 517 utils.write_keyval(None, keyval_dict) 518 519 self.mox.ReplayAll() 520 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 521 self.devserver, extra_deps=['extra'], 522 afe=self.afe, tko=self.tko) 523 suite.schedule(recorder.record_entry) 524 525 526 def testInheritedKeyvals(self): 527 """Tests should inherit some whitelisted job keyvals.""" 528 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 529 job_keyvals = { 530 constants.KEYVAL_CIDB_BUILD_ID: '111', 531 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 532 'your': 'name', 533 } 534 test_keyvals = { 535 constants.KEYVAL_CIDB_BUILD_ID: '111', 536 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 537 } 538 539 self.mock_control_file_parsing() 540 recorder = self.mox.CreateMock(base_job.base_job) 541 self.expect_job_scheduling( 542 recorder, 543 extra_keyvals=test_keyvals) 544 self.mox.StubOutWithMock(utils, 'write_keyval') 545 utils.write_keyval(None, job_keyvals) 546 utils.write_keyval(None, mox.IgnoreArg()) 547 548 self.mox.ReplayAll() 549 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 550 self.devserver, 551 afe=self.afe, tko=self.tko, 552 job_keyvals=job_keyvals) 553 suite.schedule(recorder.record_entry) 554 555 556 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 557 """Create a Suite, using mocked tests and control file contents. 558 559 @return Suite object, after mocking out behavior needed to create it. 560 """ 561 self.result_reporter = _MemoryResultReporter() 562 self.expect_control_file_parsing() 563 self.mox.ReplayAll() 564 suite = Suite.create_from_name( 565 self._TAG, 566 self._BUILDS, 567 self._BOARD, 568 self.devserver, 569 self.getter, 570 afe=self.afe, 571 tko=self.tko, 572 file_bugs=file_bugs, 573 job_retry=True, 574 result_reporter=self.result_reporter, 575 ) 576 self.mox.ResetAll() 577 return suite 578 579 580 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 581 """Create a suite, returned a set of mocked results to expect. 582 583 @param results_dir: A mock results directory. 584 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 585 586 @return List of mocked results to wait on. 587 """ 588 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 589 file_bugs=True) 590 self.suite._results_dir = results_dir 591 test_report = self._get_bad_test_report(result_status) 592 test_predicates = test_report.predicates 593 test_fallout = test_report.fallout 594 595 self.recorder = self.mox.CreateMock(base_job.base_job) 596 self.recorder.record_entry = self.mox.CreateMock( 597 base_job.base_job.record_entry) 598 self._mock_recorder_with_results([test_predicates], self.recorder) 599 return [test_predicates, test_fallout] 600 601 602 def _mock_recorder_with_results(self, results, recorder): 603 """ 604 Checks that results are recoded in order, eg: 605 START, (status, name, reason) END 606 607 @param results: list of results 608 @param recorder: status recorder 609 """ 610 for result in results: 611 status = result[0] 612 test_name = result[1] 613 recorder.record_entry( 614 StatusContains.CreateFromStrings('START', test_name), 615 log_in_subdir=False) 616 recorder.record_entry( 617 StatusContains.CreateFromStrings(*result), 618 log_in_subdir=False).InAnyOrder('results') 619 recorder.record_entry( 620 StatusContains.CreateFromStrings('END %s' % status, test_name), 621 log_in_subdir=False) 622 623 624 def schedule_and_expect_these_results(self, suite, results, recorder): 625 """Create mox stubs for call to suite.schedule and 626 job_status.wait_for_results 627 628 @param suite: suite object for which to stub out schedule(...) 629 @param results: results object to be returned from 630 job_stats_wait_for_results(...) 631 @param recorder: mocked recorder object to replay status messages 632 """ 633 def result_generator(results): 634 """A simple generator which generates results as Status objects. 635 636 This generator handles 'send' by simply ignoring it. 637 638 @param results: results object to be returned from 639 job_stats_wait_for_results(...) 640 @yield: job_status.Status objects. 641 """ 642 results = map(lambda r: job_status.Status(*r), results) 643 for r in results: 644 new_input = (yield r) 645 if new_input: 646 yield None 647 648 self.mox.StubOutWithMock(suite, 'schedule') 649 suite.schedule(recorder.record_entry) 650 suite._retry_handler = RetryHandler({}) 651 652 waiter_patch = mock.patch.object( 653 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 654 waiter_mock = waiter_patch.start() 655 waiter_mock.return_value = result_generator(results) 656 self.addCleanup(waiter_patch.stop) 657 658 659 def testRunAndWaitSuccess(self): 660 """Should record successful results.""" 661 suite = self._createSuiteWithMockedTestsAndControlFiles() 662 663 recorder = self.mox.CreateMock(base_job.base_job) 664 665 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 666 self._mock_recorder_with_results(results, recorder) 667 self.schedule_and_expect_these_results(suite, results, recorder) 668 self.mox.ReplayAll() 669 670 suite.schedule(recorder.record_entry) 671 suite.wait(recorder.record_entry) 672 673 674 def testRunAndWaitFailure(self): 675 """Should record failure to gather results.""" 676 suite = self._createSuiteWithMockedTestsAndControlFiles() 677 678 recorder = self.mox.CreateMock(base_job.base_job) 679 recorder.record_entry( 680 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 681 log_in_subdir=False) 682 683 self.mox.StubOutWithMock(suite, 'schedule') 684 suite.schedule(recorder.record_entry) 685 self.mox.ReplayAll() 686 687 with mock.patch.object( 688 job_status.JobResultWaiter, 'wait_for_results', 689 autospec=True) as wait_mock: 690 wait_mock.side_effect = Exception 691 suite.schedule(recorder.record_entry) 692 suite.wait(recorder.record_entry) 693 694 695 def testRunAndWaitScheduleFailure(self): 696 """Should record failure to schedule jobs.""" 697 suite = self._createSuiteWithMockedTestsAndControlFiles() 698 699 recorder = self.mox.CreateMock(base_job.base_job) 700 recorder.record_entry( 701 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 702 log_in_subdir=False) 703 704 recorder.record_entry( 705 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 706 log_in_subdir=False) 707 708 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 709 suite._job_creator.create_job( 710 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 711 Exception('Expected during test.')) 712 self.mox.ReplayAll() 713 714 suite.schedule(recorder.record_entry) 715 suite.wait(recorder.record_entry) 716 717 718 def testGetTestsSortedByTime(self): 719 """Should find all tests and sorted by TIME setting.""" 720 self.expect_control_file_parsing() 721 self.mox.ReplayAll() 722 # Get all tests. 723 tests = SuiteBase.find_and_parse_tests(self.getter, 724 lambda d: True, 725 self._TAG) 726 self.assertEquals(len(tests), 7) 727 times = [control_data.ControlData.get_test_time_index(test.time) 728 for test in tests] 729 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 730 'Tests are not ordered correctly.') 731 732 733 def _get_bad_test_report(self, result_status='FAIL'): 734 """ 735 Fetch the predicates of a failing test, and the parameters 736 that are a fallout of this test failing. 737 """ 738 predicates = collections.namedtuple('predicates', 739 'status, testname, reason') 740 fallout = collections.namedtuple('fallout', 741 ('time_start, time_end, job_id,' 742 'username, hostname')) 743 test_report = collections.namedtuple('test_report', 744 'predicates, fallout') 745 return test_report(predicates(result_status, 'bad_test', 746 'dreadful_reason'), 747 fallout('2014-01-01 01:01:01', 'None', 748 self._FAKE_JOB_ID, 'user', 'myhost')) 749 750 751 def testJobRetryTestFail(self): 752 """Test retry works.""" 753 test_to_retry = self.files['seven'] 754 fake_new_job_id = self._FAKE_JOB_ID + 1 755 fake_job = FakeJob(id=self._FAKE_JOB_ID) 756 fake_new_job = FakeJob(id=fake_new_job_id) 757 758 test_results = self._createSuiteMockResults() 759 self.schedule_and_expect_these_results( 760 self.suite, 761 [test_results[0] + test_results[1]], 762 self.recorder) 763 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 764 self.suite._job_creator.create_job( 765 test_to_retry, 766 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 767 self.mox.ReplayAll() 768 self.suite.schedule(self.recorder.record_entry) 769 self.suite._retry_handler._retry_map = { 770 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 771 'retry_max': 1} 772 } 773 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 774 self.suite.wait(self.recorder.record_entry) 775 expected_retry_map = { 776 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 777 'retry_max': 1}, 778 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 779 'retry_max': 0} 780 } 781 # Check retry map is correctly updated 782 self.assertEquals(self.suite._retry_handler._retry_map, 783 expected_retry_map) 784 # Check _jobs_to_tests is correctly updated 785 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 786 test_to_retry) 787 788 789 def testJobRetryTestWarn(self): 790 """Test that no retry is scheduled if test warns.""" 791 test_to_retry = self.files['seven'] 792 fake_job = FakeJob(id=self._FAKE_JOB_ID) 793 test_results = self._createSuiteMockResults(result_status='WARN') 794 self.schedule_and_expect_these_results( 795 self.suite, 796 [test_results[0] + test_results[1]], 797 self.recorder) 798 self.mox.ReplayAll() 799 self.suite.schedule(self.recorder.record_entry) 800 self.suite._retry_handler._retry_map = { 801 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 802 'retry_max': 1} 803 } 804 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 805 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 806 expected_retry_map = self.suite._retry_handler._retry_map.copy() 807 self.suite.wait(self.recorder.record_entry) 808 self.assertTrue(self.result_reporter.results) 809 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 810 self.assertEquals(self.suite._retry_handler._retry_map, 811 expected_retry_map) 812 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 813 814 815 def testFailedJobRetry(self): 816 """Make sure the suite survives even if the retry failed.""" 817 test_to_retry = self.files['seven'] 818 fake_job = FakeJob(id=self._FAKE_JOB_ID) 819 820 test_results = self._createSuiteMockResults() 821 self.schedule_and_expect_these_results( 822 self.suite, 823 [test_results[0] + test_results[1]], 824 self.recorder) 825 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 826 self.suite._job_creator.create_job( 827 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 828 error.RPCException('Expected during test')) 829 # Do not file a bug. 830 self.mox.StubOutWithMock(self.suite, '_should_report') 831 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 832 833 self.mox.ReplayAll() 834 835 self.suite.schedule(self.recorder.record_entry) 836 self.suite._retry_handler._retry_map = { 837 self._FAKE_JOB_ID: { 838 'state': RetryHandler.States.NOT_ATTEMPTED, 839 'retry_max': 1}} 840 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 841 self.suite.wait(self.recorder.record_entry) 842 expected_retry_map = { 843 self._FAKE_JOB_ID: { 844 'state': RetryHandler.States.ATTEMPTED, 845 'retry_max': 1}} 846 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 847 self.assertEquals(self.suite._retry_handler._retry_map, 848 expected_retry_map) 849 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 850 851 852 class _MemoryResultReporter(SuiteBase._ResultReporter): 853 """Reporter that stores results internally for testing.""" 854 def __init__(self): 855 self.results = [] 856 857 def report(self, result): 858 """Reports the result by storing it internally.""" 859 self.results.append(result) 860 861 862 if __name__ == '__main__': 863 unittest.main() 864
