1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10 import collections 11 from collections import OrderedDict 12 import os 13 import shutil 14 import tempfile 15 import unittest 16 17 import mock 18 import mox 19 20 import common 21 22 from autotest_lib.client.common_lib import base_job 23 from autotest_lib.client.common_lib import control_data 24 from autotest_lib.client.common_lib import error 25 from autotest_lib.client.common_lib import priorities 26 from autotest_lib.client.common_lib import utils 27 from autotest_lib.client.common_lib.cros import dev_server 28 from autotest_lib.server import frontend 29 from autotest_lib.server.cros import provision 30 from autotest_lib.server.cros.dynamic_suite import control_file_getter 31 from autotest_lib.server.cros.dynamic_suite import constants 32 from autotest_lib.server.cros.dynamic_suite import job_status 33 from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34 from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 35 from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 36 from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 37 from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 38 from autotest_lib.server.cros.dynamic_suite.suite import Suite 39 40 41 class SuiteTest(mox.MoxTestBase): 42 """Unit tests for dynamic_suite Suite class. 43 44 @var _BUILDS: fake build 45 @var _TAG: fake suite tag 46 """ 47 48 _BOARD = 'board:board' 49 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 50 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 51 _TAG = 'au' 52 _ATTR = {'attr:attr'} 53 _DEVSERVER_HOST = 'http://dontcare:8080' 54 _FAKE_JOB_ID = 10 55 56 57 def setUp(self): 58 """Setup.""" 59 super(SuiteTest, self).setUp() 60 self.maxDiff = None 61 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 62 SuiteBase.ENABLE_CONTROLS_IN_BATCH = False 63 self.afe = self.mox.CreateMock(frontend.AFE) 64 self.tko = self.mox.CreateMock(frontend.TKO) 65 66 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 67 68 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 69 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 70 71 self.files = OrderedDict( 72 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 73 'FAST')), 74 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 75 'SHORT', dependencies=['feta'])), 76 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 77 'MEDIUM')), 78 ('four', FakeControlData('other', self._ATTR, 'data_four', 79 'LONG', dependencies=['arugula'])), 80 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 81 'LONG', dependencies=['arugula', 82 'caligula'])), 83 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 84 'LENGTHY')), 85 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 86 'FAST', job_retries=1))]) 87 88 self.files_to_filter = { 89 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 90 'gets filtered'), 91 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 92 'gets filtered')} 93 94 95 def tearDown(self): 96 """Teardown.""" 97 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 98 super(SuiteTest, self).tearDown() 99 shutil.rmtree(self.tmpdir, ignore_errors=True) 100 101 102 def expect_control_file_parsing(self, suite_name=_TAG): 103 """Expect an attempt to parse the 'control files' in |self.files|. 104 105 @param suite_name: The suite name to parse control files for. 106 """ 107 all_files = self.files.keys() + self.files_to_filter.keys() 108 self._set_control_file_parsing_expectations(False, all_files, 109 self.files, suite_name) 110 111 112 def _set_control_file_parsing_expectations(self, already_stubbed, 113 file_list, files_to_parse, 114 suite_name): 115 """Expect an attempt to parse the 'control files' in |files|. 116 117 @param already_stubbed: parse_control_string already stubbed out. 118 @param file_list: the files the dev server returns 119 @param files_to_parse: the {'name': FakeControlData} dict of files we 120 expect to get parsed. 121 """ 122 if not already_stubbed: 123 self.mox.StubOutWithMock(control_data, 'parse_control_string') 124 125 self.getter.get_control_file_list( 126 suite_name=suite_name).AndReturn(file_list) 127 for file, data in files_to_parse.iteritems(): 128 self.getter.get_control_file_contents( 129 file).InAnyOrder().AndReturn(data.string) 130 control_data.parse_control_string( 131 data.string, 132 raise_warnings=True, 133 path=file).InAnyOrder().AndReturn(data) 134 135 136 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 137 """Expect an attempt to parse the contents of all control files in 138 |self.files| and |self.files_to_filter|, form them to a dict. 139 140 @param suite_name: The suite name to parse control files for. 141 """ 142 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 143 self.mox.StubOutWithMock(control_data, 'parse_control_string') 144 suite_info = {} 145 for k, v in self.files.iteritems(): 146 suite_info[k] = v.string 147 control_data.parse_control_string( 148 v.string, 149 raise_warnings=True, 150 path=k).InAnyOrder().AndReturn(v) 151 for k, v in self.files_to_filter.iteritems(): 152 suite_info[k] = v.string 153 self.getter._dev_server = self._DEVSERVER_HOST 154 self.getter.get_suite_info( 155 suite_name=suite_name).AndReturn(suite_info) 156 157 158 def testFindAllTestInBatch(self): 159 """Test switch on enable_getting_controls_in_batch for function 160 find_all_test.""" 161 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 162 self.expect_control_file_parsing_in_batch() 163 SuiteBase.ENABLE_CONTROLS_IN_BATCH = True 164 165 self.mox.ReplayAll() 166 167 predicate = lambda d: d.suite == self._TAG 168 tests = SuiteBase.find_and_parse_tests(self.getter, 169 predicate, 170 self._TAG) 171 self.assertEquals(len(tests), 6) 172 self.assertTrue(self.files['one'] in tests) 173 self.assertTrue(self.files['two'] in tests) 174 self.assertTrue(self.files['three'] in tests) 175 self.assertTrue(self.files['five'] in tests) 176 self.assertTrue(self.files['six'] in tests) 177 self.assertTrue(self.files['seven'] in tests) 178 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 179 180 181 def testFindAndParseStableTests(self): 182 """Should find only tests that match a predicate.""" 183 self.expect_control_file_parsing() 184 self.mox.ReplayAll() 185 186 predicate = lambda d: d.text == self.files['two'].string 187 tests = SuiteBase.find_and_parse_tests(self.getter, 188 predicate, 189 self._TAG) 190 self.assertEquals(len(tests), 1) 191 self.assertEquals(tests[0], self.files['two']) 192 193 194 def testFindSuiteSyntaxErrors(self): 195 """Check all control files for syntax errors. 196 197 This test actually parses all control files in the autotest directory 198 for syntax errors, by using the un-forgiving parser and pretending to 199 look for all control files with the suite attribute. 200 """ 201 autodir = os.path.abspath( 202 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 203 fs_getter = SuiteBase.create_fs_getter(autodir) 204 predicate = lambda t: hasattr(t, 'suite') 205 SuiteBase.find_and_parse_tests(fs_getter, predicate, 206 forgiving_parser=False) 207 208 209 def testFindAndParseTestsSuite(self): 210 """Should find all tests that match a predicate.""" 211 self.expect_control_file_parsing() 212 self.mox.ReplayAll() 213 214 predicate = lambda d: d.suite == self._TAG 215 tests = SuiteBase.find_and_parse_tests(self.getter, 216 predicate, 217 self._TAG) 218 self.assertEquals(len(tests), 6) 219 self.assertTrue(self.files['one'] in tests) 220 self.assertTrue(self.files['two'] in tests) 221 self.assertTrue(self.files['three'] in tests) 222 self.assertTrue(self.files['five'] in tests) 223 self.assertTrue(self.files['six'] in tests) 224 self.assertTrue(self.files['seven'] in tests) 225 226 227 def testFindAndParseTestsAttr(self): 228 """Should find all tests that match a predicate.""" 229 self.expect_control_file_parsing() 230 self.mox.ReplayAll() 231 232 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 233 tests = SuiteBase.find_and_parse_tests(self.getter, 234 predicate, 235 self._TAG) 236 self.assertEquals(len(tests), 6) 237 self.assertTrue(self.files['one'] in tests) 238 self.assertTrue(self.files['two'] in tests) 239 self.assertTrue(self.files['three'] in tests) 240 self.assertTrue(self.files['four'] in tests) 241 self.assertTrue(self.files['six'] in tests) 242 self.assertTrue(self.files['seven'] in tests) 243 244 245 def testAdHocSuiteCreation(self): 246 """Should be able to schedule an ad-hoc suite by specifying 247 a single test name.""" 248 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 249 self.mox.ReplayAll() 250 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 251 suite = Suite.create_from_predicates([predicate], self._BUILDS, 252 self._BOARD, devserver=None, 253 cf_getter=self.getter, 254 afe=self.afe, tko=self.tko) 255 256 self.assertFalse(self.files['one'] in suite.tests) 257 self.assertFalse(self.files['two'] in suite.tests) 258 self.assertFalse(self.files['four'] in suite.tests) 259 self.assertTrue(self.files['five'] in suite.tests) 260 261 262 def mock_control_file_parsing(self): 263 """Fake out find_and_parse_tests(), returning content from |self.files|. 264 """ 265 for test in self.files.values(): 266 test.text = test.string # mimic parsing. 267 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 268 SuiteBase.find_and_parse_tests( 269 mox.IgnoreArg(), 270 mox.IgnoreArg(), 271 mox.IgnoreArg(), 272 forgiving_parser=True, 273 run_prod_code=False, 274 test_args=None).AndReturn(self.files.values()) 275 276 277 def expect_job_scheduling(self, recorder, 278 tests_to_skip=[], ignore_deps=False, 279 raises=False, suite_deps=[], suite=None, 280 extra_keyvals={}): 281 """Expect jobs to be scheduled for 'tests' in |self.files|. 282 283 @param recorder: object with a record_entry to be used to record test 284 results. 285 @param tests_to_skip: [list, of, test, names] that we expect to skip. 286 @param ignore_deps: If true, ignore tests' dependencies. 287 @param raises: If True, expect exceptions. 288 @param suite_deps: If True, add suite level dependencies. 289 @param extra_keyvals: Extra keyvals set to tests. 290 """ 291 record_job_id = suite and suite._results_dir 292 if record_job_id: 293 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 294 recorder.record_entry( 295 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 296 log_in_subdir=False) 297 tests = self.files.values() 298 n = 1 299 for test in tests: 300 if test.name in tests_to_skip: 301 continue 302 dependencies = [] 303 if not ignore_deps: 304 dependencies.extend(test.dependencies) 305 if suite_deps: 306 dependencies.extend(suite_deps) 307 dependencies.append(self._BOARD) 308 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 309 keyvals = { 310 'build': build, 311 'suite': self._TAG, 312 'builds': SuiteTest._BUILDS, 313 'experimental':test.experimental, 314 } 315 keyvals.update(extra_keyvals) 316 job_mock = self.afe.create_job( 317 control_file=test.text, 318 name=mox.And(mox.StrContains(build), 319 mox.StrContains(test.name)), 320 control_type=mox.IgnoreArg(), 321 meta_hosts=[self._BOARD], 322 dependencies=dependencies, 323 keyvals=keyvals, 324 max_runtime_mins=24*60, 325 timeout_mins=1440, 326 parent_job_id=None, 327 test_retry=0, 328 priority=priorities.Priority.DEFAULT, 329 synch_count=test.sync_count, 330 require_ssp=test.require_ssp 331 ) 332 if raises: 333 job_mock.AndRaise(error.NoEligibleHostException()) 334 recorder.record_entry( 335 StatusContains.CreateFromStrings('START', test.name), 336 log_in_subdir=False) 337 recorder.record_entry( 338 StatusContains.CreateFromStrings('TEST_NA', test.name), 339 log_in_subdir=False) 340 recorder.record_entry( 341 StatusContains.CreateFromStrings('END', test.name), 342 log_in_subdir=False) 343 else: 344 fake_job = FakeJob(id=n) 345 job_mock.AndReturn(fake_job) 346 if record_job_id: 347 suite._remember_job_keyval(fake_job) 348 n += 1 349 350 351 def testScheduleTestsAndRecord(self): 352 """Should schedule stable and experimental tests with the AFE.""" 353 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 354 'name-data_four', 'name-data_five', 'name-data_six', 355 'name-data_seven'] 356 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 357 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 358 359 self.mock_control_file_parsing() 360 self.mox.ReplayAll() 361 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 362 self.devserver, 363 afe=self.afe, tko=self.tko, 364 results_dir=self.tmpdir) 365 self.mox.ResetAll() 366 recorder = self.mox.CreateMock(base_job.base_job) 367 self.expect_job_scheduling(recorder, suite=suite) 368 369 self.mox.StubOutWithMock(utils, 'write_keyval') 370 utils.write_keyval(self.tmpdir, keyval_dict) 371 self.mox.ReplayAll() 372 suite.schedule(recorder.record_entry) 373 for job in suite._jobs: 374 self.assertTrue(hasattr(job, 'test_name')) 375 376 377 def testScheduleTests(self): 378 """Should schedule tests with the AFE.""" 379 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 380 'name-data_four', 'name-data_five', 'name-data_six', 381 'name-data_seven'] 382 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 383 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 384 385 self.mock_control_file_parsing() 386 recorder = self.mox.CreateMock(base_job.base_job) 387 self.expect_job_scheduling(recorder) 388 self.mox.StubOutWithMock(utils, 'write_keyval') 389 utils.write_keyval(None, keyval_dict) 390 391 self.mox.ReplayAll() 392 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 393 self.devserver, 394 afe=self.afe, tko=self.tko) 395 suite.schedule(recorder.record_entry) 396 397 398 def testScheduleTestsIgnoreDeps(self): 399 """Test scheduling tests ignoring deps.""" 400 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 401 'name-data_four', 'name-data_five', 'name-data_six', 402 'name-data_seven'] 403 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 404 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 405 406 self.mock_control_file_parsing() 407 recorder = self.mox.CreateMock(base_job.base_job) 408 self.expect_job_scheduling(recorder, ignore_deps=True) 409 self.mox.StubOutWithMock(utils, 'write_keyval') 410 utils.write_keyval(None, keyval_dict) 411 412 self.mox.ReplayAll() 413 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 414 self.devserver, 415 afe=self.afe, tko=self.tko, 416 ignore_deps=True) 417 suite.schedule(recorder.record_entry) 418 419 420 def testScheduleUnrunnableTestsTESTNA(self): 421 """Tests which fail to schedule should be TEST_NA.""" 422 # Since all tests will be fail to schedule, the num of scheduled tests 423 # will be zero. 424 name_list = [] 425 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 426 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 427 428 self.mock_control_file_parsing() 429 recorder = self.mox.CreateMock(base_job.base_job) 430 self.expect_job_scheduling(recorder, raises=True) 431 self.mox.StubOutWithMock(utils, 'write_keyval') 432 utils.write_keyval(None, keyval_dict) 433 self.mox.ReplayAll() 434 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 435 self.devserver, 436 afe=self.afe, tko=self.tko) 437 suite.schedule(recorder.record_entry) 438 439 440 def testRetryMapAfterScheduling(self): 441 """Test job-test and test-job mapping are correctly updated.""" 442 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 443 'name-data_four', 'name-data_five', 'name-data_six', 444 'name-data_seven'] 445 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 446 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 447 448 self.mock_control_file_parsing() 449 recorder = self.mox.CreateMock(base_job.base_job) 450 self.expect_job_scheduling(recorder) 451 self.mox.StubOutWithMock(utils, 'write_keyval') 452 utils.write_keyval(None, keyval_dict) 453 454 all_files = self.files.items() 455 # Sort tests in self.files so that they are in the same 456 # order as they are scheduled. 457 expected_retry_map = {} 458 for n in range(len(all_files)): 459 test = all_files[n][1] 460 job_id = n + 1 461 expected_retry_map[job_id] = { 462 'state': RetryHandler.States.NOT_ATTEMPTED, 463 'retry_max': max(test.job_retries, 1)} 464 465 self.mox.ReplayAll() 466 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 467 self.devserver, 468 afe=self.afe, tko=self.tko, 469 job_retry=True) 470 suite.schedule(recorder.record_entry) 471 472 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 473 474 475 def testSuiteMaxRetries(self): 476 """Test suite max retries.""" 477 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 478 'name-data_four', 'name-data_five', 479 'name-data_six', 'name-data_seven'] 480 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 481 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 482 483 self.mock_control_file_parsing() 484 recorder = self.mox.CreateMock(base_job.base_job) 485 self.expect_job_scheduling(recorder) 486 self.mox.StubOutWithMock(utils, 'write_keyval') 487 utils.write_keyval(None, keyval_dict) 488 self.mox.ReplayAll() 489 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 490 self.devserver, 491 afe=self.afe, tko=self.tko, 492 job_retry=True, max_retries=1) 493 suite.schedule(recorder.record_entry) 494 self.assertEqual(suite._retry_handler._max_retries, 1) 495 # Find the job_id of the test that allows retry 496 job_id = suite._retry_handler._retry_map.iterkeys().next() 497 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 498 self.assertEqual(suite._retry_handler._max_retries, 0) 499 500 501 def testSuiteDependencies(self): 502 """Should add suite dependencies to tests scheduled.""" 503 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 504 'name-data_four', 'name-data_five', 'name-data_six', 505 'name-data_seven'] 506 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 507 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 508 509 self.mock_control_file_parsing() 510 recorder = self.mox.CreateMock(base_job.base_job) 511 self.expect_job_scheduling(recorder, suite_deps=['extra']) 512 self.mox.StubOutWithMock(utils, 'write_keyval') 513 utils.write_keyval(None, keyval_dict) 514 515 self.mox.ReplayAll() 516 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 517 self.devserver, extra_deps=['extra'], 518 afe=self.afe, tko=self.tko) 519 suite.schedule(recorder.record_entry) 520 521 522 def testInheritedKeyvals(self): 523 """Tests should inherit some whitelisted job keyvals.""" 524 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 525 job_keyvals = { 526 constants.KEYVAL_CIDB_BUILD_ID: '111', 527 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 528 'your': 'name', 529 } 530 test_keyvals = { 531 constants.KEYVAL_CIDB_BUILD_ID: '111', 532 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 533 } 534 535 self.mock_control_file_parsing() 536 recorder = self.mox.CreateMock(base_job.base_job) 537 self.expect_job_scheduling( 538 recorder, 539 extra_keyvals=test_keyvals) 540 self.mox.StubOutWithMock(utils, 'write_keyval') 541 utils.write_keyval(None, job_keyvals) 542 utils.write_keyval(None, mox.IgnoreArg()) 543 544 self.mox.ReplayAll() 545 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 546 self.devserver, 547 afe=self.afe, tko=self.tko, 548 job_keyvals=job_keyvals) 549 suite.schedule(recorder.record_entry) 550 551 552 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 553 """Create a Suite, using mocked tests and control file contents. 554 555 @return Suite object, after mocking out behavior needed to create it. 556 """ 557 self.expect_control_file_parsing() 558 self.mox.ReplayAll() 559 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 560 self.devserver, 561 self.getter, 562 afe=self.afe, tko=self.tko, 563 file_bugs=file_bugs, job_retry=True) 564 self.mox.ResetAll() 565 return suite 566 567 568 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 569 """Create a suite, returned a set of mocked results to expect. 570 571 @param results_dir: A mock results directory. 572 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 573 574 @return List of mocked results to wait on. 575 """ 576 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 577 file_bugs=True) 578 self.suite._results_dir = results_dir 579 test_report = self._get_bad_test_report(result_status) 580 test_predicates = test_report.predicates 581 test_fallout = test_report.fallout 582 583 self.recorder = self.mox.CreateMock(base_job.base_job) 584 self.recorder.record_entry = self.mox.CreateMock( 585 base_job.base_job.record_entry) 586 self._mock_recorder_with_results([test_predicates], self.recorder) 587 return [test_predicates, test_fallout] 588 589 590 def _mock_recorder_with_results(self, results, recorder): 591 """ 592 Checks that results are recoded in order, eg: 593 START, (status, name, reason) END 594 595 @param results: list of results 596 @param recorder: status recorder 597 """ 598 for result in results: 599 status = result[0] 600 test_name = result[1] 601 recorder.record_entry( 602 StatusContains.CreateFromStrings('START', test_name), 603 log_in_subdir=False) 604 recorder.record_entry( 605 StatusContains.CreateFromStrings(*result), 606 log_in_subdir=False).InAnyOrder('results') 607 recorder.record_entry( 608 StatusContains.CreateFromStrings('END %s' % status, test_name), 609 log_in_subdir=False) 610 611 612 def schedule_and_expect_these_results(self, suite, results, recorder): 613 """Create mox stubs for call to suite.schedule and 614 job_status.wait_for_results 615 616 @param suite: suite object for which to stub out schedule(...) 617 @param results: results object to be returned from 618 job_stats_wait_for_results(...) 619 @param recorder: mocked recorder object to replay status messages 620 """ 621 def result_generator(results): 622 """A simple generator which generates results as Status objects. 623 624 This generator handles 'send' by simply ignoring it. 625 626 @param results: results object to be returned from 627 job_stats_wait_for_results(...) 628 @yield: job_status.Status objects. 629 """ 630 results = map(lambda r: job_status.Status(*r), results) 631 for r in results: 632 new_input = (yield r) 633 if new_input: 634 yield None 635 636 self.mox.StubOutWithMock(suite, 'schedule') 637 suite.schedule(recorder.record_entry) 638 suite._retry_handler = RetryHandler({}) 639 640 self.mox.StubOutWithMock(job_status, 'wait_for_results') 641 job_status.wait_for_results( 642 self.afe, self.tko, suite._jobs).AndReturn( 643 result_generator(results)) 644 645 646 def testRunAndWaitSuccess(self): 647 """Should record successful results.""" 648 suite = self._createSuiteWithMockedTestsAndControlFiles() 649 650 recorder = self.mox.CreateMock(base_job.base_job) 651 652 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 653 self._mock_recorder_with_results(results, recorder) 654 self.schedule_and_expect_these_results(suite, results, recorder) 655 self.mox.ReplayAll() 656 657 suite.schedule(recorder.record_entry) 658 suite.wait(recorder.record_entry, dict()) 659 660 661 def testRunAndWaitFailure(self): 662 """Should record failure to gather results.""" 663 suite = self._createSuiteWithMockedTestsAndControlFiles() 664 665 recorder = self.mox.CreateMock(base_job.base_job) 666 recorder.record_entry( 667 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 668 log_in_subdir=False) 669 670 self.mox.StubOutWithMock(suite, 'schedule') 671 suite.schedule(recorder.record_entry) 672 self.mox.StubOutWithMock(job_status, 'wait_for_results') 673 job_status.wait_for_results(mox.IgnoreArg(), 674 mox.IgnoreArg(), 675 mox.IgnoreArg()).AndRaise( 676 Exception('Expected during test.')) 677 self.mox.ReplayAll() 678 679 suite.schedule(recorder.record_entry) 680 suite.wait(recorder.record_entry, dict()) 681 682 683 def testRunAndWaitScheduleFailure(self): 684 """Should record failure to schedule jobs.""" 685 suite = self._createSuiteWithMockedTestsAndControlFiles() 686 687 recorder = self.mox.CreateMock(base_job.base_job) 688 recorder.record_entry( 689 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 690 log_in_subdir=False) 691 692 recorder.record_entry( 693 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 694 log_in_subdir=False) 695 696 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 697 suite._job_creator.create_job( 698 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 699 Exception('Expected during test.')) 700 self.mox.ReplayAll() 701 702 suite.schedule(recorder.record_entry) 703 suite.wait(recorder.record_entry, dict()) 704 705 706 def testGetTestsSortedByTime(self): 707 """Should find all tests and sorted by TIME setting.""" 708 self.expect_control_file_parsing() 709 self.mox.ReplayAll() 710 # Get all tests. 711 tests = SuiteBase.find_and_parse_tests(self.getter, 712 lambda d: True, 713 self._TAG) 714 self.assertEquals(len(tests), 7) 715 times = [control_data.ControlData.get_test_time_index(test.time) 716 for test in tests] 717 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 718 'Tests are not ordered correctly.') 719 720 721 def _get_bad_test_report(self, result_status='FAIL'): 722 """ 723 Fetch the predicates of a failing test, and the parameters 724 that are a fallout of this test failing. 725 """ 726 predicates = collections.namedtuple('predicates', 727 'status, testname, reason') 728 fallout = collections.namedtuple('fallout', 729 ('time_start, time_end, job_id,' 730 'username, hostname')) 731 test_report = collections.namedtuple('test_report', 732 'predicates, fallout') 733 return test_report(predicates(result_status, 'bad_test', 734 'dreadful_reason'), 735 fallout('2014-01-01 01:01:01', 'None', 736 self._FAKE_JOB_ID, 'user', 'myhost')) 737 738 739 def testBugFiling(self): 740 """ 741 Confirm that all the necessary predicates are passed on to the 742 bug reporter when a test fails. 743 """ 744 test_results = self._createSuiteMockResults() 745 self.schedule_and_expect_these_results( 746 self.suite, 747 [test_results[0] + test_results[1]], 748 self.recorder) 749 750 self.mox.ReplayAll() 751 752 self.suite.schedule(self.recorder.record_entry) 753 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 754 reporter = SuiteBase.MemoryResultReporter() 755 self.suite.wait(self.recorder.record_entry, reporter=reporter) 756 757 758 def testFailedBugFiling(self): 759 """ 760 Make sure the suite survives even if we cannot file bugs. 761 """ 762 test_results = self._createSuiteMockResults(self.tmpdir) 763 self.schedule_and_expect_these_results( 764 self.suite, 765 [test_results[0] + test_results[1]], 766 self.recorder) 767 self.mox.ReplayAll() 768 769 self.suite.schedule(self.recorder.record_entry) 770 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 771 reporter = SuiteBase.MemoryResultReporter() 772 self.suite.wait(self.recorder.record_entry, reporter=reporter) 773 774 775 def testJobRetryTestFail(self): 776 """Test retry works.""" 777 test_to_retry = self.files['seven'] 778 fake_new_job_id = self._FAKE_JOB_ID + 1 779 fake_job = FakeJob(id=self._FAKE_JOB_ID) 780 fake_new_job = FakeJob(id=fake_new_job_id) 781 782 test_results = self._createSuiteMockResults() 783 self.schedule_and_expect_these_results( 784 self.suite, 785 [test_results[0] + test_results[1]], 786 self.recorder) 787 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 788 self.suite._job_creator.create_job( 789 test_to_retry, 790 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 791 self.mox.ReplayAll() 792 self.suite.schedule(self.recorder.record_entry) 793 self.suite._retry_handler._retry_map = { 794 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 795 'retry_max': 1} 796 } 797 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 798 reporter = mock.create_autospec(SuiteBase.MemoryResultReporter, 799 instance=True) 800 reporter.report.side_effect = Exception 801 self.suite.wait(self.recorder.record_entry, reporter=reporter) 802 expected_retry_map = { 803 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 804 'retry_max': 1}, 805 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 806 'retry_max': 0} 807 } 808 # Check retry map is correctly updated 809 self.assertEquals(self.suite._retry_handler._retry_map, 810 expected_retry_map) 811 # Check _jobs_to_tests is correctly updated 812 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 813 test_to_retry) 814 815 816 def testJobRetryTestWarn(self): 817 """Test that no retry is scheduled if test warns.""" 818 test_to_retry = self.files['seven'] 819 fake_job = FakeJob(id=self._FAKE_JOB_ID) 820 test_results = self._createSuiteMockResults(result_status='WARN') 821 self.schedule_and_expect_these_results( 822 self.suite, 823 [test_results[0] + test_results[1]], 824 self.recorder) 825 self.mox.ReplayAll() 826 self.suite.schedule(self.recorder.record_entry) 827 self.suite._retry_handler._retry_map = { 828 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 829 'retry_max': 1} 830 } 831 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 832 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 833 expected_retry_map = self.suite._retry_handler._retry_map.copy() 834 reporter = SuiteBase.MemoryResultReporter() 835 self.suite.wait(self.recorder.record_entry, reporter=reporter) 836 self.assertTrue(reporter.results) 837 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 838 self.assertEquals(self.suite._retry_handler._retry_map, 839 expected_retry_map) 840 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 841 842 843 def testFailedJobRetry(self): 844 """Make sure the suite survives even if the retry failed.""" 845 test_to_retry = self.files['seven'] 846 fake_job = FakeJob(id=self._FAKE_JOB_ID) 847 848 test_results = self._createSuiteMockResults() 849 self.schedule_and_expect_these_results( 850 self.suite, 851 [test_results[0] + test_results[1]], 852 self.recorder) 853 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 854 self.suite._job_creator.create_job( 855 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 856 error.RPCException('Expected during test')) 857 # Do not file a bug. 858 self.mox.StubOutWithMock(self.suite, '_should_report') 859 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 860 861 self.mox.ReplayAll() 862 863 self.suite.schedule(self.recorder.record_entry) 864 self.suite._retry_handler._retry_map = { 865 self._FAKE_JOB_ID: { 866 'state': RetryHandler.States.NOT_ATTEMPTED, 867 'retry_max': 1}} 868 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 869 reporter = SuiteBase.MemoryResultReporter() 870 self.suite.wait(self.recorder.record_entry, reporter=reporter) 871 expected_retry_map = { 872 self._FAKE_JOB_ID: { 873 'state': RetryHandler.States.ATTEMPTED, 874 'retry_max': 1}} 875 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 876 self.assertEquals(self.suite._retry_handler._retry_map, 877 expected_retry_map) 878 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 879 880 881 if __name__ == '__main__': 882 unittest.main() 883
