1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10 import collections 11 import mox 12 import os 13 import shutil 14 import tempfile 15 import unittest 16 from collections import OrderedDict 17 18 import common 19 20 from autotest_lib.client.common_lib import base_job, control_data 21 from autotest_lib.client.common_lib import priorities 22 from autotest_lib.client.common_lib import utils, error 23 from autotest_lib.client.common_lib.cros import dev_server 24 from autotest_lib.server.cros import provision 25 from autotest_lib.server.cros.dynamic_suite import control_file_getter 26 from autotest_lib.server.cros.dynamic_suite import job_status 27 from autotest_lib.server.cros.dynamic_suite import reporting 28 from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 29 from autotest_lib.server.cros.dynamic_suite.suite import Suite 30 from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 31 from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 32 from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 33 from autotest_lib.server import frontend 34 from autotest_lib.site_utils import phapi_lib 35 36 37 class SuiteTest(mox.MoxTestBase): 38 """Unit tests for dynamic_suite Suite class. 39 40 @var _BUILDS: fake build 41 @var _TAG: fake suite tag 42 """ 43 44 _BOARD = 'board:board' 45 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 46 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 47 _TAG = 'au' 48 _ATTR = {'attr:attr'} 49 _DEVSERVER_HOST = 'http://dontcare:8080' 50 _FAKE_JOB_ID = 10 51 52 53 def setUp(self): 54 super(SuiteTest, self).setUp() 55 self.maxDiff = None 56 self.afe = self.mox.CreateMock(frontend.AFE) 57 self.tko = self.mox.CreateMock(frontend.TKO) 58 59 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 60 61 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 62 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 63 64 self.files = OrderedDict( 65 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 66 'FAST', expr=True)), 67 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 68 'SHORT', dependencies=['feta'])), 69 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 70 'MEDIUM')), 71 ('four', FakeControlData('other', self._ATTR, 'data_four', 72 'LONG', dependencies=['arugula'])), 73 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 74 'LONG', dependencies=['arugula', 75 'caligula'])), 76 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 77 'LENGTHY')), 78 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 79 'FAST', job_retries=1))]) 80 81 self.files_to_filter = { 82 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 83 'gets filtered'), 84 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 85 'gets filtered')} 86 87 88 def tearDown(self): 89 super(SuiteTest, self).tearDown() 90 shutil.rmtree(self.tmpdir, ignore_errors=True) 91 92 93 def expect_control_file_parsing(self, suite_name=_TAG): 94 """Expect an attempt to parse the 'control files' in |self.files|. 95 96 @param suite_name: The suite name to parse control files for. 97 """ 98 all_files = self.files.keys() + self.files_to_filter.keys() 99 self._set_control_file_parsing_expectations(False, all_files, 100 self.files, suite_name) 101 102 103 def _set_control_file_parsing_expectations(self, already_stubbed, 104 file_list, files_to_parse, 105 suite_name): 106 """Expect an attempt to parse the 'control files' in |files|. 107 108 @param already_stubbed: parse_control_string already stubbed out. 109 @param file_list: the files the dev server returns 110 @param files_to_parse: the {'name': FakeControlData} dict of files we 111 expect to get parsed. 112 """ 113 if not already_stubbed: 114 self.mox.StubOutWithMock(control_data, 'parse_control_string') 115 116 self.getter.get_control_file_list( 117 suite_name=suite_name).AndReturn(file_list) 118 for file, data in files_to_parse.iteritems(): 119 self.getter.get_control_file_contents( 120 file).InAnyOrder().AndReturn(data.string) 121 control_data.parse_control_string( 122 data.string, raise_warnings=True).InAnyOrder().AndReturn(data) 123 124 125 def testFindAndParseStableTests(self): 126 """Should find only non-experimental tests that match a predicate.""" 127 self.expect_control_file_parsing() 128 self.mox.ReplayAll() 129 130 predicate = lambda d: d.text == self.files['two'].string 131 tests = Suite.find_and_parse_tests(self.getter, predicate, self._TAG) 132 self.assertEquals(len(tests), 1) 133 self.assertEquals(tests[0], self.files['two']) 134 135 136 def testFindSuiteSyntaxErrors(self): 137 """Check all control files for syntax errors. 138 139 This test actually parses all control files in the autotest directory 140 for syntax errors, by using the un-forgiving parser and pretending to 141 look for all control files with the suite attribute. 142 """ 143 autodir = os.path.abspath( 144 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 145 fs_getter = Suite.create_fs_getter(autodir) 146 predicate = lambda t: hasattr(t, 'suite') 147 Suite.find_and_parse_tests(fs_getter, predicate, add_experimental=True, 148 forgiving_parser=False) 149 150 151 def testFindAndParseTestsSuite(self): 152 """Should find all tests that match a predicate.""" 153 self.expect_control_file_parsing() 154 self.mox.ReplayAll() 155 156 predicate = lambda d: d.suite == self._TAG 157 tests = Suite.find_and_parse_tests(self.getter, 158 predicate, 159 self._TAG, 160 add_experimental=True) 161 self.assertEquals(len(tests), 6) 162 self.assertTrue(self.files['one'] in tests) 163 self.assertTrue(self.files['two'] in tests) 164 self.assertTrue(self.files['three'] in tests) 165 self.assertTrue(self.files['five'] in tests) 166 self.assertTrue(self.files['six'] in tests) 167 self.assertTrue(self.files['seven'] in tests) 168 169 170 def testFindAndParseTestsAttr(self): 171 """Should find all tests that match a predicate.""" 172 self.expect_control_file_parsing() 173 self.mox.ReplayAll() 174 175 predicate = Suite.matches_attribute_expression_predicate('attr:attr') 176 tests = Suite.find_and_parse_tests(self.getter, 177 predicate, 178 self._TAG, 179 add_experimental=True) 180 self.assertEquals(len(tests), 6) 181 self.assertTrue(self.files['one'] in tests) 182 self.assertTrue(self.files['two'] in tests) 183 self.assertTrue(self.files['three'] in tests) 184 self.assertTrue(self.files['four'] in tests) 185 self.assertTrue(self.files['six'] in tests) 186 self.assertTrue(self.files['seven'] in tests) 187 188 189 def testAdHocSuiteCreation(self): 190 """Should be able to schedule an ad-hoc suite by specifying 191 a single test name.""" 192 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 193 self.mox.ReplayAll() 194 predicate = Suite.test_name_equals_predicate('name-data_five') 195 suite = Suite.create_from_predicates([predicate], self._BUILDS, 196 self._BOARD, devserver=None, 197 cf_getter=self.getter, 198 afe=self.afe, tko=self.tko) 199 200 self.assertFalse(self.files['one'] in suite.tests) 201 self.assertFalse(self.files['two'] in suite.tests) 202 self.assertFalse(self.files['one'] in suite.unstable_tests()) 203 self.assertFalse(self.files['two'] in suite.stable_tests()) 204 self.assertFalse(self.files['one'] in suite.stable_tests()) 205 self.assertFalse(self.files['two'] in suite.unstable_tests()) 206 self.assertFalse(self.files['four'] in suite.tests) 207 self.assertTrue(self.files['five'] in suite.tests) 208 209 210 def testStableUnstableFilter(self): 211 """Should distinguish between experimental and stable tests.""" 212 self.expect_control_file_parsing() 213 self.mox.ReplayAll() 214 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 215 devserver=None, 216 cf_getter=self.getter, 217 afe=self.afe, tko=self.tko) 218 219 self.assertTrue(self.files['one'] in suite.tests) 220 self.assertTrue(self.files['two'] in suite.tests) 221 self.assertTrue(self.files['one'] in suite.unstable_tests()) 222 self.assertTrue(self.files['two'] in suite.stable_tests()) 223 self.assertFalse(self.files['one'] in suite.stable_tests()) 224 self.assertFalse(self.files['two'] in suite.unstable_tests()) 225 # Sanity check. 226 self.assertFalse(self.files['four'] in suite.tests) 227 228 229 def mock_control_file_parsing(self): 230 """Fake out find_and_parse_tests(), returning content from |self.files|. 231 """ 232 for test in self.files.values(): 233 test.text = test.string # mimic parsing. 234 self.mox.StubOutWithMock(Suite, 'find_and_parse_tests') 235 Suite.find_and_parse_tests( 236 mox.IgnoreArg(), 237 mox.IgnoreArg(), 238 mox.IgnoreArg(), 239 add_experimental=True, 240 forgiving_parser=True, 241 run_prod_code=False).AndReturn(self.files.values()) 242 243 244 def expect_job_scheduling(self, recorder, add_experimental, 245 tests_to_skip=[], ignore_deps=False, 246 raises=False, suite_deps=[], suite=None): 247 """Expect jobs to be scheduled for 'tests' in |self.files|. 248 249 @param add_experimental: expect jobs for experimental tests as well. 250 @param recorder: object with a record_entry to be used to record test 251 results. 252 @param tests_to_skip: [list, of, test, names] that we expect to skip. 253 @param ignore_deps: If true, ignore tests' dependencies. 254 @param raises: If True, expect exceptions. 255 @param suite_deps: If True, add suite level dependencies. 256 """ 257 record_job_id = suite and suite._results_dir 258 if record_job_id: 259 self.mox.StubOutWithMock(suite, '_remember_provided_job_id') 260 recorder.record_entry( 261 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 262 log_in_subdir=False) 263 tests = self.files.values() 264 tests.sort(key=lambda test: test.experimental) 265 n = 1 266 for test in tests: 267 if not add_experimental and test.experimental: 268 continue 269 if test.name in tests_to_skip: 270 continue 271 dependencies = [] 272 if not ignore_deps: 273 dependencies.extend(test.dependencies) 274 if suite_deps: 275 dependencies.extend(suite_deps) 276 dependencies.append(self._BOARD) 277 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 278 job_mock = self.afe.create_job( 279 control_file=test.text, 280 name=mox.And(mox.StrContains(build), 281 mox.StrContains(test.name)), 282 control_type=mox.IgnoreArg(), 283 meta_hosts=[self._BOARD], 284 dependencies=dependencies, 285 keyvals={'build': build, 'suite': self._TAG, 286 'builds': SuiteTest._BUILDS, 287 'experimental':test.experimental}, 288 max_runtime_mins=24*60, 289 timeout_mins=1440, 290 parent_job_id=None, 291 test_retry=0, 292 priority=priorities.Priority.DEFAULT, 293 synch_count=test.sync_count, 294 require_ssp=test.require_ssp 295 ) 296 if raises: 297 job_mock.AndRaise(error.NoEligibleHostException()) 298 recorder.record_entry( 299 StatusContains.CreateFromStrings('START', test.name), 300 log_in_subdir=False) 301 recorder.record_entry( 302 StatusContains.CreateFromStrings('TEST_NA', test.name), 303 log_in_subdir=False) 304 recorder.record_entry( 305 StatusContains.CreateFromStrings('END', test.name), 306 log_in_subdir=False) 307 else: 308 fake_job = FakeJob(id=n) 309 job_mock.AndReturn(fake_job) 310 if record_job_id: 311 suite._remember_provided_job_id(fake_job) 312 n += 1 313 314 315 def testScheduleTestsAndRecord(self): 316 """Should schedule stable and experimental tests with the AFE.""" 317 self.mock_control_file_parsing() 318 self.mox.ReplayAll() 319 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 320 self.devserver, 321 afe=self.afe, tko=self.tko, 322 results_dir=self.tmpdir) 323 self.mox.ResetAll() 324 recorder = self.mox.CreateMock(base_job.base_job) 325 self.expect_job_scheduling(recorder, add_experimental=True, suite=suite) 326 self.mox.ReplayAll() 327 suite.schedule(recorder.record_entry, True) 328 for job in suite._jobs: 329 self.assertTrue(hasattr(job, 'test_name')) 330 331 332 def testScheduleStableTests(self): 333 """Should schedule only stable tests with the AFE.""" 334 self.mock_control_file_parsing() 335 recorder = self.mox.CreateMock(base_job.base_job) 336 self.expect_job_scheduling(recorder, add_experimental=False) 337 338 self.mox.ReplayAll() 339 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 340 self.devserver, 341 afe=self.afe, tko=self.tko) 342 suite.schedule(recorder.record_entry, add_experimental=False) 343 344 345 def testScheduleStableTestsIgnoreDeps(self): 346 """Should schedule only stable tests with the AFE.""" 347 self.mock_control_file_parsing() 348 recorder = self.mox.CreateMock(base_job.base_job) 349 self.expect_job_scheduling(recorder, add_experimental=False, 350 ignore_deps=True) 351 352 self.mox.ReplayAll() 353 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 354 self.devserver, 355 afe=self.afe, tko=self.tko, 356 ignore_deps=True) 357 suite.schedule(recorder.record_entry, add_experimental=False) 358 359 360 def testScheduleUnrunnableTestsTESTNA(self): 361 """Tests which fail to schedule should be TEST_NA.""" 362 self.mock_control_file_parsing() 363 recorder = self.mox.CreateMock(base_job.base_job) 364 self.expect_job_scheduling(recorder, add_experimental=True, raises=True) 365 self.mox.ReplayAll() 366 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 367 self.devserver, 368 afe=self.afe, tko=self.tko) 369 suite.schedule(recorder.record_entry, add_experimental=True) 370 371 372 def testRetryMapAfterScheduling(self): 373 """Test job-test and test-job mapping are correctly updated.""" 374 self.mock_control_file_parsing() 375 recorder = self.mox.CreateMock(base_job.base_job) 376 self.expect_job_scheduling(recorder, add_experimental=True) 377 self.mox.ReplayAll() 378 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 379 self.devserver, 380 afe=self.afe, tko=self.tko, 381 job_retry=True) 382 suite.schedule(recorder.record_entry, add_experimental=True) 383 all_files = self.files.items() 384 # Sort tests in self.files so that they are in the same 385 # order as they are scheduled. 386 all_files.sort(key=lambda record: record[1].experimental) 387 expected_retry_map = {} 388 for n in range(len(all_files)): 389 test = all_files[n][1] 390 job_id = n + 1 391 if test.job_retries > 0: 392 expected_retry_map[job_id] = { 393 'state': RetryHandler.States.NOT_ATTEMPTED, 394 'retry_max': test.job_retries} 395 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 396 397 398 def testSuiteMaxRetries(self): 399 self.mock_control_file_parsing() 400 recorder = self.mox.CreateMock(base_job.base_job) 401 self.expect_job_scheduling(recorder, add_experimental=True) 402 self.mox.ReplayAll() 403 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 404 self.devserver, 405 afe=self.afe, tko=self.tko, 406 job_retry=True, max_retries=1) 407 suite.schedule(recorder.record_entry, add_experimental=True) 408 self.assertEqual(suite._retry_handler._max_retries, 1) 409 # Find the job_id of the test that allows retry 410 job_id = suite._retry_handler._retry_map.iterkeys().next() 411 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 412 self.assertEqual(suite._retry_handler._max_retries, 0) 413 414 415 def testSuiteDependencies(self): 416 """Should add suite dependencies to tests scheduled.""" 417 self.mock_control_file_parsing() 418 recorder = self.mox.CreateMock(base_job.base_job) 419 self.expect_job_scheduling(recorder, add_experimental=False, 420 suite_deps=['extra']) 421 422 self.mox.ReplayAll() 423 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 424 self.devserver, extra_deps=['extra'], 425 afe=self.afe, tko=self.tko) 426 suite.schedule(recorder.record_entry, add_experimental=False) 427 428 429 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 430 """Create a Suite, using mocked tests and control file contents. 431 432 @return Suite object, after mocking out behavior needed to create it. 433 """ 434 self.expect_control_file_parsing() 435 self.mox.ReplayAll() 436 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 437 self.devserver, 438 self.getter, 439 afe=self.afe, tko=self.tko, 440 file_bugs=file_bugs, job_retry=True) 441 self.mox.ResetAll() 442 return suite 443 444 445 def _createSuiteMockResults(self, results_dir=None, will_file_bug=True, 446 result_status='FAIL'): 447 """Create a suite, returned a set of mocked results to expect. 448 449 @param results_dir: A mock results directory. 450 @param will_file_bug: Whether later a bug will be filed. 451 If true, will mock out tko method. 452 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 453 454 @return List of mocked results to wait on. 455 """ 456 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 457 file_bugs=True) 458 self.suite._results_dir = results_dir 459 test_report = self._get_bad_test_report(result_status) 460 test_predicates = test_report.predicates 461 test_fallout = test_report.fallout 462 463 self.recorder = self.mox.CreateMock(base_job.base_job) 464 self.recorder.record_entry = self.mox.CreateMock( 465 base_job.base_job.record_entry) 466 self._mock_recorder_with_results([test_predicates], self.recorder) 467 if will_file_bug: 468 self.suite._tko.run = self.mox.CreateMock(frontend.RpcClient.run) 469 self.suite._tko.run('get_detailed_test_views', 470 afe_job_id=self._FAKE_JOB_ID) 471 return [test_predicates, test_fallout] 472 473 474 def _mock_recorder_with_results(self, results, recorder): 475 """ 476 Checks that results are recoded in order, eg: 477 START, (status, name, reason) END 478 479 @param results: list of results 480 @param recorder: status recorder 481 """ 482 for result in results: 483 status = result[0] 484 test_name = result[1] 485 recorder.record_entry( 486 StatusContains.CreateFromStrings('START', test_name), 487 log_in_subdir=False) 488 recorder.record_entry( 489 StatusContains.CreateFromStrings(*result), 490 log_in_subdir=False).InAnyOrder('results') 491 recorder.record_entry( 492 StatusContains.CreateFromStrings('END %s' % status, test_name), 493 log_in_subdir=False) 494 495 496 def schedule_and_expect_these_results(self, suite, results, recorder): 497 """Create mox stubs for call to suite.schedule and 498 job_status.wait_for_results 499 500 @param suite: suite object for which to stub out schedule(...) 501 @param results: results object to be returned from 502 job_stats_wait_for_results(...) 503 @param recorder: mocked recorder object to replay status messages 504 """ 505 def result_generator(results): 506 """A simple generator which generates results as Status objects. 507 508 This generator handles 'send' by simply ignoring it. 509 510 @param results: results object to be returned from 511 job_stats_wait_for_results(...) 512 @yield: job_status.Status objects. 513 """ 514 results = map(lambda r: job_status.Status(*r), results) 515 for r in results: 516 new_input = (yield r) 517 if new_input: 518 yield None 519 520 self.mox.StubOutWithMock(suite, 'schedule') 521 suite.schedule(recorder.record_entry, True) 522 suite._retry_handler = RetryHandler({}) 523 524 self.mox.StubOutWithMock(job_status, 'wait_for_results') 525 job_status.wait_for_results( 526 self.afe, self.tko, suite._jobs).AndReturn( 527 result_generator(results)) 528 529 530 def testRunAndWaitSuccess(self): 531 """Should record successful results.""" 532 suite = self._createSuiteWithMockedTestsAndControlFiles() 533 534 recorder = self.mox.CreateMock(base_job.base_job) 535 536 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 537 self._mock_recorder_with_results(results, recorder) 538 self.schedule_and_expect_these_results(suite, results, recorder) 539 self.mox.ReplayAll() 540 541 suite.schedule(recorder.record_entry, True) 542 suite.wait(recorder.record_entry) 543 544 545 def testRunAndWaitFailure(self): 546 """Should record failure to gather results.""" 547 suite = self._createSuiteWithMockedTestsAndControlFiles() 548 549 recorder = self.mox.CreateMock(base_job.base_job) 550 recorder.record_entry( 551 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 552 log_in_subdir=False) 553 554 self.mox.StubOutWithMock(suite, 'schedule') 555 suite.schedule(recorder.record_entry, True) 556 self.mox.StubOutWithMock(job_status, 'wait_for_results') 557 job_status.wait_for_results(mox.IgnoreArg(), 558 mox.IgnoreArg(), 559 mox.IgnoreArg()).AndRaise( 560 Exception('Expected during test.')) 561 self.mox.ReplayAll() 562 563 suite.schedule(recorder.record_entry, True) 564 suite.wait(recorder.record_entry) 565 566 567 def testRunAndWaitScheduleFailure(self): 568 """Should record failure to schedule jobs.""" 569 suite = self._createSuiteWithMockedTestsAndControlFiles() 570 571 recorder = self.mox.CreateMock(base_job.base_job) 572 recorder.record_entry( 573 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 574 log_in_subdir=False) 575 576 recorder.record_entry( 577 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 578 log_in_subdir=False) 579 580 self.mox.StubOutWithMock(suite, '_create_job') 581 suite._create_job(mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 582 Exception('Expected during test.')) 583 self.mox.ReplayAll() 584 585 suite.schedule(recorder.record_entry, True) 586 suite.wait(recorder.record_entry) 587 588 589 def testGetTestsSortedByTime(self): 590 """Should find all tests and sorted by TIME setting.""" 591 self.expect_control_file_parsing() 592 self.mox.ReplayAll() 593 # Get all tests. 594 tests = Suite.find_and_parse_tests(self.getter, 595 lambda d: True, 596 self._TAG, 597 add_experimental=True) 598 self.assertEquals(len(tests), 7) 599 times = [control_data.ControlData.get_test_time_index(test.time) 600 for test in tests] 601 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 602 'Tests are not ordered correctly.') 603 604 605 def _get_bad_test_report(self, result_status='FAIL'): 606 """ 607 Fetch the predicates of a failing test, and the parameters 608 that are a fallout of this test failing. 609 """ 610 predicates = collections.namedtuple('predicates', 611 'status, testname, reason') 612 fallout = collections.namedtuple('fallout', 613 ('time_start, time_end, job_id,' 614 'username, hostname')) 615 test_report = collections.namedtuple('test_report', 616 'predicates, fallout') 617 return test_report(predicates(result_status, 'bad_test', 618 'dreadful_reason'), 619 fallout('2014-01-01 01:01:01', 'None', 620 self._FAKE_JOB_ID, 'user', 'myhost')) 621 622 623 def mock_bug_filing(self, test_results): 624 """A helper function that mocks bug filing. 625 626 @param test_results: A named tuple (predicates, fallout) representing 627 a bad test report. 628 """ 629 def check_result(result): 630 """ 631 Checks to see if the status passed to the bug reporter contains all 632 the arguments required to file bugs. 633 634 @param result: The result we get when a test fails. 635 """ 636 test_predicates = test_results[0] 637 test_fallout = test_results[1] 638 expected_result = job_status.Status( 639 test_predicates.status, test_predicates.testname, 640 reason=test_predicates.reason, 641 job_id=test_fallout.job_id, owner=test_fallout.username, 642 hostname=test_fallout.hostname, 643 begin_time_str=test_fallout.time_start) 644 645 return all(getattr(result, k, None) == v for k, v in 646 expected_result.__dict__.iteritems() 647 if 'timestamp' not in str(k)) 648 649 self.mox.StubOutWithMock(reporting, 'TestBug') 650 reporting.TestBug(self._BUILDS[provision.CROS_VERSION_PREFIX], 651 mox.IgnoreArg(), mox.IgnoreArg(), 652 mox.Func(check_result)) 653 654 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 655 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 656 mox.IgnoreArg()) 657 self.mox.StubOutWithMock(reporting.Reporter, 'report') 658 reporting.Reporter.report(mox.IgnoreArg(), 659 mox.IgnoreArg()).AndReturn((0, 0)) 660 661 self.mox.StubOutWithMock(utils, 'write_keyval') 662 utils.write_keyval(mox.IgnoreArg(), mox.IgnoreArg()) 663 664 665 def testBugFiling(self): 666 """ 667 Confirm that all the necessary predicates are passed on to the 668 bug reporter when a test fails. 669 """ 670 test_results = self._createSuiteMockResults() 671 self.schedule_and_expect_these_results( 672 self.suite, 673 [test_results[0] + test_results[1]], 674 self.recorder) 675 676 self.mock_bug_filing(test_results) 677 self.mox.ReplayAll() 678 679 self.suite.schedule(self.recorder.record_entry, True) 680 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 681 self.suite.wait(self.recorder.record_entry) 682 683 684 def testFailedBugFiling(self): 685 """ 686 Make sure the suite survives even if we cannot file bugs. 687 """ 688 test_results = self._createSuiteMockResults(self.tmpdir) 689 self.schedule_and_expect_these_results( 690 self.suite, 691 [test_results[0] + test_results[1]], 692 self.recorder) 693 self.mox.StubOutWithMock(reporting.Reporter, '_check_tracker') 694 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 695 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 696 mox.IgnoreArg()) 697 reporting.Reporter._check_tracker().AndReturn(False) 698 699 self.mox.ReplayAll() 700 701 self.suite.schedule(self.recorder.record_entry, True) 702 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 703 self.suite.wait(self.recorder.record_entry) 704 705 706 def testJobRetryTestFail(self): 707 """Test retry works.""" 708 test_to_retry = self.files['seven'] 709 fake_new_job_id = self._FAKE_JOB_ID + 1 710 fake_job = FakeJob(id=self._FAKE_JOB_ID) 711 fake_new_job = FakeJob(id=fake_new_job_id) 712 713 test_results = self._createSuiteMockResults(will_file_bug=False) 714 self.schedule_and_expect_these_results( 715 self.suite, 716 [test_results[0] + test_results[1]], 717 self.recorder) 718 self.mox.StubOutWithMock(self.suite, '_create_job') 719 self.suite._create_job( 720 test_to_retry, 721 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 722 self.mox.ReplayAll() 723 self.suite.schedule(self.recorder.record_entry, True) 724 self.suite._retry_handler._retry_map = { 725 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 726 'retry_max': 1} 727 } 728 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 729 self.suite.wait(self.recorder.record_entry) 730 expected_retry_map = { 731 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 732 'retry_max': 1}, 733 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 734 'retry_max': 0} 735 } 736 # Check retry map is correctly updated 737 self.assertEquals(self.suite._retry_handler._retry_map, 738 expected_retry_map) 739 # Check _jobs_to_tests is correctly updated 740 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 741 test_to_retry) 742 743 744 def testJobRetryTestWarn(self): 745 """Test that no retry is scheduled if test warns.""" 746 test_to_retry = self.files['seven'] 747 fake_job = FakeJob(id=self._FAKE_JOB_ID) 748 test_results = self._createSuiteMockResults( 749 will_file_bug=True, result_status='WARN') 750 self.schedule_and_expect_these_results( 751 self.suite, 752 [test_results[0] + test_results[1]], 753 self.recorder) 754 # A bug should be filed if test warns. 755 self.mock_bug_filing(test_results) 756 self.mox.ReplayAll() 757 self.suite.schedule(self.recorder.record_entry, True) 758 self.suite._retry_handler._retry_map = { 759 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 760 'retry_max': 1} 761 } 762 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 763 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 764 expected_retry_map = self.suite._retry_handler._retry_map.copy() 765 self.suite.wait(self.recorder.record_entry) 766 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 767 self.assertEquals(self.suite._retry_handler._retry_map, 768 expected_retry_map) 769 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 770 771 772 def testFailedJobRetry(self): 773 """Make sure the suite survives even if the retry failed.""" 774 test_to_retry = self.files['seven'] 775 fake_job = FakeJob(id=self._FAKE_JOB_ID) 776 777 test_results = self._createSuiteMockResults(will_file_bug=False) 778 self.schedule_and_expect_these_results( 779 self.suite, 780 [test_results[0] + test_results[1]], 781 self.recorder) 782 self.mox.StubOutWithMock(self.suite, '_create_job') 783 self.suite._create_job( 784 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 785 error.RPCException('Expected during test')) 786 # Do not file a bug. 787 self.mox.StubOutWithMock(self.suite, 'should_report') 788 self.suite.should_report(mox.IgnoreArg()).AndReturn(False) 789 790 self.mox.ReplayAll() 791 792 self.suite.schedule(self.recorder.record_entry, True) 793 self.suite._retry_handler._retry_map = { 794 self._FAKE_JOB_ID: { 795 'state': RetryHandler.States.NOT_ATTEMPTED, 796 'retry_max': 1}} 797 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 798 self.suite.wait(self.recorder.record_entry) 799 expected_retry_map = { 800 self._FAKE_JOB_ID: { 801 'state': RetryHandler.States.ATTEMPTED, 802 'retry_max': 1}} 803 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 804 self.assertEquals(self.suite._retry_handler._retry_map, 805 expected_retry_map) 806 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 807 808 809 if __name__ == '__main__': 810 unittest.main() 811
