1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10 import collections 11 from collections import OrderedDict 12 import os 13 import shutil 14 import tempfile 15 import unittest 16 17 import mox 18 19 import common 20 21 from autotest_lib.client.common_lib import base_job 22 from autotest_lib.client.common_lib import control_data 23 from autotest_lib.client.common_lib import error 24 from autotest_lib.client.common_lib import priorities 25 from autotest_lib.client.common_lib import utils 26 from autotest_lib.client.common_lib.cros import dev_server 27 from autotest_lib.server import frontend 28 from autotest_lib.server.cros import provision 29 from autotest_lib.server.cros.dynamic_suite import control_file_getter 30 from autotest_lib.server.cros.dynamic_suite import constants 31 from autotest_lib.server.cros.dynamic_suite import job_status 32 from autotest_lib.server.cros.dynamic_suite import reporting 33 from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34 from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 35 from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 36 from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 37 from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 38 from autotest_lib.server.cros.dynamic_suite.suite import Suite 39 from autotest_lib.site_utils import phapi_lib 40 41 42 class SuiteTest(mox.MoxTestBase): 43 """Unit tests for dynamic_suite Suite class. 44 45 @var _BUILDS: fake build 46 @var _TAG: fake suite tag 47 """ 48 49 _BOARD = 'board:board' 50 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 51 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 52 _TAG = 'au' 53 _ATTR = {'attr:attr'} 54 _DEVSERVER_HOST = 'http://dontcare:8080' 55 _FAKE_JOB_ID = 10 56 57 58 def setUp(self): 59 super(SuiteTest, self).setUp() 60 self.maxDiff = None 61 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 62 SuiteBase.ENABLE_CONTROLS_IN_BATCH = False 63 self.afe = self.mox.CreateMock(frontend.AFE) 64 self.tko = self.mox.CreateMock(frontend.TKO) 65 66 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 67 68 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 69 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 70 71 self.files = OrderedDict( 72 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 73 'FAST', expr=True)), 74 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 75 'SHORT', dependencies=['feta'])), 76 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 77 'MEDIUM')), 78 ('four', FakeControlData('other', self._ATTR, 'data_four', 79 'LONG', dependencies=['arugula'])), 80 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 81 'LONG', dependencies=['arugula', 82 'caligula'])), 83 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 84 'LENGTHY')), 85 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 86 'FAST', job_retries=1))]) 87 88 self.files_to_filter = { 89 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 90 'gets filtered'), 91 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 92 'gets filtered')} 93 94 95 def tearDown(self): 96 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 97 super(SuiteTest, self).tearDown() 98 shutil.rmtree(self.tmpdir, ignore_errors=True) 99 100 101 def expect_control_file_parsing(self, suite_name=_TAG): 102 """Expect an attempt to parse the 'control files' in |self.files|. 103 104 @param suite_name: The suite name to parse control files for. 105 """ 106 all_files = self.files.keys() + self.files_to_filter.keys() 107 self._set_control_file_parsing_expectations(False, all_files, 108 self.files, suite_name) 109 110 111 def _set_control_file_parsing_expectations(self, already_stubbed, 112 file_list, files_to_parse, 113 suite_name): 114 """Expect an attempt to parse the 'control files' in |files|. 115 116 @param already_stubbed: parse_control_string already stubbed out. 117 @param file_list: the files the dev server returns 118 @param files_to_parse: the {'name': FakeControlData} dict of files we 119 expect to get parsed. 120 """ 121 if not already_stubbed: 122 self.mox.StubOutWithMock(control_data, 'parse_control_string') 123 124 self.getter.get_control_file_list( 125 suite_name=suite_name).AndReturn(file_list) 126 for file, data in files_to_parse.iteritems(): 127 self.getter.get_control_file_contents( 128 file).InAnyOrder().AndReturn(data.string) 129 control_data.parse_control_string( 130 data.string, 131 raise_warnings=True, 132 path=file).InAnyOrder().AndReturn(data) 133 134 135 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 136 """Expect an attempt to parse the contents of all control files in 137 |self.files| and |self.files_to_filter|, form them to a dict. 138 139 @param suite_name: The suite name to parse control files for. 140 """ 141 self.mox.StubOutWithMock(control_data, 'parse_control_string') 142 suite_info = {} 143 for k, v in self.files.iteritems(): 144 suite_info[k] = v.string 145 control_data.parse_control_string( 146 v.string, 147 raise_warnings=True, 148 path=k).InAnyOrder().AndReturn(v) 149 for k, v in self.files_to_filter.iteritems(): 150 suite_info[k] = v.string 151 self.getter._dev_server = self._DEVSERVER_HOST 152 self.getter.get_suite_info( 153 suite_name=suite_name).AndReturn(suite_info) 154 155 156 def testFindAllTestInBatch(self): 157 """Test switch on enable_getting_controls_in_batch for function 158 find_all_test.""" 159 self.use_batch = SuiteBase.ENABLE_CONTROLS_IN_BATCH 160 self.expect_control_file_parsing_in_batch() 161 SuiteBase.ENABLE_CONTROLS_IN_BATCH = True 162 163 self.mox.ReplayAll() 164 165 predicate = lambda d: d.suite == self._TAG 166 tests = Suite.find_and_parse_tests(self.getter, 167 predicate, 168 self._TAG, 169 add_experimental=True) 170 self.assertEquals(len(tests), 6) 171 self.assertTrue(self.files['one'] in tests) 172 self.assertTrue(self.files['two'] in tests) 173 self.assertTrue(self.files['three'] in tests) 174 self.assertTrue(self.files['five'] in tests) 175 self.assertTrue(self.files['six'] in tests) 176 self.assertTrue(self.files['seven'] in tests) 177 SuiteBase.ENABLE_CONTROLS_IN_BATCH = self.use_batch 178 179 180 def testFindAndParseStableTests(self): 181 """Should find only non-experimental tests that match a predicate.""" 182 self.expect_control_file_parsing() 183 self.mox.ReplayAll() 184 185 predicate = lambda d: d.text == self.files['two'].string 186 tests = Suite.find_and_parse_tests(self.getter, predicate, self._TAG) 187 self.assertEquals(len(tests), 1) 188 self.assertEquals(tests[0], self.files['two']) 189 190 191 def testFindSuiteSyntaxErrors(self): 192 """Check all control files for syntax errors. 193 194 This test actually parses all control files in the autotest directory 195 for syntax errors, by using the un-forgiving parser and pretending to 196 look for all control files with the suite attribute. 197 """ 198 autodir = os.path.abspath( 199 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 200 fs_getter = Suite.create_fs_getter(autodir) 201 predicate = lambda t: hasattr(t, 'suite') 202 Suite.find_and_parse_tests(fs_getter, predicate, add_experimental=True, 203 forgiving_parser=False) 204 205 206 def testFindAndParseTestsSuite(self): 207 """Should find all tests that match a predicate.""" 208 self.expect_control_file_parsing() 209 self.mox.ReplayAll() 210 211 predicate = lambda d: d.suite == self._TAG 212 tests = Suite.find_and_parse_tests(self.getter, 213 predicate, 214 self._TAG, 215 add_experimental=True) 216 self.assertEquals(len(tests), 6) 217 self.assertTrue(self.files['one'] in tests) 218 self.assertTrue(self.files['two'] in tests) 219 self.assertTrue(self.files['three'] in tests) 220 self.assertTrue(self.files['five'] in tests) 221 self.assertTrue(self.files['six'] in tests) 222 self.assertTrue(self.files['seven'] in tests) 223 224 225 def testFindAndParseTestsAttr(self): 226 """Should find all tests that match a predicate.""" 227 self.expect_control_file_parsing() 228 self.mox.ReplayAll() 229 230 predicate = Suite.matches_attribute_expression_predicate('attr:attr') 231 tests = Suite.find_and_parse_tests(self.getter, 232 predicate, 233 self._TAG, 234 add_experimental=True) 235 self.assertEquals(len(tests), 6) 236 self.assertTrue(self.files['one'] in tests) 237 self.assertTrue(self.files['two'] in tests) 238 self.assertTrue(self.files['three'] in tests) 239 self.assertTrue(self.files['four'] in tests) 240 self.assertTrue(self.files['six'] in tests) 241 self.assertTrue(self.files['seven'] in tests) 242 243 244 def testAdHocSuiteCreation(self): 245 """Should be able to schedule an ad-hoc suite by specifying 246 a single test name.""" 247 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 248 self.mox.ReplayAll() 249 predicate = Suite.test_name_equals_predicate('name-data_five') 250 suite = Suite.create_from_predicates([predicate], self._BUILDS, 251 self._BOARD, devserver=None, 252 cf_getter=self.getter, 253 afe=self.afe, tko=self.tko) 254 255 self.assertFalse(self.files['one'] in suite.tests) 256 self.assertFalse(self.files['two'] in suite.tests) 257 self.assertFalse(self.files['four'] in suite.tests) 258 self.assertTrue(self.files['five'] in suite.tests) 259 260 discoverer = SuiteBase._DynamicSuiteDiscoverer(suite.tests) 261 self.assertFalse(self.files['one'] in discoverer.unstable_tests) 262 self.assertFalse(self.files['two'] in discoverer.stable_tests) 263 self.assertFalse(self.files['one'] in discoverer.stable_tests) 264 self.assertFalse(self.files['two'] in discoverer.unstable_tests) 265 266 267 def testStableUnstableFilter(self): 268 """Should distinguish between experimental and stable tests.""" 269 self.expect_control_file_parsing() 270 self.mox.ReplayAll() 271 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 272 devserver=None, 273 cf_getter=self.getter, 274 afe=self.afe, tko=self.tko) 275 276 self.assertTrue(self.files['one'] in suite.tests) 277 self.assertTrue(self.files['two'] in suite.tests) 278 # Sanity check. 279 self.assertFalse(self.files['four'] in suite.tests) 280 281 discoverer = SuiteBase._DynamicSuiteDiscoverer(suite.tests) 282 self.assertTrue(self.files['one'] in discoverer.unstable_tests) 283 self.assertTrue(self.files['two'] in discoverer.stable_tests) 284 self.assertFalse(self.files['one'] in discoverer.stable_tests) 285 self.assertFalse(self.files['two'] in discoverer.unstable_tests) 286 287 288 def mock_control_file_parsing(self): 289 """Fake out find_and_parse_tests(), returning content from |self.files|. 290 """ 291 for test in self.files.values(): 292 test.text = test.string # mimic parsing. 293 self.mox.StubOutWithMock(Suite, 'find_and_parse_tests') 294 Suite.find_and_parse_tests( 295 mox.IgnoreArg(), 296 mox.IgnoreArg(), 297 mox.IgnoreArg(), 298 add_experimental=True, 299 forgiving_parser=True, 300 run_prod_code=False, 301 test_args=None).AndReturn(self.files.values()) 302 303 304 def expect_job_scheduling(self, recorder, add_experimental, 305 tests_to_skip=[], ignore_deps=False, 306 raises=False, suite_deps=[], suite=None): 307 """Expect jobs to be scheduled for 'tests' in |self.files|. 308 309 @param add_experimental: expect jobs for experimental tests as well. 310 @param recorder: object with a record_entry to be used to record test 311 results. 312 @param tests_to_skip: [list, of, test, names] that we expect to skip. 313 @param ignore_deps: If true, ignore tests' dependencies. 314 @param raises: If True, expect exceptions. 315 @param suite_deps: If True, add suite level dependencies. 316 """ 317 record_job_id = suite and suite._results_dir 318 if record_job_id: 319 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 320 recorder.record_entry( 321 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 322 log_in_subdir=False) 323 tests = self.files.values() 324 tests.sort(key=lambda test: test.experimental) 325 n = 1 326 for test in tests: 327 if not add_experimental and test.experimental: 328 continue 329 if test.name in tests_to_skip: 330 continue 331 dependencies = [] 332 if not ignore_deps: 333 dependencies.extend(test.dependencies) 334 if suite_deps: 335 dependencies.extend(suite_deps) 336 dependencies.append(self._BOARD) 337 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 338 job_mock = self.afe.create_job( 339 control_file=test.text, 340 name=mox.And(mox.StrContains(build), 341 mox.StrContains(test.name)), 342 control_type=mox.IgnoreArg(), 343 meta_hosts=[self._BOARD], 344 dependencies=dependencies, 345 keyvals={'build': build, 'suite': self._TAG, 346 'builds': SuiteTest._BUILDS, 347 'experimental':test.experimental}, 348 max_runtime_mins=24*60, 349 timeout_mins=1440, 350 parent_job_id=None, 351 test_retry=0, 352 priority=priorities.Priority.DEFAULT, 353 synch_count=test.sync_count, 354 require_ssp=test.require_ssp 355 ) 356 if raises: 357 job_mock.AndRaise(error.NoEligibleHostException()) 358 recorder.record_entry( 359 StatusContains.CreateFromStrings('START', test.name), 360 log_in_subdir=False) 361 recorder.record_entry( 362 StatusContains.CreateFromStrings('TEST_NA', test.name), 363 log_in_subdir=False) 364 recorder.record_entry( 365 StatusContains.CreateFromStrings('END', test.name), 366 log_in_subdir=False) 367 else: 368 fake_job = FakeJob(id=n) 369 job_mock.AndReturn(fake_job) 370 if record_job_id: 371 suite._remember_job_keyval(fake_job) 372 n += 1 373 374 375 def testScheduleTestsAndRecord(self): 376 """Should schedule stable and experimental tests with the AFE.""" 377 name_list = ['name-data_two', 'name-data_three', 378 'name-data_four', 'name-data_five', 'name-data_six', 379 'name-data_seven', 'experimental_name-data_one'] 380 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 381 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 382 383 self.mock_control_file_parsing() 384 self.mox.ReplayAll() 385 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 386 self.devserver, 387 afe=self.afe, tko=self.tko, 388 results_dir=self.tmpdir) 389 self.mox.ResetAll() 390 recorder = self.mox.CreateMock(base_job.base_job) 391 self.expect_job_scheduling(recorder, add_experimental=True, suite=suite) 392 393 self.mox.StubOutWithMock(utils, 'write_keyval') 394 utils.write_keyval(self.tmpdir, keyval_dict) 395 self.mox.ReplayAll() 396 suite.schedule(recorder.record_entry, True) 397 for job in suite._jobs: 398 self.assertTrue(hasattr(job, 'test_name')) 399 400 401 def testScheduleStableTests(self): 402 """Should schedule only stable tests with the AFE.""" 403 # Since test data_one is experimental, it will be skip. 404 name_list = ['name-data_two', 'name-data_three', 405 'name-data_four', 'name-data_five', 'name-data_six', 406 'name-data_seven'] 407 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 408 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 409 410 self.mock_control_file_parsing() 411 recorder = self.mox.CreateMock(base_job.base_job) 412 self.expect_job_scheduling(recorder, add_experimental=False) 413 self.mox.StubOutWithMock(utils, 'write_keyval') 414 utils.write_keyval(None, keyval_dict) 415 416 self.mox.ReplayAll() 417 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 418 self.devserver, 419 afe=self.afe, tko=self.tko) 420 suite.schedule(recorder.record_entry, add_experimental=False) 421 422 423 def testScheduleStableTestsIgnoreDeps(self): 424 """Should schedule only stable tests with the AFE.""" 425 # Since test data_one is experimental, it will be skip. 426 name_list = ['name-data_two', 'name-data_three', 427 'name-data_four', 'name-data_five', 'name-data_six', 428 'name-data_seven'] 429 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 430 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 431 432 self.mock_control_file_parsing() 433 recorder = self.mox.CreateMock(base_job.base_job) 434 self.expect_job_scheduling(recorder, add_experimental=False, 435 ignore_deps=True) 436 self.mox.StubOutWithMock(utils, 'write_keyval') 437 utils.write_keyval(None, keyval_dict) 438 439 self.mox.ReplayAll() 440 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 441 self.devserver, 442 afe=self.afe, tko=self.tko, 443 ignore_deps=True) 444 suite.schedule(recorder.record_entry, add_experimental=False) 445 446 447 def testScheduleUnrunnableTestsTESTNA(self): 448 """Tests which fail to schedule should be TEST_NA.""" 449 # Since all tests will be fail to schedule, the num of scheduled tests 450 # will be zero. 451 name_list = [] 452 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 453 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 454 455 self.mock_control_file_parsing() 456 recorder = self.mox.CreateMock(base_job.base_job) 457 self.expect_job_scheduling(recorder, add_experimental=True, raises=True) 458 self.mox.StubOutWithMock(utils, 'write_keyval') 459 utils.write_keyval(None, keyval_dict) 460 self.mox.ReplayAll() 461 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 462 self.devserver, 463 afe=self.afe, tko=self.tko) 464 suite.schedule(recorder.record_entry, add_experimental=True) 465 466 467 def testRetryMapAfterScheduling(self): 468 """Test job-test and test-job mapping are correctly updated.""" 469 name_list = ['name-data_two', 'name-data_three', 470 'name-data_four', 'name-data_five', 'name-data_six', 471 'name-data_seven', 'experimental_name-data_one'] 472 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 473 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 474 475 self.mock_control_file_parsing() 476 recorder = self.mox.CreateMock(base_job.base_job) 477 self.expect_job_scheduling(recorder, add_experimental=True) 478 self.mox.StubOutWithMock(utils, 'write_keyval') 479 utils.write_keyval(None, keyval_dict) 480 self.mox.ReplayAll() 481 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 482 self.devserver, 483 afe=self.afe, tko=self.tko, 484 job_retry=True) 485 suite.schedule(recorder.record_entry, add_experimental=True) 486 all_files = self.files.items() 487 # Sort tests in self.files so that they are in the same 488 # order as they are scheduled. 489 all_files.sort(key=lambda record: record[1].experimental) 490 expected_retry_map = {} 491 for n in range(len(all_files)): 492 test = all_files[n][1] 493 job_id = n + 1 494 if test.job_retries > 0: 495 expected_retry_map[job_id] = { 496 'state': RetryHandler.States.NOT_ATTEMPTED, 497 'retry_max': test.job_retries} 498 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 499 500 501 def testSuiteMaxRetries(self): 502 """Test suite max retries.""" 503 name_list = ['name-data_two', 'name-data_three', 504 'name-data_four', 'name-data_five', 'name-data_six', 505 'name-data_seven', 'experimental_name-data_one'] 506 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 507 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 508 509 self.mock_control_file_parsing() 510 recorder = self.mox.CreateMock(base_job.base_job) 511 self.expect_job_scheduling(recorder, add_experimental=True) 512 self.mox.StubOutWithMock(utils, 'write_keyval') 513 utils.write_keyval(None, keyval_dict) 514 self.mox.ReplayAll() 515 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 516 self.devserver, 517 afe=self.afe, tko=self.tko, 518 job_retry=True, max_retries=1) 519 suite.schedule(recorder.record_entry, add_experimental=True) 520 self.assertEqual(suite._retry_handler._max_retries, 1) 521 # Find the job_id of the test that allows retry 522 job_id = suite._retry_handler._retry_map.iterkeys().next() 523 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 524 self.assertEqual(suite._retry_handler._max_retries, 0) 525 526 527 def testSuiteDependencies(self): 528 """Should add suite dependencies to tests scheduled.""" 529 # Since add_experimental set to False, will skip experimental data_one. 530 name_list = ['name-data_two', 'name-data_three', 531 'name-data_four', 'name-data_five', 'name-data_six', 532 'name-data_seven'] 533 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6, 534 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 535 536 self.mock_control_file_parsing() 537 recorder = self.mox.CreateMock(base_job.base_job) 538 self.expect_job_scheduling(recorder, add_experimental=False, 539 suite_deps=['extra']) 540 self.mox.StubOutWithMock(utils, 'write_keyval') 541 utils.write_keyval(None, keyval_dict) 542 543 self.mox.ReplayAll() 544 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 545 self.devserver, extra_deps=['extra'], 546 afe=self.afe, tko=self.tko) 547 suite.schedule(recorder.record_entry, add_experimental=False) 548 549 550 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 551 """Create a Suite, using mocked tests and control file contents. 552 553 @return Suite object, after mocking out behavior needed to create it. 554 """ 555 self.expect_control_file_parsing() 556 self.mox.ReplayAll() 557 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 558 self.devserver, 559 self.getter, 560 afe=self.afe, tko=self.tko, 561 file_bugs=file_bugs, job_retry=True) 562 self.mox.ResetAll() 563 return suite 564 565 566 def _createSuiteMockResults(self, results_dir=None, will_file_bug=True, 567 result_status='FAIL'): 568 """Create a suite, returned a set of mocked results to expect. 569 570 @param results_dir: A mock results directory. 571 @param will_file_bug: Whether later a bug will be filed. 572 If true, will mock out tko method. 573 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 574 575 @return List of mocked results to wait on. 576 """ 577 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 578 file_bugs=True) 579 self.suite._results_dir = results_dir 580 test_report = self._get_bad_test_report(result_status) 581 test_predicates = test_report.predicates 582 test_fallout = test_report.fallout 583 584 self.recorder = self.mox.CreateMock(base_job.base_job) 585 self.recorder.record_entry = self.mox.CreateMock( 586 base_job.base_job.record_entry) 587 self._mock_recorder_with_results([test_predicates], self.recorder) 588 if will_file_bug: 589 self.suite._tko.run = self.mox.CreateMock(frontend.RpcClient.run) 590 self.suite._tko.run('get_detailed_test_views', 591 afe_job_id=self._FAKE_JOB_ID) 592 return [test_predicates, test_fallout] 593 594 595 def _mock_recorder_with_results(self, results, recorder): 596 """ 597 Checks that results are recoded in order, eg: 598 START, (status, name, reason) END 599 600 @param results: list of results 601 @param recorder: status recorder 602 """ 603 for result in results: 604 status = result[0] 605 test_name = result[1] 606 recorder.record_entry( 607 StatusContains.CreateFromStrings('START', test_name), 608 log_in_subdir=False) 609 recorder.record_entry( 610 StatusContains.CreateFromStrings(*result), 611 log_in_subdir=False).InAnyOrder('results') 612 recorder.record_entry( 613 StatusContains.CreateFromStrings('END %s' % status, test_name), 614 log_in_subdir=False) 615 616 617 def schedule_and_expect_these_results(self, suite, results, recorder): 618 """Create mox stubs for call to suite.schedule and 619 job_status.wait_for_results 620 621 @param suite: suite object for which to stub out schedule(...) 622 @param results: results object to be returned from 623 job_stats_wait_for_results(...) 624 @param recorder: mocked recorder object to replay status messages 625 """ 626 def result_generator(results): 627 """A simple generator which generates results as Status objects. 628 629 This generator handles 'send' by simply ignoring it. 630 631 @param results: results object to be returned from 632 job_stats_wait_for_results(...) 633 @yield: job_status.Status objects. 634 """ 635 results = map(lambda r: job_status.Status(*r), results) 636 for r in results: 637 new_input = (yield r) 638 if new_input: 639 yield None 640 641 self.mox.StubOutWithMock(suite, 'schedule') 642 suite.schedule(recorder.record_entry, True) 643 suite._retry_handler = RetryHandler({}) 644 645 self.mox.StubOutWithMock(job_status, 'wait_for_results') 646 job_status.wait_for_results( 647 self.afe, self.tko, suite._jobs).AndReturn( 648 result_generator(results)) 649 650 651 def testRunAndWaitSuccess(self): 652 """Should record successful results.""" 653 suite = self._createSuiteWithMockedTestsAndControlFiles() 654 655 recorder = self.mox.CreateMock(base_job.base_job) 656 657 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 658 self._mock_recorder_with_results(results, recorder) 659 self.schedule_and_expect_these_results(suite, results, recorder) 660 self.mox.ReplayAll() 661 662 suite.schedule(recorder.record_entry, True) 663 suite.wait(recorder.record_entry) 664 665 666 def testRunAndWaitFailure(self): 667 """Should record failure to gather results.""" 668 suite = self._createSuiteWithMockedTestsAndControlFiles() 669 670 recorder = self.mox.CreateMock(base_job.base_job) 671 recorder.record_entry( 672 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 673 log_in_subdir=False) 674 675 self.mox.StubOutWithMock(suite, 'schedule') 676 suite.schedule(recorder.record_entry, True) 677 self.mox.StubOutWithMock(job_status, 'wait_for_results') 678 job_status.wait_for_results(mox.IgnoreArg(), 679 mox.IgnoreArg(), 680 mox.IgnoreArg()).AndRaise( 681 Exception('Expected during test.')) 682 self.mox.ReplayAll() 683 684 suite.schedule(recorder.record_entry, True) 685 suite.wait(recorder.record_entry) 686 687 688 def testRunAndWaitScheduleFailure(self): 689 """Should record failure to schedule jobs.""" 690 suite = self._createSuiteWithMockedTestsAndControlFiles() 691 692 recorder = self.mox.CreateMock(base_job.base_job) 693 recorder.record_entry( 694 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 695 log_in_subdir=False) 696 697 recorder.record_entry( 698 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 699 log_in_subdir=False) 700 701 self.mox.StubOutWithMock(suite, '_create_job') 702 suite._create_job(mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 703 Exception('Expected during test.')) 704 self.mox.ReplayAll() 705 706 suite.schedule(recorder.record_entry, True) 707 suite.wait(recorder.record_entry) 708 709 710 def testGetTestsSortedByTime(self): 711 """Should find all tests and sorted by TIME setting.""" 712 self.expect_control_file_parsing() 713 self.mox.ReplayAll() 714 # Get all tests. 715 tests = Suite.find_and_parse_tests(self.getter, 716 lambda d: True, 717 self._TAG, 718 add_experimental=True) 719 self.assertEquals(len(tests), 7) 720 times = [control_data.ControlData.get_test_time_index(test.time) 721 for test in tests] 722 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 723 'Tests are not ordered correctly.') 724 725 726 def _get_bad_test_report(self, result_status='FAIL'): 727 """ 728 Fetch the predicates of a failing test, and the parameters 729 that are a fallout of this test failing. 730 """ 731 predicates = collections.namedtuple('predicates', 732 'status, testname, reason') 733 fallout = collections.namedtuple('fallout', 734 ('time_start, time_end, job_id,' 735 'username, hostname')) 736 test_report = collections.namedtuple('test_report', 737 'predicates, fallout') 738 return test_report(predicates(result_status, 'bad_test', 739 'dreadful_reason'), 740 fallout('2014-01-01 01:01:01', 'None', 741 self._FAKE_JOB_ID, 'user', 'myhost')) 742 743 744 def mock_bug_filing(self, test_results): 745 """A helper function that mocks bug filing. 746 747 @param test_results: A named tuple (predicates, fallout) representing 748 a bad test report. 749 """ 750 def check_result(result): 751 """ 752 Checks to see if the status passed to the bug reporter contains all 753 the arguments required to file bugs. 754 755 @param result: The result we get when a test fails. 756 """ 757 test_predicates = test_results[0] 758 test_fallout = test_results[1] 759 expected_result = job_status.Status( 760 test_predicates.status, test_predicates.testname, 761 reason=test_predicates.reason, 762 job_id=test_fallout.job_id, owner=test_fallout.username, 763 hostname=test_fallout.hostname, 764 begin_time_str=test_fallout.time_start) 765 766 return all(getattr(result, k, None) == v for k, v in 767 expected_result.__dict__.iteritems() 768 if 'timestamp' not in str(k)) 769 770 self.mox.StubOutWithMock(reporting, 'TestBug') 771 reporting.TestBug(self._BUILDS[provision.CROS_VERSION_PREFIX], 772 mox.IgnoreArg(), mox.IgnoreArg(), 773 mox.Func(check_result)) 774 775 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 776 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 777 mox.IgnoreArg(), 778 mox.IgnoreArg()) 779 self.mox.StubOutWithMock(reporting.Reporter, 'report') 780 reporting.Reporter.report(mox.IgnoreArg(), 781 mox.IgnoreArg()).AndReturn((0, 0)) 782 783 self.mox.StubOutWithMock(utils, 'write_keyval') 784 utils.write_keyval(mox.IgnoreArg(), mox.IgnoreArg()) 785 786 787 def testBugFiling(self): 788 """ 789 Confirm that all the necessary predicates are passed on to the 790 bug reporter when a test fails. 791 """ 792 test_results = self._createSuiteMockResults() 793 self.schedule_and_expect_these_results( 794 self.suite, 795 [test_results[0] + test_results[1]], 796 self.recorder) 797 798 self.mock_bug_filing(test_results) 799 self.mox.ReplayAll() 800 801 self.suite.schedule(self.recorder.record_entry, True) 802 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 803 self.suite.wait(self.recorder.record_entry) 804 805 806 def testFailedBugFiling(self): 807 """ 808 Make sure the suite survives even if we cannot file bugs. 809 """ 810 test_results = self._createSuiteMockResults(self.tmpdir) 811 self.schedule_and_expect_these_results( 812 self.suite, 813 [test_results[0] + test_results[1]], 814 self.recorder) 815 self.mox.StubOutWithMock(reporting.Reporter, '_check_tracker') 816 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 817 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 818 mox.IgnoreArg(), 819 mox.IgnoreArg()) 820 reporting.Reporter._check_tracker().AndReturn(False) 821 822 self.mox.ReplayAll() 823 824 self.suite.schedule(self.recorder.record_entry, True) 825 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = self.files['seven'] 826 self.suite.wait(self.recorder.record_entry) 827 828 829 def testJobRetryTestFail(self): 830 """Test retry works.""" 831 test_to_retry = self.files['seven'] 832 fake_new_job_id = self._FAKE_JOB_ID + 1 833 fake_job = FakeJob(id=self._FAKE_JOB_ID) 834 fake_new_job = FakeJob(id=fake_new_job_id) 835 836 test_results = self._createSuiteMockResults(will_file_bug=False) 837 self.schedule_and_expect_these_results( 838 self.suite, 839 [test_results[0] + test_results[1]], 840 self.recorder) 841 self.mox.StubOutWithMock(self.suite, '_create_job') 842 self.suite._create_job( 843 test_to_retry, 844 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 845 self.mox.ReplayAll() 846 self.suite.schedule(self.recorder.record_entry, True) 847 self.suite._retry_handler._retry_map = { 848 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 849 'retry_max': 1} 850 } 851 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 852 self.suite.wait(self.recorder.record_entry) 853 expected_retry_map = { 854 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 855 'retry_max': 1}, 856 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 857 'retry_max': 0} 858 } 859 # Check retry map is correctly updated 860 self.assertEquals(self.suite._retry_handler._retry_map, 861 expected_retry_map) 862 # Check _jobs_to_tests is correctly updated 863 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 864 test_to_retry) 865 866 867 def testJobRetryTestWarn(self): 868 """Test that no retry is scheduled if test warns.""" 869 test_to_retry = self.files['seven'] 870 fake_job = FakeJob(id=self._FAKE_JOB_ID) 871 test_results = self._createSuiteMockResults( 872 will_file_bug=True, result_status='WARN') 873 self.schedule_and_expect_these_results( 874 self.suite, 875 [test_results[0] + test_results[1]], 876 self.recorder) 877 # A bug should be filed if test warns. 878 self.mock_bug_filing(test_results) 879 self.mox.ReplayAll() 880 self.suite.schedule(self.recorder.record_entry, True) 881 self.suite._retry_handler._retry_map = { 882 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 883 'retry_max': 1} 884 } 885 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 886 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 887 expected_retry_map = self.suite._retry_handler._retry_map.copy() 888 self.suite.wait(self.recorder.record_entry) 889 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 890 self.assertEquals(self.suite._retry_handler._retry_map, 891 expected_retry_map) 892 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 893 894 895 def testFailedJobRetry(self): 896 """Make sure the suite survives even if the retry failed.""" 897 test_to_retry = self.files['seven'] 898 fake_job = FakeJob(id=self._FAKE_JOB_ID) 899 900 test_results = self._createSuiteMockResults(will_file_bug=False) 901 self.schedule_and_expect_these_results( 902 self.suite, 903 [test_results[0] + test_results[1]], 904 self.recorder) 905 self.mox.StubOutWithMock(self.suite, '_create_job') 906 self.suite._create_job( 907 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 908 error.RPCException('Expected during test')) 909 # Do not file a bug. 910 self.mox.StubOutWithMock(self.suite, '_should_report') 911 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 912 913 self.mox.ReplayAll() 914 915 self.suite.schedule(self.recorder.record_entry, True) 916 self.suite._retry_handler._retry_map = { 917 self._FAKE_JOB_ID: { 918 'state': RetryHandler.States.NOT_ATTEMPTED, 919 'retry_max': 1}} 920 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 921 self.suite.wait(self.recorder.record_entry) 922 expected_retry_map = { 923 self._FAKE_JOB_ID: { 924 'state': RetryHandler.States.ATTEMPTED, 925 'retry_max': 1}} 926 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 927 self.assertEquals(self.suite._retry_handler._retry_map, 928 expected_retry_map) 929 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 930 931 932 if __name__ == '__main__': 933 unittest.main() 934
