1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Unit tests for server/cros/dynamic_suite/dynamic_suite.py.""" 9 10 import collections 11 from collections import OrderedDict 12 import os 13 import shutil 14 import tempfile 15 import unittest 16 17 import mock 18 import mox 19 20 import common 21 22 from autotest_lib.client.common_lib import base_job 23 from autotest_lib.client.common_lib import control_data 24 from autotest_lib.client.common_lib import error 25 from autotest_lib.client.common_lib import priorities 26 from autotest_lib.client.common_lib import utils 27 from autotest_lib.client.common_lib.cros import dev_server 28 from autotest_lib.server import frontend 29 from autotest_lib.server.cros import provision 30 from autotest_lib.server.cros.dynamic_suite import control_file_getter 31 from autotest_lib.server.cros.dynamic_suite import constants 32 from autotest_lib.server.cros.dynamic_suite import job_status 33 from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase 34 from autotest_lib.server.cros.dynamic_suite import suite_common 35 from autotest_lib.server.cros.dynamic_suite.comparators import StatusContains 36 from autotest_lib.server.cros.dynamic_suite.fakes import FakeControlData 37 from autotest_lib.server.cros.dynamic_suite.fakes import FakeJob 38 from autotest_lib.server.cros.dynamic_suite.fakes import FakeMultiprocessingPool 39 from autotest_lib.server.cros.dynamic_suite.suite import RetryHandler 40 from autotest_lib.server.cros.dynamic_suite.suite import Suite 41 42 class SuiteTest(mox.MoxTestBase): 43 """Unit tests for dynamic_suite Suite class. 44 45 @var _BUILDS: fake build 46 @var _TAG: fake suite tag 47 """ 48 49 _BOARD = 'board:board' 50 _BUILDS = {provision.CROS_VERSION_PREFIX:'build_1', 51 provision.FW_RW_VERSION_PREFIX:'fwrw_build_1'} 52 _TAG = 'au' 53 _ATTR = {'attr:attr'} 54 _DEVSERVER_HOST = 'http://dontcare:8080' 55 _FAKE_JOB_ID = 10 56 57 58 def setUp(self): 59 """Setup.""" 60 super(SuiteTest, self).setUp() 61 self.maxDiff = None 62 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 63 suite_common.ENABLE_CONTROLS_IN_BATCH = False 64 self.afe = self.mox.CreateMock(frontend.AFE) 65 self.tko = self.mox.CreateMock(frontend.TKO) 66 67 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 68 69 self.getter = self.mox.CreateMock(control_file_getter.ControlFileGetter) 70 self.devserver = dev_server.ImageServer(self._DEVSERVER_HOST) 71 72 self.files = OrderedDict( 73 [('one', FakeControlData(self._TAG, self._ATTR, 'data_one', 74 'FAST', job_retries=None)), 75 ('two', FakeControlData(self._TAG, self._ATTR, 'data_two', 76 'SHORT', dependencies=['feta'])), 77 ('three', FakeControlData(self._TAG, self._ATTR, 'data_three', 78 'MEDIUM')), 79 ('four', FakeControlData('other', self._ATTR, 'data_four', 80 'LONG', dependencies=['arugula'])), 81 ('five', FakeControlData(self._TAG, {'other'}, 'data_five', 82 'LONG', dependencies=['arugula', 83 'caligula'])), 84 ('six', FakeControlData(self._TAG, self._ATTR, 'data_six', 85 'LENGTHY')), 86 ('seven', FakeControlData(self._TAG, self._ATTR, 'data_seven', 87 'FAST', job_retries=1))]) 88 89 self.files_to_filter = { 90 'with/deps/...': FakeControlData(self._TAG, self._ATTR, 91 'gets filtered'), 92 'with/profilers/...': FakeControlData(self._TAG, self._ATTR, 93 'gets filtered')} 94 95 96 def tearDown(self): 97 """Teardown.""" 98 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 99 super(SuiteTest, self).tearDown() 100 shutil.rmtree(self.tmpdir, ignore_errors=True) 101 102 103 def expect_control_file_parsing(self, suite_name=_TAG): 104 """Expect an attempt to parse the 'control files' in |self.files|. 105 106 @param suite_name: The suite name to parse control files for. 107 """ 108 all_files = self.files.keys() + self.files_to_filter.keys() 109 self._set_control_file_parsing_expectations(False, all_files, 110 self.files, suite_name) 111 112 113 def _set_control_file_parsing_expectations(self, already_stubbed, 114 file_list, files_to_parse, 115 suite_name): 116 """Expect an attempt to parse the 'control files' in |files|. 117 118 @param already_stubbed: parse_control_string already stubbed out. 119 @param file_list: the files the dev server returns 120 @param files_to_parse: the {'name': FakeControlData} dict of files we 121 expect to get parsed. 122 """ 123 if not already_stubbed: 124 self.mox.StubOutWithMock(control_data, 'parse_control_string') 125 126 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 127 suite_common.multiprocessing.Pool( 128 processes=suite_common.get_process_limit()).AndReturn( 129 FakeMultiprocessingPool()) 130 131 self.getter.get_control_file_list( 132 suite_name=suite_name).AndReturn(file_list) 133 for file, data in files_to_parse.iteritems(): 134 self.getter.get_control_file_contents( 135 file).InAnyOrder().AndReturn(data.string) 136 control_data.parse_control_string( 137 data.string, 138 raise_warnings=True, 139 path=file).InAnyOrder().AndReturn(data) 140 141 142 def expect_control_file_parsing_in_batch(self, suite_name=_TAG): 143 """Expect an attempt to parse the contents of all control files in 144 |self.files| and |self.files_to_filter|, form them to a dict. 145 146 @param suite_name: The suite name to parse control files for. 147 """ 148 self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter) 149 self.mox.StubOutWithMock(control_data, 'parse_control_string') 150 151 self.mox.StubOutWithMock(suite_common.multiprocessing, 'Pool') 152 suite_common.multiprocessing.Pool( 153 processes=suite_common.get_process_limit()).AndReturn( 154 FakeMultiprocessingPool()) 155 156 suite_info = {} 157 for k, v in self.files.iteritems(): 158 suite_info[k] = v.string 159 control_data.parse_control_string( 160 v.string, 161 raise_warnings=True, 162 path=k).InAnyOrder().AndReturn(v) 163 for k, v in self.files_to_filter.iteritems(): 164 suite_info[k] = v.string 165 self.getter._dev_server = self._DEVSERVER_HOST 166 self.getter.get_suite_info( 167 suite_name=suite_name).AndReturn(suite_info) 168 169 170 def testFindAllTestInBatch(self): 171 """Test switch on enable_getting_controls_in_batch for function 172 find_all_test.""" 173 self.use_batch = suite_common.ENABLE_CONTROLS_IN_BATCH 174 self.expect_control_file_parsing_in_batch() 175 suite_common.ENABLE_CONTROLS_IN_BATCH = True 176 177 self.mox.ReplayAll() 178 179 predicate = lambda d: d.suite == self._TAG 180 tests = SuiteBase.find_and_parse_tests(self.getter, 181 predicate, 182 self._TAG) 183 self.assertEquals(len(tests), 6) 184 self.assertTrue(self.files['one'] in tests) 185 self.assertTrue(self.files['two'] in tests) 186 self.assertTrue(self.files['three'] in tests) 187 self.assertTrue(self.files['five'] in tests) 188 self.assertTrue(self.files['six'] in tests) 189 self.assertTrue(self.files['seven'] in tests) 190 suite_common.ENABLE_CONTROLS_IN_BATCH = self.use_batch 191 192 193 def testFindAndParseStableTests(self): 194 """Should find only tests that match a predicate.""" 195 self.expect_control_file_parsing() 196 self.mox.ReplayAll() 197 198 predicate = lambda d: d.text == self.files['two'].string 199 tests = SuiteBase.find_and_parse_tests(self.getter, 200 predicate, 201 self._TAG) 202 self.assertEquals(len(tests), 1) 203 self.assertEquals(tests[0], self.files['two']) 204 205 206 def testFindSuiteSyntaxErrors(self): 207 """Check all control files for syntax errors. 208 209 This test actually parses all control files in the autotest directory 210 for syntax errors, by using the un-forgiving parser and pretending to 211 look for all control files with the suite attribute. 212 """ 213 autodir = os.path.abspath( 214 os.path.join(os.path.dirname(__file__), '..', '..', '..')) 215 fs_getter = SuiteBase.create_fs_getter(autodir) 216 predicate = lambda t: hasattr(t, 'suite') 217 SuiteBase.find_and_parse_tests(fs_getter, predicate, 218 forgiving_parser=False) 219 220 221 def testFindAndParseTestsSuite(self): 222 """Should find all tests that match a predicate.""" 223 self.expect_control_file_parsing() 224 self.mox.ReplayAll() 225 226 predicate = lambda d: d.suite == self._TAG 227 tests = SuiteBase.find_and_parse_tests(self.getter, 228 predicate, 229 self._TAG) 230 self.assertEquals(len(tests), 6) 231 self.assertTrue(self.files['one'] in tests) 232 self.assertTrue(self.files['two'] in tests) 233 self.assertTrue(self.files['three'] in tests) 234 self.assertTrue(self.files['five'] in tests) 235 self.assertTrue(self.files['six'] in tests) 236 self.assertTrue(self.files['seven'] in tests) 237 238 239 def testFindAndParseTestsAttr(self): 240 """Should find all tests that match a predicate.""" 241 self.expect_control_file_parsing() 242 self.mox.ReplayAll() 243 244 predicate = SuiteBase.matches_attribute_expression_predicate('attr:attr') 245 tests = SuiteBase.find_and_parse_tests(self.getter, 246 predicate, 247 self._TAG) 248 self.assertEquals(len(tests), 6) 249 self.assertTrue(self.files['one'] in tests) 250 self.assertTrue(self.files['two'] in tests) 251 self.assertTrue(self.files['three'] in tests) 252 self.assertTrue(self.files['four'] in tests) 253 self.assertTrue(self.files['six'] in tests) 254 self.assertTrue(self.files['seven'] in tests) 255 256 257 def testAdHocSuiteCreation(self): 258 """Should be able to schedule an ad-hoc suite by specifying 259 a single test name.""" 260 self.expect_control_file_parsing(suite_name='ad_hoc_suite') 261 self.mox.ReplayAll() 262 predicate = SuiteBase.test_name_equals_predicate('name-data_five') 263 suite = Suite.create_from_predicates([predicate], self._BUILDS, 264 self._BOARD, devserver=None, 265 cf_getter=self.getter, 266 afe=self.afe, tko=self.tko) 267 268 self.assertFalse(self.files['one'] in suite.tests) 269 self.assertFalse(self.files['two'] in suite.tests) 270 self.assertFalse(self.files['four'] in suite.tests) 271 self.assertTrue(self.files['five'] in suite.tests) 272 273 274 def mock_control_file_parsing(self): 275 """Fake out find_and_parse_tests(), returning content from |self.files|. 276 """ 277 for test in self.files.values(): 278 test.text = test.string # mimic parsing. 279 self.mox.StubOutWithMock(SuiteBase, 'find_and_parse_tests') 280 SuiteBase.find_and_parse_tests( 281 mox.IgnoreArg(), 282 mox.IgnoreArg(), 283 mox.IgnoreArg(), 284 forgiving_parser=True, 285 run_prod_code=False, 286 test_args=None).AndReturn(self.files.values()) 287 288 289 def expect_job_scheduling(self, recorder, 290 tests_to_skip=[], ignore_deps=False, 291 raises=False, suite_deps=[], suite=None, 292 extra_keyvals={}): 293 """Expect jobs to be scheduled for 'tests' in |self.files|. 294 295 @param recorder: object with a record_entry to be used to record test 296 results. 297 @param tests_to_skip: [list, of, test, names] that we expect to skip. 298 @param ignore_deps: If true, ignore tests' dependencies. 299 @param raises: If True, expect exceptions. 300 @param suite_deps: If True, add suite level dependencies. 301 @param extra_keyvals: Extra keyvals set to tests. 302 """ 303 record_job_id = suite and suite._results_dir 304 if record_job_id: 305 self.mox.StubOutWithMock(suite, '_remember_job_keyval') 306 recorder.record_entry( 307 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 308 log_in_subdir=False) 309 tests = self.files.values() 310 n = 1 311 for test in tests: 312 if test.name in tests_to_skip: 313 continue 314 dependencies = [] 315 if not ignore_deps: 316 dependencies.extend(test.dependencies) 317 if suite_deps: 318 dependencies.extend(suite_deps) 319 dependencies.append(self._BOARD) 320 build = self._BUILDS[provision.CROS_VERSION_PREFIX] 321 keyvals = { 322 'build': build, 323 'suite': self._TAG, 324 'builds': SuiteTest._BUILDS, 325 'experimental':test.experimental, 326 } 327 keyvals.update(extra_keyvals) 328 job_mock = self.afe.create_job( 329 control_file=test.text, 330 name=mox.And(mox.StrContains(build), 331 mox.StrContains(test.name)), 332 control_type=mox.IgnoreArg(), 333 meta_hosts=[self._BOARD], 334 dependencies=dependencies, 335 keyvals=keyvals, 336 max_runtime_mins=24*60, 337 timeout_mins=1440, 338 parent_job_id=None, 339 reboot_before=mox.IgnoreArg(), 340 run_reset=mox.IgnoreArg(), 341 priority=priorities.Priority.DEFAULT, 342 synch_count=test.sync_count, 343 require_ssp=test.require_ssp 344 ) 345 if raises: 346 job_mock.AndRaise(error.NoEligibleHostException()) 347 recorder.record_entry( 348 StatusContains.CreateFromStrings('START', test.name), 349 log_in_subdir=False) 350 recorder.record_entry( 351 StatusContains.CreateFromStrings('TEST_NA', test.name), 352 log_in_subdir=False) 353 recorder.record_entry( 354 StatusContains.CreateFromStrings('END', test.name), 355 log_in_subdir=False) 356 else: 357 fake_job = FakeJob(id=n) 358 job_mock.AndReturn(fake_job) 359 if record_job_id: 360 suite._remember_job_keyval(fake_job) 361 n += 1 362 363 364 def testScheduleTestsAndRecord(self): 365 """Should schedule stable and experimental tests with the AFE.""" 366 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 367 'name-data_four', 'name-data_five', 'name-data_six', 368 'name-data_seven'] 369 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 370 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 371 372 self.mock_control_file_parsing() 373 self.mox.ReplayAll() 374 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 375 self.devserver, 376 afe=self.afe, tko=self.tko, 377 results_dir=self.tmpdir) 378 self.mox.ResetAll() 379 recorder = self.mox.CreateMock(base_job.base_job) 380 self.expect_job_scheduling(recorder, suite=suite) 381 382 self.mox.StubOutWithMock(utils, 'write_keyval') 383 utils.write_keyval(self.tmpdir, keyval_dict) 384 self.mox.ReplayAll() 385 suite.schedule(recorder.record_entry) 386 for job in suite._jobs: 387 self.assertTrue(hasattr(job, 'test_name')) 388 389 390 def testScheduleTests(self): 391 """Should schedule tests with the AFE.""" 392 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 393 'name-data_four', 'name-data_five', 'name-data_six', 394 'name-data_seven'] 395 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 396 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 397 398 self.mock_control_file_parsing() 399 recorder = self.mox.CreateMock(base_job.base_job) 400 self.expect_job_scheduling(recorder) 401 self.mox.StubOutWithMock(utils, 'write_keyval') 402 utils.write_keyval(None, keyval_dict) 403 404 self.mox.ReplayAll() 405 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 406 self.devserver, 407 afe=self.afe, tko=self.tko) 408 suite.schedule(recorder.record_entry) 409 410 411 def testScheduleTestsIgnoreDeps(self): 412 """Test scheduling tests ignoring deps.""" 413 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 414 'name-data_four', 'name-data_five', 'name-data_six', 415 'name-data_seven'] 416 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 417 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 418 419 self.mock_control_file_parsing() 420 recorder = self.mox.CreateMock(base_job.base_job) 421 self.expect_job_scheduling(recorder, ignore_deps=True) 422 self.mox.StubOutWithMock(utils, 'write_keyval') 423 utils.write_keyval(None, keyval_dict) 424 425 self.mox.ReplayAll() 426 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 427 self.devserver, 428 afe=self.afe, tko=self.tko, 429 ignore_deps=True) 430 suite.schedule(recorder.record_entry) 431 432 433 def testScheduleUnrunnableTestsTESTNA(self): 434 """Tests which fail to schedule should be TEST_NA.""" 435 # Since all tests will be fail to schedule, the num of scheduled tests 436 # will be zero. 437 name_list = [] 438 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0, 439 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 440 441 self.mock_control_file_parsing() 442 recorder = self.mox.CreateMock(base_job.base_job) 443 self.expect_job_scheduling(recorder, raises=True) 444 self.mox.StubOutWithMock(utils, 'write_keyval') 445 utils.write_keyval(None, keyval_dict) 446 self.mox.ReplayAll() 447 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 448 self.devserver, 449 afe=self.afe, tko=self.tko) 450 suite.schedule(recorder.record_entry) 451 452 453 def testRetryMapAfterScheduling(self): 454 """Test job-test and test-job mapping are correctly updated.""" 455 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 456 'name-data_four', 'name-data_five', 'name-data_six', 457 'name-data_seven'] 458 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 459 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 460 461 self.mock_control_file_parsing() 462 recorder = self.mox.CreateMock(base_job.base_job) 463 self.expect_job_scheduling(recorder) 464 self.mox.StubOutWithMock(utils, 'write_keyval') 465 utils.write_keyval(None, keyval_dict) 466 467 all_files = self.files.items() 468 # Sort tests in self.files so that they are in the same 469 # order as they are scheduled. 470 expected_retry_map = {} 471 for n in range(len(all_files)): 472 test = all_files[n][1] 473 job_id = n + 1 474 job_retries = 1 if test.job_retries is None else test.job_retries 475 if job_retries > 0: 476 expected_retry_map[job_id] = { 477 'state': RetryHandler.States.NOT_ATTEMPTED, 478 'retry_max': job_retries} 479 480 self.mox.ReplayAll() 481 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 482 self.devserver, 483 afe=self.afe, tko=self.tko, 484 job_retry=True) 485 suite.schedule(recorder.record_entry) 486 487 self.assertEqual(expected_retry_map, suite._retry_handler._retry_map) 488 489 490 def testSuiteMaxRetries(self): 491 """Test suite max retries.""" 492 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 493 'name-data_four', 'name-data_five', 494 'name-data_six', 'name-data_seven'] 495 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7, 496 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 497 498 self.mock_control_file_parsing() 499 recorder = self.mox.CreateMock(base_job.base_job) 500 self.expect_job_scheduling(recorder) 501 self.mox.StubOutWithMock(utils, 'write_keyval') 502 utils.write_keyval(None, keyval_dict) 503 self.mox.ReplayAll() 504 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 505 self.devserver, 506 afe=self.afe, tko=self.tko, 507 job_retry=True, max_retries=1) 508 suite.schedule(recorder.record_entry) 509 self.assertEqual(suite._retry_handler._max_retries, 1) 510 # Find the job_id of the test that allows retry 511 job_id = suite._retry_handler._retry_map.iterkeys().next() 512 suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10) 513 self.assertEqual(suite._retry_handler._max_retries, 0) 514 515 516 def testSuiteDependencies(self): 517 """Should add suite dependencies to tests scheduled.""" 518 name_list = ['name-data_one', 'name-data_two', 'name-data_three', 519 'name-data_four', 'name-data_five', 'name-data_six', 520 'name-data_seven'] 521 keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: len(name_list), 522 constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)} 523 524 self.mock_control_file_parsing() 525 recorder = self.mox.CreateMock(base_job.base_job) 526 self.expect_job_scheduling(recorder, suite_deps=['extra']) 527 self.mox.StubOutWithMock(utils, 'write_keyval') 528 utils.write_keyval(None, keyval_dict) 529 530 self.mox.ReplayAll() 531 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 532 self.devserver, extra_deps=['extra'], 533 afe=self.afe, tko=self.tko) 534 suite.schedule(recorder.record_entry) 535 536 537 def testInheritedKeyvals(self): 538 """Tests should inherit some whitelisted job keyvals.""" 539 # Only keyvals in constants.INHERITED_KEYVALS are inherited to tests. 540 job_keyvals = { 541 constants.KEYVAL_CIDB_BUILD_ID: '111', 542 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 543 'your': 'name', 544 } 545 test_keyvals = { 546 constants.KEYVAL_CIDB_BUILD_ID: '111', 547 constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222', 548 } 549 550 self.mock_control_file_parsing() 551 recorder = self.mox.CreateMock(base_job.base_job) 552 self.expect_job_scheduling( 553 recorder, 554 extra_keyvals=test_keyvals) 555 self.mox.StubOutWithMock(utils, 'write_keyval') 556 utils.write_keyval(None, job_keyvals) 557 utils.write_keyval(None, mox.IgnoreArg()) 558 559 self.mox.ReplayAll() 560 suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD, 561 self.devserver, 562 afe=self.afe, tko=self.tko, 563 job_keyvals=job_keyvals) 564 suite.schedule(recorder.record_entry) 565 566 567 def _createSuiteWithMockedTestsAndControlFiles(self, file_bugs=False): 568 """Create a Suite, using mocked tests and control file contents. 569 570 @return Suite object, after mocking out behavior needed to create it. 571 """ 572 self.result_reporter = _MemoryResultReporter() 573 self.expect_control_file_parsing() 574 self.mox.ReplayAll() 575 suite = Suite.create_from_name( 576 self._TAG, 577 self._BUILDS, 578 self._BOARD, 579 self.devserver, 580 self.getter, 581 afe=self.afe, 582 tko=self.tko, 583 file_bugs=file_bugs, 584 job_retry=True, 585 result_reporter=self.result_reporter, 586 ) 587 self.mox.ResetAll() 588 return suite 589 590 591 def _createSuiteMockResults(self, results_dir=None, result_status='FAIL'): 592 """Create a suite, returned a set of mocked results to expect. 593 594 @param results_dir: A mock results directory. 595 @param result_status: A desired result status, e.g. 'FAIL', 'WARN'. 596 597 @return List of mocked results to wait on. 598 """ 599 self.suite = self._createSuiteWithMockedTestsAndControlFiles( 600 file_bugs=True) 601 self.suite._results_dir = results_dir 602 test_report = self._get_bad_test_report(result_status) 603 test_predicates = test_report.predicates 604 test_fallout = test_report.fallout 605 606 self.recorder = self.mox.CreateMock(base_job.base_job) 607 self.recorder.record_entry = self.mox.CreateMock( 608 base_job.base_job.record_entry) 609 self._mock_recorder_with_results([test_predicates], self.recorder) 610 return [test_predicates, test_fallout] 611 612 613 def _mock_recorder_with_results(self, results, recorder): 614 """ 615 Checks that results are recoded in order, eg: 616 START, (status, name, reason) END 617 618 @param results: list of results 619 @param recorder: status recorder 620 """ 621 for result in results: 622 status = result[0] 623 test_name = result[1] 624 recorder.record_entry( 625 StatusContains.CreateFromStrings('START', test_name), 626 log_in_subdir=False) 627 recorder.record_entry( 628 StatusContains.CreateFromStrings(*result), 629 log_in_subdir=False).InAnyOrder('results') 630 recorder.record_entry( 631 StatusContains.CreateFromStrings('END %s' % status, test_name), 632 log_in_subdir=False) 633 634 635 def schedule_and_expect_these_results(self, suite, results, recorder): 636 """Create mox stubs for call to suite.schedule and 637 job_status.wait_for_results 638 639 @param suite: suite object for which to stub out schedule(...) 640 @param results: results object to be returned from 641 job_stats_wait_for_results(...) 642 @param recorder: mocked recorder object to replay status messages 643 """ 644 def result_generator(results): 645 """A simple generator which generates results as Status objects. 646 647 This generator handles 'send' by simply ignoring it. 648 649 @param results: results object to be returned from 650 job_stats_wait_for_results(...) 651 @yield: job_status.Status objects. 652 """ 653 results = map(lambda r: job_status.Status(*r), results) 654 for r in results: 655 new_input = (yield r) 656 if new_input: 657 yield None 658 659 self.mox.StubOutWithMock(suite, 'schedule') 660 suite.schedule(recorder.record_entry) 661 suite._retry_handler = RetryHandler({}) 662 663 waiter_patch = mock.patch.object( 664 job_status.JobResultWaiter, 'wait_for_results', autospec=True) 665 waiter_mock = waiter_patch.start() 666 waiter_mock.return_value = result_generator(results) 667 self.addCleanup(waiter_patch.stop) 668 669 670 def testRunAndWaitSuccess(self): 671 """Should record successful results.""" 672 suite = self._createSuiteWithMockedTestsAndControlFiles() 673 674 recorder = self.mox.CreateMock(base_job.base_job) 675 676 results = [('GOOD', 'good'), ('FAIL', 'bad', 'reason')] 677 self._mock_recorder_with_results(results, recorder) 678 self.schedule_and_expect_these_results(suite, results, recorder) 679 self.mox.ReplayAll() 680 681 suite.schedule(recorder.record_entry) 682 suite.wait(recorder.record_entry) 683 684 685 def testRunAndWaitFailure(self): 686 """Should record failure to gather results.""" 687 suite = self._createSuiteWithMockedTestsAndControlFiles() 688 689 recorder = self.mox.CreateMock(base_job.base_job) 690 recorder.record_entry( 691 StatusContains.CreateFromStrings('FAIL', self._TAG, 'waiting'), 692 log_in_subdir=False) 693 694 self.mox.StubOutWithMock(suite, 'schedule') 695 suite.schedule(recorder.record_entry) 696 self.mox.ReplayAll() 697 698 with mock.patch.object( 699 job_status.JobResultWaiter, 'wait_for_results', 700 autospec=True) as wait_mock: 701 wait_mock.side_effect = Exception 702 suite.schedule(recorder.record_entry) 703 suite.wait(recorder.record_entry) 704 705 706 def testRunAndWaitScheduleFailure(self): 707 """Should record failure to schedule jobs.""" 708 suite = self._createSuiteWithMockedTestsAndControlFiles() 709 710 recorder = self.mox.CreateMock(base_job.base_job) 711 recorder.record_entry( 712 StatusContains.CreateFromStrings('INFO', 'Start %s' % self._TAG), 713 log_in_subdir=False) 714 715 recorder.record_entry( 716 StatusContains.CreateFromStrings('FAIL', self._TAG, 'scheduling'), 717 log_in_subdir=False) 718 719 self.mox.StubOutWithMock(suite._job_creator, 'create_job') 720 suite._job_creator.create_job( 721 mox.IgnoreArg(), retry_for=mox.IgnoreArg()).AndRaise( 722 Exception('Expected during test.')) 723 self.mox.ReplayAll() 724 725 suite.schedule(recorder.record_entry) 726 suite.wait(recorder.record_entry) 727 728 729 def testGetTestsSortedByTime(self): 730 """Should find all tests and sorted by TIME setting.""" 731 self.expect_control_file_parsing() 732 self.mox.ReplayAll() 733 # Get all tests. 734 tests = SuiteBase.find_and_parse_tests(self.getter, 735 lambda d: True, 736 self._TAG) 737 self.assertEquals(len(tests), 7) 738 times = [control_data.ControlData.get_test_time_index(test.time) 739 for test in tests] 740 self.assertTrue(all(x>=y for x, y in zip(times, times[1:])), 741 'Tests are not ordered correctly.') 742 743 744 def _get_bad_test_report(self, result_status='FAIL'): 745 """ 746 Fetch the predicates of a failing test, and the parameters 747 that are a fallout of this test failing. 748 """ 749 predicates = collections.namedtuple('predicates', 750 'status, testname, reason') 751 fallout = collections.namedtuple('fallout', 752 ('time_start, time_end, job_id,' 753 'username, hostname')) 754 test_report = collections.namedtuple('test_report', 755 'predicates, fallout') 756 return test_report(predicates(result_status, 'bad_test', 757 'dreadful_reason'), 758 fallout('2014-01-01 01:01:01', 'None', 759 self._FAKE_JOB_ID, 'user', 'myhost')) 760 761 762 def testJobRetryTestFail(self): 763 """Test retry works.""" 764 test_to_retry = self.files['seven'] 765 fake_new_job_id = self._FAKE_JOB_ID + 1 766 fake_job = FakeJob(id=self._FAKE_JOB_ID) 767 fake_new_job = FakeJob(id=fake_new_job_id) 768 769 test_results = self._createSuiteMockResults() 770 self.schedule_and_expect_these_results( 771 self.suite, 772 [test_results[0] + test_results[1]], 773 self.recorder) 774 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 775 self.suite._job_creator.create_job( 776 test_to_retry, 777 retry_for=self._FAKE_JOB_ID).AndReturn(fake_new_job) 778 self.mox.ReplayAll() 779 self.suite.schedule(self.recorder.record_entry) 780 self.suite._retry_handler._retry_map = { 781 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 782 'retry_max': 1} 783 } 784 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 785 self.suite.wait(self.recorder.record_entry) 786 expected_retry_map = { 787 self._FAKE_JOB_ID: {'state': RetryHandler.States.RETRIED, 788 'retry_max': 1}, 789 fake_new_job_id: {'state': RetryHandler.States.NOT_ATTEMPTED, 790 'retry_max': 0} 791 } 792 # Check retry map is correctly updated 793 self.assertEquals(self.suite._retry_handler._retry_map, 794 expected_retry_map) 795 # Check _jobs_to_tests is correctly updated 796 self.assertEquals(self.suite._jobs_to_tests[fake_new_job_id], 797 test_to_retry) 798 799 800 def testJobRetryTestWarn(self): 801 """Test that no retry is scheduled if test warns.""" 802 test_to_retry = self.files['seven'] 803 fake_job = FakeJob(id=self._FAKE_JOB_ID) 804 test_results = self._createSuiteMockResults(result_status='WARN') 805 self.schedule_and_expect_these_results( 806 self.suite, 807 [test_results[0] + test_results[1]], 808 self.recorder) 809 self.mox.ReplayAll() 810 self.suite.schedule(self.recorder.record_entry) 811 self.suite._retry_handler._retry_map = { 812 self._FAKE_JOB_ID: {'state': RetryHandler.States.NOT_ATTEMPTED, 813 'retry_max': 1} 814 } 815 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 816 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 817 expected_retry_map = self.suite._retry_handler._retry_map.copy() 818 self.suite.wait(self.recorder.record_entry) 819 self.assertTrue(self.result_reporter.results) 820 # Check retry map and _jobs_to_tests, ensure no retry was scheduled. 821 self.assertEquals(self.suite._retry_handler._retry_map, 822 expected_retry_map) 823 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 824 825 826 def testFailedJobRetry(self): 827 """Make sure the suite survives even if the retry failed.""" 828 test_to_retry = self.files['seven'] 829 fake_job = FakeJob(id=self._FAKE_JOB_ID) 830 831 test_results = self._createSuiteMockResults() 832 self.schedule_and_expect_these_results( 833 self.suite, 834 [test_results[0] + test_results[1]], 835 self.recorder) 836 self.mox.StubOutWithMock(self.suite._job_creator, 'create_job') 837 self.suite._job_creator.create_job( 838 test_to_retry, retry_for=self._FAKE_JOB_ID).AndRaise( 839 error.RPCException('Expected during test')) 840 # Do not file a bug. 841 self.mox.StubOutWithMock(self.suite, '_should_report') 842 self.suite._should_report(mox.IgnoreArg()).AndReturn(False) 843 844 self.mox.ReplayAll() 845 846 self.suite.schedule(self.recorder.record_entry) 847 self.suite._retry_handler._retry_map = { 848 self._FAKE_JOB_ID: { 849 'state': RetryHandler.States.NOT_ATTEMPTED, 850 'retry_max': 1}} 851 self.suite._jobs_to_tests[self._FAKE_JOB_ID] = test_to_retry 852 self.suite.wait(self.recorder.record_entry) 853 expected_retry_map = { 854 self._FAKE_JOB_ID: { 855 'state': RetryHandler.States.ATTEMPTED, 856 'retry_max': 1}} 857 expected_jobs_to_tests = self.suite._jobs_to_tests.copy() 858 self.assertEquals(self.suite._retry_handler._retry_map, 859 expected_retry_map) 860 self.assertEquals(self.suite._jobs_to_tests, expected_jobs_to_tests) 861 862 863 class _MemoryResultReporter(SuiteBase._ResultReporter): 864 """Reporter that stores results internally for testing.""" 865 def __init__(self): 866 self.results = [] 867 868 def report(self, result): 869 """Reports the result by storing it internally.""" 870 self.results.append(result) 871 872 873 if __name__ == '__main__': 874 unittest.main() 875
