1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Unit tests for the `repair` module.""" 6 7 # pylint: disable=missing-docstring 8 9 import functools 10 import logging 11 import unittest 12 13 import common 14 from autotest_lib.client.common_lib import hosts 15 from autotest_lib.client.common_lib.hosts import repair 16 from autotest_lib.server import constants 17 from autotest_lib.server.hosts import host_info 18 19 20 class _StubHost(object): 21 """ 22 Stub class to fill in the relevant methods of `Host`. 23 24 This class provides mocking and stub behaviors for `Host` for use by 25 tests within this module. The class implements only those methods 26 that `Verifier` and `RepairAction` actually use. 27 """ 28 29 def __init__(self): 30 self._record_sequence = [] 31 fake_board_name = constants.Labels.BOARD_PREFIX + 'fubar' 32 self.host_info_store = host_info.HostInfo(labels=[fake_board_name]) 33 34 35 def record(self, status_code, subdir, operation, status=''): 36 """ 37 Mock method to capture records written to `status.log`. 38 39 Each record is remembered in order to be checked for correctness 40 by individual tests later. 41 42 @param status_code As for `Host.record()`. 43 @param subdir As for `Host.record()`. 44 @param operation As for `Host.record()`. 45 @param status As for `Host.record()`. 46 """ 47 full_record = (status_code, subdir, operation, status) 48 self._record_sequence.append(full_record) 49 50 51 def get_log_records(self): 52 """ 53 Return the records logged for this fake host. 54 55 The returned list of records excludes records where the 56 `operation` parameter is not in `tagset`. 57 58 @param tagset Only include log records with these tags. 59 """ 60 return self._record_sequence 61 62 63 def reset_log_records(self): 64 """Clear our history of log records to allow re-testing.""" 65 self._record_sequence = [] 66 67 68 class _StubVerifier(hosts.Verifier): 69 """ 70 Stub implementation of `Verifier` for testing purposes. 71 72 This is a full implementation of a concrete `Verifier` subclass 73 designed to allow calling unit tests control over whether verify 74 passes or fails. 75 76 A `_StubVerifier()` will pass whenever the value of `_fail_count` 77 is non-zero. Calls to `try_repair()` (typically made by a 78 `_StubRepairAction()`) will reduce this count, eventually 79 "repairing" the verifier. 80 81 @property verify_count The number of calls made to the instance's 82 `verify()` method. 83 @property message If verification fails, the exception raised, 84 when converted to a string, will have this 85 value. 86 @property _fail_count The number of repair attempts required 87 before this verifier will succeed. A 88 non-zero value means verification will fail. 89 @property _description The value of the `description` property. 90 """ 91 92 def __init__(self, tag, deps, fail_count): 93 super(_StubVerifier, self).__init__(tag, deps) 94 self.verify_count = 0 95 self.message = 'Failing "%s" by request' % tag 96 self._fail_count = fail_count 97 self._description = 'Testing verify() for "%s"' % tag 98 self._log_record_map = { 99 r[0]: r for r in [ 100 ('GOOD', None, self._record_tag, ''), 101 ('FAIL', None, self._record_tag, self.message), 102 ] 103 } 104 105 106 def __repr__(self): 107 return '_StubVerifier(%r, %r, %r)' % ( 108 self.tag, self._dependency_list, self._fail_count) 109 110 111 def verify(self, host): 112 self.verify_count += 1 113 if self._fail_count: 114 raise hosts.AutoservVerifyError(self.message) 115 116 117 def try_repair(self): 118 """Bring ourselves one step closer to working.""" 119 if self._fail_count: 120 self._fail_count -= 1 121 122 123 def unrepair(self): 124 """Make ourselves more broken.""" 125 self._fail_count += 1 126 127 128 def get_log_record(self, status): 129 """ 130 Return a host log record for this verifier. 131 132 Calculates the arguments expected to be passed to 133 `Host.record()` by `Verifier._verify_host()` when this verifier 134 runs. The passed in `status` corresponds to the argument of the 135 same name to be passed to `Host.record()`. 136 137 @param status Status value of the log record. 138 """ 139 return self._log_record_map[status] 140 141 142 @property 143 def description(self): 144 return self._description 145 146 147 class _StubRepairFailure(Exception): 148 """Exception to be raised by `_StubRepairAction.repair()`.""" 149 pass 150 151 152 class _StubRepairAction(hosts.RepairAction): 153 """Stub implementation of `RepairAction` for testing purposes. 154 155 This is a full implementation of a concrete `RepairAction` subclass 156 designed to allow calling unit tests control over whether repair 157 passes or fails. 158 159 The behavior of `repair()` depends on the `_success` property of a 160 `_StubRepairAction`. When the property is true, repair will call 161 `try_repair()` for all triggers, and then report success. When the 162 property is false, repair reports failure. 163 164 @property repair_count The number of calls made to the instance's 165 `repair()` method. 166 @property message If repair fails, the exception raised, when 167 converted to a string, will have this value. 168 @property _success Whether repair will follow its "success" or 169 "failure" paths. 170 @property _description The value of the `description` property. 171 """ 172 173 def __init__(self, tag, deps, triggers, success): 174 super(_StubRepairAction, self).__init__(tag, deps, triggers) 175 self.repair_count = 0 176 self.message = 'Failed repair for "%s"' % tag 177 self._success = success 178 self._description = 'Testing repair for "%s"' % tag 179 self._log_record_map = { 180 r[0]: r for r in [ 181 ('START', None, self._record_tag, ''), 182 ('FAIL', None, self._record_tag, self.message), 183 ('END FAIL', None, self._record_tag, ''), 184 ('END GOOD', None, self._record_tag, ''), 185 ] 186 } 187 188 189 def __repr__(self): 190 return '_StubRepairAction(%r, %r, %r, %r)' % ( 191 self.tag, self._dependency_list, 192 self._trigger_list, self._success) 193 194 195 def repair(self, host): 196 self.repair_count += 1 197 if not self._success: 198 raise _StubRepairFailure(self.message) 199 for v in self._trigger_list: 200 v.try_repair() 201 202 203 def get_log_record(self, status): 204 """ 205 Return a host log record for this repair action. 206 207 Calculates the arguments expected to be passed to 208 `Host.record()` by `RepairAction._repair_host()` when repair 209 runs. The passed in `status` corresponds to the argument of the 210 same name to be passed to `Host.record()`. 211 212 @param status Status value of the log record. 213 """ 214 return self._log_record_map[status] 215 216 217 @property 218 def description(self): 219 return self._description 220 221 222 class _DependencyNodeTestCase(unittest.TestCase): 223 """ 224 Abstract base class for `RepairAction` and `Verifier` test cases. 225 226 This class provides `_make_verifier()` and `_make_repair_action()` 227 methods to create `_StubVerifier` and `_StubRepairAction` instances, 228 respectively, for testing. Constructed verifiers and repair actions 229 are remembered in `self.nodes`, a dictionary indexed by the tag 230 used to construct the object. 231 """ 232 233 def setUp(self): 234 logging.disable(logging.CRITICAL) 235 self._fake_host = _StubHost() 236 self.nodes = {} 237 238 239 def tearDown(self): 240 logging.disable(logging.NOTSET) 241 242 243 def _make_verifier(self, count, tag, deps): 244 """ 245 Make a `_StubVerifier` and remember it in `self.nodes`. 246 247 @param count As for the `_StubVerifer` constructor. 248 @param tag As for the `_StubVerifer` constructor. 249 @param deps As for the `_StubVerifer` constructor. 250 """ 251 verifier = _StubVerifier(tag, deps, count) 252 self.nodes[tag] = verifier 253 return verifier 254 255 256 def _make_repair_action(self, success, tag, deps, triggers): 257 """ 258 Make a `_StubRepairAction` and remember it in `self.nodes`. 259 260 @param success As for the `_StubRepairAction` constructor. 261 @param tag As for the `_StubRepairAction` constructor. 262 @param deps As for the `_StubRepairAction` constructor. 263 @param triggers As for the `_StubRepairAction` constructor. 264 """ 265 repair_action = _StubRepairAction(tag, deps, triggers, success) 266 self.nodes[tag] = repair_action 267 return repair_action 268 269 270 def _make_expected_failures(self, *verifiers): 271 """ 272 Make a set of `_DependencyFailure` objects from `verifiers`. 273 274 Return the set of `_DependencyFailure` objects that we would 275 expect to see in the `failures` attribute of an 276 `AutoservVerifyDependencyError` if all of the given verifiers 277 report failure. 278 279 @param verifiers A list of `_StubVerifier` objects that are 280 expected to fail. 281 282 @return A set of `_DependencyFailure` objects. 283 """ 284 failures = [repair._DependencyFailure(v.description, v.message) 285 for v in verifiers] 286 return set(failures) 287 288 289 def _generate_silent(self): 290 """ 291 Iterator to test different settings of the `silent` parameter. 292 293 This iterator exists to standardize testing assertions that 294 This iterator exists to standardize testing common 295 assertions about the `silent` parameter: 296 * When the parameter is true, no calls are made to the 297 `record()` method on the target host. 298 * When the parameter is false, certain expected calls are made 299 to the `record()` method on the target host. 300 301 The iterator is meant to be used like this: 302 303 for silent in self._generate_silent(): 304 # run test case that uses the silent parameter 305 self._check_log_records(silent, ... expected records ... ) 306 307 The code above will run its test case twice, once with 308 `silent=True` and once with `silent=False`. In between the 309 calls, log records are cleared. 310 311 @yields A boolean setting for `silent`. 312 """ 313 for silent in [False, True]: 314 yield silent 315 self._fake_host.reset_log_records() 316 317 318 def _check_log_records(self, silent, *record_data): 319 """ 320 Assert that log records occurred as expected. 321 322 Elements of `record_data` should be tuples of the form 323 `(tag, status)`, describing one expected log record. 324 The verifier or repair action for `tag` provides the expected 325 log record based on the status value. 326 327 The `silent` parameter is the value that was passed to the 328 verifier or repair action that did the logging. When true, 329 it indicates that no records should have been logged. 330 331 @param record_data List describing the expected record events. 332 @param silent When true, ignore `record_data` and assert 333 that nothing was logged. 334 """ 335 expected_records = [] 336 if not silent: 337 for tag, status in record_data: 338 expected_records.append( 339 self.nodes[tag].get_log_record(status)) 340 actual_records = self._fake_host.get_log_records() 341 self.assertEqual(expected_records, actual_records) 342 343 344 class VerifyTests(_DependencyNodeTestCase): 345 """ 346 Unit tests for `Verifier`. 347 348 The tests in this class test the fundamental behaviors of the 349 `Verifier` class: 350 * Results from the `verify()` method are cached; the method is 351 only called the first time that `_verify_host()` is called. 352 * The `_verify_host()` method makes the expected calls to 353 `Host.record()` for every call to the `verify()` method. 354 * When a dependency fails, the dependent verifier isn't called. 355 * Verifier calls are made in the order required by the DAG. 356 357 The test cases don't use `RepairStrategy` to build DAG structures, 358 but instead rely on custom-built DAGs. 359 """ 360 361 def _generate_verify_count(self, verifier): 362 """ 363 Iterator to force a standard sequence with calls to `_reverify()`. 364 365 This iterator exists to standardize testing two common 366 assertions: 367 * The side effects from calling `_verify_host()` only 368 happen on the first call to the method, except... 369 * Calling `_reverify()` resets a verifier so that the 370 next call to `_verify_host()` will repeat the side 371 effects. 372 373 The iterator is meant to be used like this: 374 375 for count in self._generate_verify_cases(verifier): 376 # run a verifier._verify_host() test case 377 self.assertEqual(verifier.verify_count, count) 378 self._check_log_records(silent, ... expected records ... ) 379 380 The code above will run the `_verify_host()` test case twice, 381 then call `_reverify()` to clear cached results, then re-run 382 the test case two more times. 383 384 @param verifier The verifier to be tested and reverified. 385 @yields Each iteration yields the number of times `_reverify()` 386 has been called. 387 """ 388 for i in range(1, 3): 389 for _ in range(0, 2): 390 yield i 391 verifier._reverify() 392 self._fake_host.reset_log_records() 393 394 395 def test_success(self): 396 """ 397 Test proper handling of a successful verification. 398 399 Construct and call a simple, single-node verification that will 400 pass. Assert the following: 401 * The `verify()` method is called once. 402 * The expected 'GOOD' record is logged via `Host.record()`. 403 * If `_verify_host()` is called more than once, there are no 404 visible side-effects after the first call. 405 * Calling `_reverify()` clears all cached results. 406 """ 407 for silent in self._generate_silent(): 408 verifier = self._make_verifier(0, 'pass', []) 409 for count in self._generate_verify_count(verifier): 410 verifier._verify_host(self._fake_host, silent) 411 self.assertEqual(verifier.verify_count, count) 412 self._check_log_records(silent, ('pass', 'GOOD')) 413 414 415 def test_fail(self): 416 """ 417 Test proper handling of verification failure. 418 419 Construct and call a simple, single-node verification that will 420 fail. Assert the following: 421 * The failure is reported with the actual exception raised 422 by the verifier. 423 * The `verify()` method is called once. 424 * The expected 'FAIL' record is logged via `Host.record()`. 425 * If `_verify_host()` is called more than once, there are no 426 visible side-effects after the first call. 427 * Calling `_reverify()` clears all cached results. 428 """ 429 for silent in self._generate_silent(): 430 verifier = self._make_verifier(1, 'fail', []) 431 for count in self._generate_verify_count(verifier): 432 with self.assertRaises(hosts.AutoservVerifyError) as e: 433 verifier._verify_host(self._fake_host, silent) 434 self.assertEqual(verifier.verify_count, count) 435 self.assertEqual(verifier.message, str(e.exception)) 436 self._check_log_records(silent, ('fail', 'FAIL')) 437 438 439 def test_dependency_success(self): 440 """ 441 Test proper handling of dependencies that succeed. 442 443 Construct and call a two-node verification with one node 444 dependent on the other, where both nodes will pass. Assert the 445 following: 446 * The `verify()` method for both nodes is called once. 447 * The expected 'GOOD' record is logged via `Host.record()` 448 for both nodes. 449 * If `_verify_host()` is called more than once, there are no 450 visible side-effects after the first call. 451 * Calling `_reverify()` clears all cached results. 452 """ 453 for silent in self._generate_silent(): 454 child = self._make_verifier(0, 'pass', []) 455 parent = self._make_verifier(0, 'parent', [child]) 456 for count in self._generate_verify_count(parent): 457 parent._verify_host(self._fake_host, silent) 458 self.assertEqual(parent.verify_count, count) 459 self.assertEqual(child.verify_count, count) 460 self._check_log_records(silent, 461 ('pass', 'GOOD'), 462 ('parent', 'GOOD')) 463 464 465 def test_dependency_fail(self): 466 """ 467 Test proper handling of dependencies that fail. 468 469 Construct and call a two-node verification with one node 470 dependent on the other, where the dependency will fail. Assert 471 the following: 472 * The verification exception is `AutoservVerifyDependencyError`, 473 and the exception argument is the description of the failed 474 node. 475 * The `verify()` method for the failing node is called once, 476 and for the other node, not at all. 477 * The expected 'FAIL' record is logged via `Host.record()` 478 for the single failed node. 479 * If `_verify_host()` is called more than once, there are no 480 visible side-effects after the first call. 481 * Calling `_reverify()` clears all cached results. 482 """ 483 for silent in self._generate_silent(): 484 child = self._make_verifier(1, 'fail', []) 485 parent = self._make_verifier(0, 'parent', [child]) 486 failures = self._make_expected_failures(child) 487 for count in self._generate_verify_count(parent): 488 expected_exception = hosts.AutoservVerifyDependencyError 489 with self.assertRaises(expected_exception) as e: 490 parent._verify_host(self._fake_host, silent) 491 self.assertEqual(e.exception.failures, failures) 492 self.assertEqual(child.verify_count, count) 493 self.assertEqual(parent.verify_count, 0) 494 self._check_log_records(silent, ('fail', 'FAIL')) 495 496 497 def test_two_dependencies_pass(self): 498 """ 499 Test proper handling with two passing dependencies. 500 501 Construct and call a three-node verification with one node 502 dependent on the other two, where all nodes will pass. Assert 503 the following: 504 * The `verify()` method for all nodes is called once. 505 * The expected 'GOOD' records are logged via `Host.record()` 506 for all three nodes. 507 * If `_verify_host()` is called more than once, there are no 508 visible side-effects after the first call. 509 * Calling `_reverify()` clears all cached results. 510 """ 511 for silent in self._generate_silent(): 512 left = self._make_verifier(0, 'left', []) 513 right = self._make_verifier(0, 'right', []) 514 top = self._make_verifier(0, 'top', [left, right]) 515 for count in self._generate_verify_count(top): 516 top._verify_host(self._fake_host, silent) 517 self.assertEqual(top.verify_count, count) 518 self.assertEqual(left.verify_count, count) 519 self.assertEqual(right.verify_count, count) 520 self._check_log_records(silent, 521 ('left', 'GOOD'), 522 ('right', 'GOOD'), 523 ('top', 'GOOD')) 524 525 526 def test_two_dependencies_fail(self): 527 """ 528 Test proper handling with two failing dependencies. 529 530 Construct and call a three-node verification with one node 531 dependent on the other two, where both dependencies will fail. 532 Assert the following: 533 * The verification exception is `AutoservVerifyDependencyError`, 534 and the exception argument has the descriptions of both the 535 failed nodes. 536 * The `verify()` method for each failing node is called once, 537 and for the parent node not at all. 538 * The expected 'FAIL' records are logged via `Host.record()` 539 for the failing nodes. 540 * If `_verify_host()` is called more than once, there are no 541 visible side-effects after the first call. 542 * Calling `_reverify()` clears all cached results. 543 """ 544 for silent in self._generate_silent(): 545 left = self._make_verifier(1, 'left', []) 546 right = self._make_verifier(1, 'right', []) 547 top = self._make_verifier(0, 'top', [left, right]) 548 failures = self._make_expected_failures(left, right) 549 for count in self._generate_verify_count(top): 550 expected_exception = hosts.AutoservVerifyDependencyError 551 with self.assertRaises(expected_exception) as e: 552 top._verify_host(self._fake_host, silent) 553 self.assertEqual(e.exception.failures, failures) 554 self.assertEqual(top.verify_count, 0) 555 self.assertEqual(left.verify_count, count) 556 self.assertEqual(right.verify_count, count) 557 self._check_log_records(silent, 558 ('left', 'FAIL'), 559 ('right', 'FAIL')) 560 561 562 def test_two_dependencies_mixed(self): 563 """ 564 Test proper handling with mixed dependencies. 565 566 Construct and call a three-node verification with one node 567 dependent on the other two, where one dependency will pass, 568 and one will fail. Assert the following: 569 * The verification exception is `AutoservVerifyDependencyError`, 570 and the exception argument has the descriptions of the 571 single failed node. 572 * The `verify()` method for each dependency is called once, 573 and for the parent node not at all. 574 * The expected 'GOOD' and 'FAIL' records are logged via 575 `Host.record()` for the dependencies. 576 * If `_verify_host()` is called more than once, there are no 577 visible side-effects after the first call. 578 * Calling `_reverify()` clears all cached results. 579 """ 580 for silent in self._generate_silent(): 581 left = self._make_verifier(1, 'left', []) 582 right = self._make_verifier(0, 'right', []) 583 top = self._make_verifier(0, 'top', [left, right]) 584 failures = self._make_expected_failures(left) 585 for count in self._generate_verify_count(top): 586 expected_exception = hosts.AutoservVerifyDependencyError 587 with self.assertRaises(expected_exception) as e: 588 top._verify_host(self._fake_host, silent) 589 self.assertEqual(e.exception.failures, failures) 590 self.assertEqual(top.verify_count, 0) 591 self.assertEqual(left.verify_count, count) 592 self.assertEqual(right.verify_count, count) 593 self._check_log_records(silent, 594 ('left', 'FAIL'), 595 ('right', 'GOOD')) 596 597 598 def test_diamond_pass(self): 599 """ 600 Test a "diamond" structure DAG with all nodes passing. 601 602 Construct and call a "diamond" structure DAG where all nodes 603 will pass: 604 605 TOP 606 / \ 607 LEFT RIGHT 608 \ / 609 BOTTOM 610 611 Assert the following: 612 * The `verify()` method for all nodes is called once. 613 * The expected 'GOOD' records are logged via `Host.record()` 614 for all nodes. 615 * If `_verify_host()` is called more than once, there are no 616 visible side-effects after the first call. 617 * Calling `_reverify()` clears all cached results. 618 """ 619 for silent in self._generate_silent(): 620 bottom = self._make_verifier(0, 'bottom', []) 621 left = self._make_verifier(0, 'left', [bottom]) 622 right = self._make_verifier(0, 'right', [bottom]) 623 top = self._make_verifier(0, 'top', [left, right]) 624 for count in self._generate_verify_count(top): 625 top._verify_host(self._fake_host, silent) 626 self.assertEqual(top.verify_count, count) 627 self.assertEqual(left.verify_count, count) 628 self.assertEqual(right.verify_count, count) 629 self.assertEqual(bottom.verify_count, count) 630 self._check_log_records(silent, 631 ('bottom', 'GOOD'), 632 ('left', 'GOOD'), 633 ('right', 'GOOD'), 634 ('top', 'GOOD')) 635 636 637 def test_diamond_fail(self): 638 """ 639 Test a "diamond" structure DAG with the bottom node failing. 640 641 Construct and call a "diamond" structure DAG where the bottom 642 node will fail: 643 644 TOP 645 / \ 646 LEFT RIGHT 647 \ / 648 BOTTOM 649 650 Assert the following: 651 * The verification exception is `AutoservVerifyDependencyError`, 652 and the exception argument has the description of the 653 "bottom" node. 654 * The `verify()` method for the "bottom" node is called once, 655 and for the other nodes not at all. 656 * The expected 'FAIL' record is logged via `Host.record()` 657 for the "bottom" node. 658 * If `_verify_host()` is called more than once, there are no 659 visible side-effects after the first call. 660 * Calling `_reverify()` clears all cached results. 661 """ 662 for silent in self._generate_silent(): 663 bottom = self._make_verifier(1, 'bottom', []) 664 left = self._make_verifier(0, 'left', [bottom]) 665 right = self._make_verifier(0, 'right', [bottom]) 666 top = self._make_verifier(0, 'top', [left, right]) 667 failures = self._make_expected_failures(bottom) 668 for count in self._generate_verify_count(top): 669 expected_exception = hosts.AutoservVerifyDependencyError 670 with self.assertRaises(expected_exception) as e: 671 top._verify_host(self._fake_host, silent) 672 self.assertEqual(e.exception.failures, failures) 673 self.assertEqual(top.verify_count, 0) 674 self.assertEqual(left.verify_count, 0) 675 self.assertEqual(right.verify_count, 0) 676 self.assertEqual(bottom.verify_count, count) 677 self._check_log_records(silent, ('bottom', 'FAIL')) 678 679 680 class RepairActionTests(_DependencyNodeTestCase): 681 """ 682 Unit tests for `RepairAction`. 683 684 The tests in this class test the fundamental behaviors of the 685 `RepairAction` class: 686 * Repair doesn't run unless all dependencies pass. 687 * Repair doesn't run unless at least one trigger fails. 688 * Repair reports the expected value of `status` for metrics. 689 * The `_repair_host()` method makes the expected calls to 690 `Host.record()` for every call to the `repair()` method. 691 692 The test cases don't use `RepairStrategy` to build repair 693 graphs, but instead rely on custom-built structures. 694 """ 695 696 def test_repair_not_triggered(self): 697 """ 698 Test a repair that doesn't trigger. 699 700 Construct and call a repair action with a verification trigger 701 that passes. Assert the following: 702 * The `verify()` method for the trigger is called. 703 * The `repair()` method is not called. 704 * The repair action's `status` field is 'untriggered'. 705 * The verifier logs the expected 'GOOD' message with 706 `Host.record()`. 707 * The repair action logs no messages with `Host.record()`. 708 """ 709 for silent in self._generate_silent(): 710 verifier = self._make_verifier(0, 'check', []) 711 repair_action = self._make_repair_action(True, 'unneeded', 712 [], [verifier]) 713 repair_action._repair_host(self._fake_host, silent) 714 self.assertEqual(verifier.verify_count, 1) 715 self.assertEqual(repair_action.repair_count, 0) 716 self.assertEqual(repair_action.status, 'untriggered') 717 self._check_log_records(silent, ('check', 'GOOD')) 718 719 720 def test_repair_fails(self): 721 """ 722 Test a repair that triggers and fails. 723 724 Construct and call a repair action with a verification trigger 725 that fails. The repair fails by raising `_StubRepairFailure`. 726 Assert the following: 727 * The repair action fails with the `_StubRepairFailure` raised 728 by `repair()`. 729 * The `verify()` method for the trigger is called once. 730 * The `repair()` method is called once. 731 * The repair action's `status` field is 'failed-action'. 732 * The expected 'START', 'FAIL', and 'END FAIL' messages are 733 logged with `Host.record()` for the failed verifier and the 734 failed repair. 735 """ 736 for silent in self._generate_silent(): 737 verifier = self._make_verifier(1, 'fail', []) 738 repair_action = self._make_repair_action(False, 'nofix', 739 [], [verifier]) 740 with self.assertRaises(_StubRepairFailure) as e: 741 repair_action._repair_host(self._fake_host, silent) 742 self.assertEqual(repair_action.message, str(e.exception)) 743 self.assertEqual(verifier.verify_count, 1) 744 self.assertEqual(repair_action.repair_count, 1) 745 self.assertEqual(repair_action.status, 'failed-action') 746 self._check_log_records(silent, 747 ('fail', 'FAIL'), 748 ('nofix', 'START'), 749 ('nofix', 'FAIL'), 750 ('nofix', 'END FAIL')) 751 752 753 def test_repair_success(self): 754 """ 755 Test a repair that fixes its trigger. 756 757 Construct and call a repair action that raises no exceptions, 758 using a repair trigger that fails first, then passes after 759 repair. Assert the following: 760 * The `repair()` method is called once. 761 * The trigger's `verify()` method is called twice. 762 * The repair action's `status` field is 'repaired'. 763 * The expected 'START', 'FAIL', 'GOOD', and 'END GOOD' 764 messages are logged with `Host.record()` for the verifier 765 and the repair. 766 """ 767 for silent in self._generate_silent(): 768 verifier = self._make_verifier(1, 'fail', []) 769 repair_action = self._make_repair_action(True, 'fix', 770 [], [verifier]) 771 repair_action._repair_host(self._fake_host, silent) 772 self.assertEqual(repair_action.repair_count, 1) 773 self.assertEqual(verifier.verify_count, 2) 774 self.assertEqual(repair_action.status, 'repaired') 775 self._check_log_records(silent, 776 ('fail', 'FAIL'), 777 ('fix', 'START'), 778 ('fail', 'GOOD'), 779 ('fix', 'END GOOD')) 780 781 782 def test_repair_noop(self): 783 """ 784 Test a repair that doesn't fix a failing trigger. 785 786 Construct and call a repair action with a trigger that fails. 787 The repair action raises no exceptions, and after repair, the 788 trigger still fails. Assert the following: 789 * The `_repair_host()` call fails with `AutoservRepairError`. 790 * The `repair()` method is called once. 791 * The trigger's `verify()` method is called twice. 792 * The repair action's `status` field is 'failed-trigger'. 793 * The expected 'START', 'FAIL', and 'END FAIL' messages are 794 logged with `Host.record()` for the verifier and the repair. 795 """ 796 for silent in self._generate_silent(): 797 verifier = self._make_verifier(2, 'fail', []) 798 repair_action = self._make_repair_action(True, 'nofix', 799 [], [verifier]) 800 with self.assertRaises(hosts.AutoservRepairError) as e: 801 repair_action._repair_host(self._fake_host, silent) 802 self.assertEqual(repair_action.repair_count, 1) 803 self.assertEqual(verifier.verify_count, 2) 804 self.assertEqual(repair_action.status, 'failed-trigger') 805 self._check_log_records(silent, 806 ('fail', 'FAIL'), 807 ('nofix', 'START'), 808 ('fail', 'FAIL'), 809 ('nofix', 'END FAIL')) 810 811 812 def test_dependency_pass(self): 813 """ 814 Test proper handling of repair dependencies that pass. 815 816 Construct and call a repair action with a dependency and a 817 trigger. The dependency will pass and the trigger will fail and 818 be repaired. Assert the following: 819 * Repair passes. 820 * The `verify()` method for the dependency is called once. 821 * The `verify()` method for the trigger is called twice. 822 * The `repair()` method is called once. 823 * The repair action's `status` field is 'repaired'. 824 * The expected records are logged via `Host.record()` 825 for the successful dependency, the failed trigger, and 826 the successful repair. 827 """ 828 for silent in self._generate_silent(): 829 dep = self._make_verifier(0, 'dep', []) 830 trigger = self._make_verifier(1, 'trig', []) 831 repair = self._make_repair_action(True, 'fixit', 832 [dep], [trigger]) 833 repair._repair_host(self._fake_host, silent) 834 self.assertEqual(dep.verify_count, 1) 835 self.assertEqual(trigger.verify_count, 2) 836 self.assertEqual(repair.repair_count, 1) 837 self.assertEqual(repair.status, 'repaired') 838 self._check_log_records(silent, 839 ('dep', 'GOOD'), 840 ('trig', 'FAIL'), 841 ('fixit', 'START'), 842 ('trig', 'GOOD'), 843 ('fixit', 'END GOOD')) 844 845 846 def test_dependency_fail(self): 847 """ 848 Test proper handling of repair dependencies that fail. 849 850 Construct and call a repair action with a dependency and a 851 trigger, both of which fail. Assert the following: 852 * Repair fails with `AutoservVerifyDependencyError`, 853 and the exception argument is the description of the failed 854 dependency. 855 * The `verify()` method for the failing dependency is called 856 once. 857 * The trigger and the repair action aren't invoked at all. 858 * The repair action's `status` field is 'blocked'. 859 * The expected 'FAIL' record is logged via `Host.record()` 860 for the single failed dependency. 861 """ 862 for silent in self._generate_silent(): 863 dep = self._make_verifier(1, 'dep', []) 864 trigger = self._make_verifier(1, 'trig', []) 865 repair = self._make_repair_action(True, 'fixit', 866 [dep], [trigger]) 867 expected_exception = hosts.AutoservVerifyDependencyError 868 with self.assertRaises(expected_exception) as e: 869 repair._repair_host(self._fake_host, silent) 870 self.assertEqual(e.exception.failures, 871 self._make_expected_failures(dep)) 872 self.assertEqual(dep.verify_count, 1) 873 self.assertEqual(trigger.verify_count, 0) 874 self.assertEqual(repair.repair_count, 0) 875 self.assertEqual(repair.status, 'blocked') 876 self._check_log_records(silent, ('dep', 'FAIL')) 877 878 879 class _RepairStrategyTestCase(_DependencyNodeTestCase): 880 """Shared base class for testing `RepairStrategy` methods.""" 881 882 def _make_verify_data(self, *input_data): 883 """ 884 Create `verify_data` for the `RepairStrategy` constructor. 885 886 `RepairStrategy` expects `verify_data` as a list of tuples 887 of the form `(constructor, tag, deps)`. Each item in 888 `input_data` is a tuple of the form `(tag, count, deps)` that 889 creates one entry in the returned list of `verify_data` tuples 890 as follows: 891 * `count` is used to create a constructor function that calls 892 `self._make_verifier()` with that value plus plus the 893 arguments provided by the `RepairStrategy` constructor. 894 * `tag` and `deps` will be passed as-is to the `RepairStrategy` 895 constructor. 896 897 @param input_data A list of tuples, each representing one 898 tuple in the `verify_data` list. 899 @return A list suitable to be the `verify_data` parameter for 900 the `RepairStrategy` constructor. 901 """ 902 strategy_data = [] 903 for tag, count, deps in input_data: 904 construct = functools.partial(self._make_verifier, count) 905 strategy_data.append((construct, tag, deps)) 906 return strategy_data 907 908 909 def _make_repair_data(self, *input_data): 910 """ 911 Create `repair_data` for the `RepairStrategy` constructor. 912 913 `RepairStrategy` expects `repair_data` as a list of tuples 914 of the form `(constructor, tag, deps, triggers)`. Each item in 915 `input_data` is a tuple of the form `(tag, success, deps, triggers)` 916 that creates one entry in the returned list of `repair_data` 917 tuples as follows: 918 * `success` is used to create a constructor function that calls 919 `self._make_verifier()` with that value plus plus the 920 arguments provided by the `RepairStrategy` constructor. 921 * `tag`, `deps`, and `triggers` will be passed as-is to the 922 `RepairStrategy` constructor. 923 924 @param input_data A list of tuples, each representing one 925 tuple in the `repair_data` list. 926 @return A list suitable to be the `repair_data` parameter for 927 the `RepairStrategy` constructor. 928 """ 929 strategy_data = [] 930 for tag, success, deps, triggers in input_data: 931 construct = functools.partial(self._make_repair_action, success) 932 strategy_data.append((construct, tag, deps, triggers)) 933 return strategy_data 934 935 936 def _make_strategy(self, verify_input, repair_input): 937 """ 938 Create a `RepairStrategy` from the given arguments. 939 940 @param verify_input As for `input_data` in 941 `_make_verify_data()`. 942 @param repair_input As for `input_data` in 943 `_make_repair_data()`. 944 """ 945 verify_data = self._make_verify_data(*verify_input) 946 repair_data = self._make_repair_data(*repair_input) 947 return hosts.RepairStrategy(verify_data, repair_data) 948 949 def _check_silent_records(self, silent): 950 """ 951 Check that logging honored the `silent` parameter. 952 953 Asserts that logging with `Host.record()` occurred (or did not 954 occur) in accordance with the value of `silent`. 955 956 This method only asserts the presence or absence of log records. 957 Coverage for the contents of the log records is handled in other 958 test cases. 959 960 @param silent When true, there should be no log records; 961 otherwise there should be records present. 962 """ 963 log_records = self._fake_host.get_log_records() 964 if silent: 965 self.assertEqual(log_records, []) 966 else: 967 self.assertNotEqual(log_records, []) 968 969 970 class RepairStrategyVerifyTests(_RepairStrategyTestCase): 971 """ 972 Unit tests for `RepairStrategy.verify()`. 973 974 These unit tests focus on verifying that the `RepairStrategy` 975 constructor creates the expected DAG structure from given 976 `verify_data`. Functional testing here is mainly confined to 977 asserting that `RepairStrategy.verify()` properly distinguishes 978 success from failure. Testing the behavior of specific DAG 979 structures is left to tests in `VerifyTests`. 980 """ 981 982 def test_single_node(self): 983 """ 984 Test construction of a single-node verification DAG. 985 986 Assert that the structure looks like this: 987 988 Root Node -> Main Node 989 """ 990 verify_data = self._make_verify_data(('main', 0, ())) 991 strategy = hosts.RepairStrategy(verify_data, []) 992 verifier = self.nodes['main'] 993 self.assertEqual( 994 strategy._verify_root._dependency_list, 995 [verifier]) 996 self.assertEqual(verifier._dependency_list, []) 997 998 999 def test_single_dependency(self): 1000 """ 1001 Test construction of a two-node dependency chain. 1002 1003 Assert that the structure looks like this: 1004 1005 Root Node -> Parent Node -> Child Node 1006 """ 1007 verify_data = self._make_verify_data( 1008 ('child', 0, ()), 1009 ('parent', 0, ('child',))) 1010 strategy = hosts.RepairStrategy(verify_data, []) 1011 parent = self.nodes['parent'] 1012 child = self.nodes['child'] 1013 self.assertEqual( 1014 strategy._verify_root._dependency_list, [parent]) 1015 self.assertEqual( 1016 parent._dependency_list, [child]) 1017 self.assertEqual( 1018 child._dependency_list, []) 1019 1020 1021 def test_two_nodes_and_dependency(self): 1022 """ 1023 Test construction of two nodes with a shared dependency. 1024 1025 Assert that the structure looks like this: 1026 1027 Root Node -> Left Node ---\ 1028 \ -> Bottom Node 1029 -> Right Node / 1030 """ 1031 verify_data = self._make_verify_data( 1032 ('bottom', 0, ()), 1033 ('left', 0, ('bottom',)), 1034 ('right', 0, ('bottom',))) 1035 strategy = hosts.RepairStrategy(verify_data, []) 1036 bottom = self.nodes['bottom'] 1037 left = self.nodes['left'] 1038 right = self.nodes['right'] 1039 self.assertEqual( 1040 strategy._verify_root._dependency_list, 1041 [left, right]) 1042 self.assertEqual(left._dependency_list, [bottom]) 1043 self.assertEqual(right._dependency_list, [bottom]) 1044 self.assertEqual(bottom._dependency_list, []) 1045 1046 1047 def test_three_nodes(self): 1048 """ 1049 Test construction of three nodes with no dependencies. 1050 1051 Assert that the structure looks like this: 1052 1053 -> Node One 1054 / 1055 Root Node -> Node Two 1056 \ 1057 -> Node Three 1058 1059 N.B. This test exists to enforce ordering expectations of 1060 root-level DAG nodes. Three nodes are used to make it unlikely 1061 that randomly ordered roots will match expectations. 1062 """ 1063 verify_data = self._make_verify_data( 1064 ('one', 0, ()), 1065 ('two', 0, ()), 1066 ('three', 0, ())) 1067 strategy = hosts.RepairStrategy(verify_data, []) 1068 one = self.nodes['one'] 1069 two = self.nodes['two'] 1070 three = self.nodes['three'] 1071 self.assertEqual( 1072 strategy._verify_root._dependency_list, 1073 [one, two, three]) 1074 self.assertEqual(one._dependency_list, []) 1075 self.assertEqual(two._dependency_list, []) 1076 self.assertEqual(three._dependency_list, []) 1077 1078 1079 def test_verify(self): 1080 """ 1081 Test behavior of the `verify()` method. 1082 1083 Build a `RepairStrategy` with a single verifier. Assert the 1084 following: 1085 * If the verifier passes, `verify()` passes. 1086 * If the verifier fails, `verify()` fails. 1087 * The verifier is reinvoked with every call to `verify()`; 1088 cached results are not re-used. 1089 """ 1090 verify_data = self._make_verify_data(('tester', 0, ())) 1091 strategy = hosts.RepairStrategy(verify_data, []) 1092 verifier = self.nodes['tester'] 1093 count = 0 1094 for silent in self._generate_silent(): 1095 for i in range(0, 2): 1096 for j in range(0, 2): 1097 strategy.verify(self._fake_host, silent) 1098 self._check_silent_records(silent) 1099 count += 1 1100 self.assertEqual(verifier.verify_count, count) 1101 verifier.unrepair() 1102 for j in range(0, 2): 1103 with self.assertRaises(Exception) as e: 1104 strategy.verify(self._fake_host, silent) 1105 self._check_silent_records(silent) 1106 count += 1 1107 self.assertEqual(verifier.verify_count, count) 1108 verifier.try_repair() 1109 1110 1111 class RepairStrategyRepairTests(_RepairStrategyTestCase): 1112 """ 1113 Unit tests for `RepairStrategy.repair()`. 1114 1115 These unit tests focus on verifying that the `RepairStrategy` 1116 constructor creates the expected repair list from given 1117 `repair_data`. Functional testing here is confined to asserting 1118 that `RepairStrategy.repair()` properly distinguishes success from 1119 failure. Testing the behavior of specific repair structures is left 1120 to tests in `RepairActionTests`. 1121 """ 1122 1123 def _check_common_trigger(self, strategy, repair_tags, triggers): 1124 self.assertEqual(strategy._repair_actions, 1125 [self.nodes[tag] for tag in repair_tags]) 1126 for tag in repair_tags: 1127 self.assertEqual(self.nodes[tag]._trigger_list, 1128 triggers) 1129 self.assertEqual(self.nodes[tag]._dependency_list, []) 1130 1131 1132 def test_single_repair_with_trigger(self): 1133 """ 1134 Test constructing a strategy with a single repair trigger. 1135 1136 Build a `RepairStrategy` with a single repair action and a 1137 single trigger. Assert that the trigger graph looks like this: 1138 1139 Repair -> Trigger 1140 1141 Assert that there are no repair dependencies. 1142 """ 1143 verify_input = (('base', 0, ()),) 1144 repair_input = (('fixit', True, (), ('base',)),) 1145 strategy = self._make_strategy(verify_input, repair_input) 1146 self._check_common_trigger(strategy, 1147 ['fixit'], 1148 [self.nodes['base']]) 1149 1150 1151 def test_repair_with_root_trigger(self): 1152 """ 1153 Test construction of a repair triggering on the root verifier. 1154 1155 Build a `RepairStrategy` with a single repair action that 1156 triggers on the root verifier. Assert that the trigger graph 1157 looks like this: 1158 1159 Repair -> Root Verifier 1160 1161 Assert that there are no repair dependencies. 1162 """ 1163 root_tag = hosts.RepairStrategy.ROOT_TAG 1164 repair_input = (('fixit', True, (), (root_tag,)),) 1165 strategy = self._make_strategy([], repair_input) 1166 self._check_common_trigger(strategy, 1167 ['fixit'], 1168 [strategy._verify_root]) 1169 1170 1171 def test_three_repairs(self): 1172 """ 1173 Test constructing a strategy with three repair actions. 1174 1175 Build a `RepairStrategy` with a three repair actions sharing a 1176 single trigger. Assert that the trigger graph looks like this: 1177 1178 Repair A -> Trigger 1179 Repair B -> Trigger 1180 Repair C -> Trigger 1181 1182 Assert that there are no repair dependencies. 1183 1184 N.B. This test exists to enforce ordering expectations of 1185 repair nodes. Three nodes are used to make it unlikely that 1186 randomly ordered actions will match expectations. 1187 """ 1188 verify_input = (('base', 0, ()),) 1189 repair_tags = ['a', 'b', 'c'] 1190 repair_input = ( 1191 (tag, True, (), ('base',)) for tag in repair_tags) 1192 strategy = self._make_strategy(verify_input, repair_input) 1193 self._check_common_trigger(strategy, 1194 repair_tags, 1195 [self.nodes['base']]) 1196 1197 1198 def test_repair_dependency(self): 1199 """ 1200 Test construction of a repair with a dependency. 1201 1202 Build a `RepairStrategy` with a single repair action that 1203 depends on a single verifier. Assert that the dependency graph 1204 looks like this: 1205 1206 Repair -> Verifier 1207 1208 Assert that there are no repair triggers. 1209 """ 1210 verify_input = (('base', 0, ()),) 1211 repair_input = (('fixit', True, ('base',), ()),) 1212 strategy = self._make_strategy(verify_input, repair_input) 1213 self.assertEqual(strategy._repair_actions, 1214 [self.nodes['fixit']]) 1215 self.assertEqual(self.nodes['fixit']._trigger_list, []) 1216 self.assertEqual(self.nodes['fixit']._dependency_list, 1217 [self.nodes['base']]) 1218 1219 1220 def _check_repair_failure(self, strategy, silent): 1221 """ 1222 Check the effects of a call to `repair()` that fails. 1223 1224 For the given strategy object, call the `repair()` method; the 1225 call is expected to fail and all repair actions are expected to 1226 trigger. 1227 1228 Assert the following: 1229 * The call raises an exception. 1230 * For each repair action in the strategy, its `repair()` 1231 method is called exactly once. 1232 1233 @param strategy The strategy to be tested. 1234 """ 1235 action_counts = [(a, a.repair_count) 1236 for a in strategy._repair_actions] 1237 with self.assertRaises(Exception) as e: 1238 strategy.repair(self._fake_host, silent) 1239 self._check_silent_records(silent) 1240 for action, count in action_counts: 1241 self.assertEqual(action.repair_count, count + 1) 1242 1243 1244 def _check_repair_success(self, strategy, silent): 1245 """ 1246 Check the effects of a call to `repair()` that succeeds. 1247 1248 For the given strategy object, call the `repair()` method; the 1249 call is expected to succeed without raising an exception and all 1250 repair actions are expected to trigger. 1251 1252 Assert that for each repair action in the strategy, its 1253 `repair()` method is called exactly once. 1254 1255 @param strategy The strategy to be tested. 1256 """ 1257 action_counts = [(a, a.repair_count) 1258 for a in strategy._repair_actions] 1259 strategy.repair(self._fake_host, silent) 1260 self._check_silent_records(silent) 1261 for action, count in action_counts: 1262 self.assertEqual(action.repair_count, count + 1) 1263 1264 1265 def test_repair(self): 1266 """ 1267 Test behavior of the `repair()` method. 1268 1269 Build a `RepairStrategy` with two repair actions each depending 1270 on its own verifier. Set up calls to `repair()` for each of 1271 the following conditions: 1272 * Both repair actions trigger and fail. 1273 * Both repair actions trigger and succeed. 1274 * Both repair actions trigger; the first one fails, but the 1275 second one succeeds. 1276 * Both repair actions trigger; the first one succeeds, but the 1277 second one fails. 1278 1279 Assert the following: 1280 * When both repair actions succeed, `repair()` succeeds. 1281 * When either repair action fails, `repair()` fails. 1282 * After each call to the strategy's `repair()` method, each 1283 repair action triggered exactly once. 1284 """ 1285 verify_input = (('a', 2, ()), ('b', 2, ())) 1286 repair_input = (('afix', True, (), ('a',)), 1287 ('bfix', True, (), ('b',))) 1288 strategy = self._make_strategy(verify_input, repair_input) 1289 1290 for silent in self._generate_silent(): 1291 # call where both 'afix' and 'bfix' fail 1292 self._check_repair_failure(strategy, silent) 1293 # repair counts are now 1 for both verifiers 1294 1295 # call where both 'afix' and 'bfix' succeed 1296 self._check_repair_success(strategy, silent) 1297 # repair counts are now 0 for both verifiers 1298 1299 # call where 'afix' fails and 'bfix' succeeds 1300 for tag in ['a', 'a', 'b']: 1301 self.nodes[tag].unrepair() 1302 self._check_repair_failure(strategy, silent) 1303 # 'a' repair count is 1; 'b' count is 0 1304 1305 # call where 'afix' succeeds and 'bfix' fails 1306 for tag in ['b', 'b']: 1307 self.nodes[tag].unrepair() 1308 self._check_repair_failure(strategy, silent) 1309 # 'a' repair count is 0; 'b' count is 1 1310 1311 for tag in ['a', 'a', 'b']: 1312 self.nodes[tag].unrepair() 1313 # repair counts are now 2 for both verifiers 1314 1315 1316 if __name__ == '__main__': 1317 unittest.main() 1318
