1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """ 6 Framework for host verification and repair in Autotest. 7 8 The framework provides implementation code in support of `Host.verify()` 9 and `Host.repair()` used in Verify and Repair special tasks. 10 11 The framework consists of these classes: 12 * `Verifier`: A class representing a single verification check. 13 * `RepairAction`: A class representing a repair operation that can fix 14 a failed verification check. 15 * `RepairStrategy`: A class for organizing a collection of `Verifier` 16 and `RepairAction` instances, and invoking them in order. 17 18 Individual operations during verification and repair are handled by 19 instances of `Verifier` and `RepairAction`. `Verifier` objects are 20 meant to test for specific conditions that may cause tests to fail. 21 `RepairAction` objects provide operations designed to fix one or 22 more failures identified by a `Verifier` object. 23 """ 24 25 import collections 26 import logging 27 28 import common 29 from autotest_lib.client.common_lib import error 30 31 try: 32 from chromite.lib import metrics 33 except ImportError: 34 from autotest_lib.client.bin.utils import metrics_mock as metrics 35 36 37 class AutoservVerifyError(error.AutoservError): 38 """ 39 Generic Exception for failures from `Verifier` objects. 40 41 Instances of this exception can be raised when a `verify()` 42 method fails, if no more specific exception is available. 43 """ 44 pass 45 46 47 _DependencyFailure = collections.namedtuple( 48 '_DependencyFailure', ('dependency', 'error')) 49 50 51 class AutoservVerifyDependencyError(error.AutoservError): 52 """ 53 Exception raised for failures in dependencies. 54 55 This exception is used to distinguish an original failure from a 56 failure being passed back from a verification dependency. That is, 57 if 'B' depends on 'A', and 'A' fails, 'B' will raise this exception 58 to signal that the original failure is further down the dependency 59 chain. 60 61 The `failures` argument to the constructor for this class is a set 62 of instances of `_DependencyFailure`, each corresponding to one 63 failed dependency: 64 * The `dependency` attribute of each failure is the description 65 of the failed dependency. 66 * The `error` attribute of each failure is the string value of 67 the exception from the failed dependency. 68 69 Multiple methods in this module recognize and handle this exception 70 specially. 71 72 @property failures Set of failures passed to the constructor. 73 @property _node Instance of `_DependencyNode` reporting the 74 failed dependencies. 75 """ 76 77 def __init__(self, node, failures): 78 """ 79 Constructor for `AutoservVerifyDependencyError`. 80 81 @param node Instance of _DependencyNode reporting the 82 failed dependencies. 83 @param failures List of failure tuples as described above. 84 """ 85 super(AutoservVerifyDependencyError, self).__init__( 86 '\n'.join([f.error for f in failures])) 87 self.failures = failures 88 self._node = node 89 90 def log_dependencies(self, action, deps): 91 """ 92 Log an `AutoservVerifyDependencyError`. 93 94 This writes a short summary of the dependency failures captured 95 in this exception, using standard Python logging. 96 97 The passed in `action` string plus `self._node.description` 98 are logged at INFO level. The `action` argument should 99 introduce or describe an action relative to `self._node`. 100 101 The passed in `deps` string and the description of each failed 102 dependency in `self` are be logged at DEBUG level. The `deps` 103 argument is used to introduce the various failed dependencies. 104 105 @param action A string mentioning the action being logged 106 relative to `self._node`. 107 @param deps A string introducing the dependencies that 108 failed. 109 """ 110 logging.info('%s: %s', action, self._node.description) 111 logging.debug('%s:', deps) 112 for failure in self.failures: 113 logging.debug(' %s', failure.dependency) 114 115 116 class AutoservRepairError(error.AutoservError): 117 """ 118 Generic Exception for failures from `RepairAction` objects. 119 120 Instances of this exception can be raised when a `repair()` 121 method fails, if no more specific exception is available. 122 """ 123 pass 124 125 126 class _DependencyNode(object): 127 """ 128 An object that can depend on verifiers. 129 130 Both repair and verify operations have the notion of dependencies 131 that must pass before the operation proceeds. This class captures 132 the shared behaviors required by both classes. 133 134 @property tag Short identifier to be used in logging. 135 @property description Text summary of this node's action, to be 136 used in debug logs. 137 @property _dependency_list Dependency pre-requisites. 138 """ 139 140 def __init__(self, tag, record_type, dependencies): 141 self._dependency_list = dependencies 142 self._tag = tag 143 self._record_tag = record_type + '.' + tag 144 145 def _record(self, host, silent, status_code, *record_args): 146 """ 147 Log a status record for `host`. 148 149 Call `host.record()` using the given status_code, and 150 operation tag `self._record_tag`, plus any extra arguments in 151 `record_args`. Do nothing if `silent` is a true value. 152 153 @param host Host which will record the status record. 154 @param silent Don't record the event if this is a true 155 value. 156 @param status_code Value for the `status_code` parameter to 157 `host.record()`. 158 @param record_args Additional arguments to pass to 159 `host.record()`. 160 """ 161 if not silent: 162 host.record(status_code, None, self._record_tag, 163 *record_args) 164 165 def _record_good(self, host, silent): 166 """Log a 'GOOD' status line. 167 168 @param host Host which will record the status record. 169 @param silent Don't record the event if this is a true 170 value. 171 """ 172 self._record(host, silent, 'GOOD') 173 174 def _record_fail(self, host, silent, exc): 175 """Log a 'FAIL' status line. 176 177 @param host Host which will record the status record. 178 @param silent Don't record the event if this is a true 179 value. 180 @param exc Exception describing the cause of failure. 181 """ 182 self._record(host, silent, 'FAIL', str(exc)) 183 184 def _verify_list(self, host, verifiers, silent): 185 """ 186 Test a list of verifiers against a given host. 187 188 This invokes `_verify_host()` on every verifier in the given 189 list. If any verifier in the transitive closure of dependencies 190 in the list fails, an `AutoservVerifyDependencyError` is raised 191 containing the description of each failed verifier. Only 192 original failures are reported; verifiers that don't run due 193 to a failed dependency are omitted. 194 195 By design, original failures are logged once in `_verify_host()` 196 when `verify()` originally fails. The additional data gathered 197 here is for the debug logs to indicate why a subsequent 198 operation never ran. 199 200 @param host The host to be tested against the verifiers. 201 @param verifiers List of verifiers to be checked. 202 @param silent If true, don't log host status records. 203 204 @raises AutoservVerifyDependencyError Raised when at least 205 one verifier in the list has failed. 206 """ 207 failures = set() 208 for v in verifiers: 209 try: 210 v._verify_host(host, silent) 211 except AutoservVerifyDependencyError as e: 212 failures.update(e.failures) 213 except Exception as e: 214 failures.add(_DependencyFailure(v.description, str(e))) 215 if failures: 216 raise AutoservVerifyDependencyError(self, failures) 217 218 def _verify_dependencies(self, host, silent): 219 """ 220 Verify that all of this node's dependencies pass for a host. 221 222 @param host The host to be verified. 223 @param silent If true, don't log host status records. 224 """ 225 try: 226 self._verify_list(host, self._dependency_list, silent) 227 except AutoservVerifyDependencyError as e: 228 e.log_dependencies( 229 'Skipping this operation', 230 'The following dependencies failed') 231 raise 232 233 @property 234 def tag(self): 235 """ 236 Tag for use in logging status records. 237 238 This is a property with a short string used to identify the node 239 in the 'status.log' file and during node construction. The tag 240 should contain only letters, digits, and '_' characters. This 241 tag is not used alone, but is combined with other identifiers, 242 based on the operation being logged. 243 244 @return A short identifier-like string. 245 """ 246 return self._tag 247 248 @property 249 def description(self): 250 """ 251 Text description of this node for log messages. 252 253 This string will be logged with failures, and should describe 254 the condition required for success. 255 256 N.B. Subclasses are required to override this method, but we 257 _don't_ raise NotImplementedError here. Various methods fail in 258 inscrutable ways if this method raises any exception, so for 259 debugging purposes, it's better to return a default value. 260 261 @return A descriptive string. 262 """ 263 return ('Class %s fails to implement description().' % 264 type(self).__name__) 265 266 267 class Verifier(_DependencyNode): 268 """ 269 Abstract class embodying one verification check. 270 271 A concrete subclass of `Verifier` provides a simple check that can 272 determine a host's fitness for testing. Failure indicates that the 273 check found a problem that can cause at least one test to fail. 274 275 `Verifier` objects are organized in a DAG identifying dependencies 276 among operations. The DAG controls ordering and prevents wasted 277 effort: If verification operation V2 requires that verification 278 operation V1 pass, then a) V1 will run before V2, and b) if V1 279 fails, V2 won't run at all. The `_verify_host()` method ensures 280 that all dependencies run and pass before invoking the `verify()` 281 method. 282 283 A `Verifier` object caches its result the first time it calls 284 `verify()`. Subsequent calls return the cached result, without 285 re-running the check code. The `_reverify()` method clears the 286 cached result in the current node, and in all dependencies. 287 288 Subclasses must supply these properties and methods: 289 * `verify()`: This is the method to perform the actual 290 verification check. 291 * `description`: A one-line summary of the verification check for 292 debug log messages. 293 294 Subclasses must override all of the above attributes; subclasses 295 should not override or extend any other attributes of this class. 296 297 The description string should be a simple sentence explaining what 298 must be true for the verifier to pass. Do not include a terminating 299 period. For example: 300 301 Host is available via ssh 302 303 The base class manages the following private data: 304 * `_result`: The cached result of verification. 305 * `_dependency_list`: The list of dependencies. 306 Subclasses should not use these attributes. 307 308 @property _result Cached result of verification. 309 """ 310 311 def __init__(self, tag, dependencies): 312 super(Verifier, self).__init__(tag, 'verify', dependencies) 313 self._result = None 314 315 def _reverify(self): 316 """ 317 Discard cached verification results. 318 319 Reset the cached verification result for this node, and for the 320 transitive closure of all dependencies. 321 """ 322 if self._result is not None: 323 self._result = None 324 for v in self._dependency_list: 325 v._reverify() 326 327 def _verify_host(self, host, silent): 328 """ 329 Determine the result of verification, and log results. 330 331 If this verifier does not have a cached verification result, 332 check dependencies, and if they pass, run `verify()`. Log 333 informational messages regarding failed dependencies. If we 334 call `verify()`, log the result in `status.log`. 335 336 If we already have a cached result, return that result without 337 logging any message. 338 339 @param host The host to be tested for a problem. 340 @param silent If true, don't log host status records. 341 """ 342 if self._result is not None: 343 if isinstance(self._result, Exception): 344 raise self._result # cached failure 345 elif self._result: 346 return # cached success 347 self._result = False 348 self._verify_dependencies(host, silent) 349 logging.info('Verifying this condition: %s', self.description) 350 try: 351 self.verify(host) 352 self._record_good(host, silent) 353 except Exception as e: 354 logging.exception('Failed: %s', self.description) 355 self._result = e 356 self._record_fail(host, silent, e) 357 raise 358 self._result = True 359 360 def verify(self, host): 361 """ 362 Unconditionally perform a verification check. 363 364 This method is responsible for testing for a single problem on a 365 host. Implementations should follow these guidelines: 366 * The check should find a problem that will cause testing to 367 fail. 368 * Verification checks on a working system should run quickly 369 and should be optimized for success; a check that passes 370 should finish within seconds. 371 * Verification checks are not expected have side effects, but 372 may apply trivial fixes if they will finish within the time 373 constraints above. 374 375 A verification check should normally trigger a single set of 376 repair actions. If two different failures can require two 377 different repairs, ideally they should use two different 378 subclasses of `Verifier`. 379 380 Implementations indicate failure by raising an exception. The 381 exception text should be a short, 1-line summary of the error. 382 The text should be concise and diagnostic, as it will appear in 383 `status.log` files. 384 385 If this method finds no problems, it returns without raising any 386 exception. 387 388 Implementations should avoid most logging actions, but can log 389 DEBUG level messages if they provide significant information for 390 diagnosing failures. 391 392 @param host The host to be tested for a problem. 393 """ 394 raise NotImplementedError('Class %s does not implement ' 395 'verify()' % type(self).__name__) 396 397 398 class RepairAction(_DependencyNode): 399 """ 400 Abstract class embodying one repair procedure. 401 402 A `RepairAction` is responsible for fixing one or more failed 403 `Verifier` checks, in order to make those checks pass. 404 405 Each repair action includes one or more verifier triggers that 406 determine when the repair action should run. A repair action 407 will call its `repair()` method if one or more of its triggers 408 fails. A repair action is successful if all of its triggers pass 409 after calling `repair()`. 410 411 A `RepairAction` is a subclass of `_DependencyNode`; if any of a 412 repair action's dependencies fail, the action does not check its 413 triggers, and doesn't call `repair()`. 414 415 Subclasses must supply these attributes: 416 * `repair()`: This is the method to perform the necessary 417 repair. The method should avoid most logging actions, but 418 can log DEBUG level messages if they provide significant 419 information for diagnosing failures. 420 * `description`: A one-line summary of the repair action for 421 debug log messages. 422 423 Subclasses must override both of the above attributes and should 424 not override any other attributes of this class. 425 426 The description string should be a simple sentence explaining the 427 operation that will be performed. Do not include a terminating 428 period. For example: 429 430 Re-install the stable build via AU 431 432 @property _trigger_list List of verification checks that will 433 trigger this repair when they fail. 434 """ 435 436 def __init__(self, tag, dependencies, triggers): 437 super(RepairAction, self).__init__(tag, 'repair', dependencies) 438 self._trigger_list = triggers 439 440 def _record_start(self, host, silent): 441 """Log a 'START' status line. 442 443 @param host Host which will record the status record. 444 @param silent Don't record the event if this is a true 445 value. 446 """ 447 self._record(host, silent, 'START') 448 449 def _record_end_good(self, host, silent): 450 """Log an 'END GOOD' status line. 451 452 @param host Host which will record the status record. 453 @param silent Don't record the event if this is a true 454 value. 455 """ 456 self._record(host, silent, 'END GOOD') 457 self.status = 'repaired' 458 459 def _record_end_fail(self, host, silent, status, *args): 460 """Log an 'END FAIL' status line. 461 462 @param host Host which will record the status record. 463 @param silent Don't record the event if this is a true 464 value. 465 @param args Extra arguments to `self._record()` 466 """ 467 self._record(host, silent, 'END FAIL', *args) 468 self.status = status 469 470 def _repair_host(self, host, silent): 471 """ 472 Apply this repair action if any triggers fail. 473 474 Repair is triggered when all dependencies are successful, and at 475 least one trigger fails. 476 477 If the `repair()` method triggers, the success or failure of 478 this operation is logged in `status.log` bracketed by 'START' 479 and 'END' records. Details of whether or why `repair()` 480 triggered are written to the debug logs. If repair doesn't 481 trigger, nothing is logged to `status.log`. 482 483 @param host The host to be repaired. 484 @param silent If true, don't log host status records. 485 """ 486 # Note: Every exit path from the method must set `self.status`. 487 # There's a lot of exit paths, so be careful. 488 # 489 # If we're blocked by a failed dependency, we exit with an 490 # exception. So set status to 'blocked' first. 491 self.status = 'blocked' 492 self._verify_dependencies(host, silent) 493 # This is a defensive action. Every path below should overwrite 494 # this setting, but if it doesn't, we want our status to reflect 495 # a coding error. 496 self.status = 'unknown' 497 try: 498 self._verify_list(host, self._trigger_list, silent) 499 except AutoservVerifyDependencyError as e: 500 e.log_dependencies( 501 'Attempting this repair action', 502 'Repairing because these triggers failed') 503 self._record_start(host, silent) 504 try: 505 self.repair(host) 506 except Exception as e: 507 logging.exception('Repair failed: %s', self.description) 508 self._record_fail(host, silent, e) 509 self._record_end_fail(host, silent, 'failed-action') 510 raise 511 try: 512 for v in self._trigger_list: 513 v._reverify() 514 self._verify_list(host, self._trigger_list, silent) 515 self._record_end_good(host, silent) 516 except AutoservVerifyDependencyError as e: 517 e.log_dependencies( 518 'This repair action reported success', 519 'However, these triggers still fail') 520 self._record_end_fail(host, silent, 'failed-trigger') 521 raise AutoservRepairError( 522 'Some verification checks still fail') 523 except Exception: 524 # The specification for `self._verify_list()` says 525 # that this can't happen; this is a defensive 526 # precaution. 527 self._record_end_fail(host, silent, 'unknown', 528 'Internal error in repair') 529 raise 530 else: 531 self.status = 'untriggered' 532 logging.info('No failed triggers, skipping repair: %s', 533 self.description) 534 535 def repair(self, host): 536 """ 537 Apply this repair action to the given host. 538 539 This method is responsible for applying changes to fix failures 540 in one or more verification checks. The repair is considered 541 successful if the DUT passes the specific checks after this 542 method completes. 543 544 Implementations indicate failure by raising an exception. The 545 exception text should be a short, 1-line summary of the error. 546 The text should be concise and diagnostic, as it will appear in 547 `status.log` files. 548 549 If this method completes successfully, it returns without 550 raising any exception. 551 552 Implementations should avoid most logging actions, but can log 553 DEBUG level messages if they provide significant information for 554 diagnosing failures. 555 556 @param host The host to be repaired. 557 """ 558 raise NotImplementedError('Class %s does not implement ' 559 'repair()' % type(self).__name__) 560 561 562 class _RootVerifier(Verifier): 563 """ 564 Utility class used by `RepairStrategy`. 565 566 A node of this class by itself does nothing; it always passes (if it 567 can run). This class exists merely to be the root of a DAG of 568 dependencies in an instance of `RepairStrategy`. 569 """ 570 571 def verify(self, host): 572 pass 573 574 @property 575 def description(self): 576 return 'All host verification checks pass' 577 578 579 580 class RepairStrategy(object): 581 """ 582 A class for organizing `Verifier` and `RepairAction` objects. 583 584 An instance of `RepairStrategy` is organized as a DAG of `Verifier` 585 objects, plus a list of `RepairAction` objects. The class provides 586 methods for invoking those objects in the required order, when 587 needed: 588 * The `verify()` method walks the verifier DAG in dependency 589 order. 590 * The `repair()` method invokes the repair actions in list order. 591 Each repair action will invoke its dependencies and triggers as 592 needed. 593 594 # The Verifier DAG 595 The verifier DAG is constructed from the first argument passed to 596 the passed to the `RepairStrategy` constructor. That argument is an 597 iterable consisting of three-element tuples in the form 598 `(constructor, tag, deps)`: 599 * The `constructor` value is a callable that creates a `Verifier` 600 as for the interface of the class constructor. For classes 601 that inherit the default constructor from `Verifier`, this can 602 be the class itself. 603 * The `tag` value is the tag to be associated with the constructed 604 verifier. 605 * The `deps` value is an iterable (e.g. list or tuple) of strings. 606 Each string corresponds to the `tag` member of a `Verifier` 607 dependency. 608 609 The tag names of verifiers in the constructed DAG must all be 610 unique. The tag name defined by `RepairStrategy.ROOT_TAG` is 611 reserved and may not be used by any verifier. 612 613 In the input data for the constructor, dependencies must appear 614 before the nodes that depend on them. Thus: 615 616 ((A, 'a', ()), (B, 'b', ('a',))) # This is valid 617 ((B, 'b', ('a',)), (A, 'a', ())) # This will fail! 618 619 Internally, the DAG of verifiers is given unique root node. So, 620 given this input: 621 622 ((C, 'c', ()), 623 (A, 'a', ('c',)), 624 (B, 'b', ('c',))) 625 626 The following DAG is constructed: 627 628 Root 629 / \ 630 A B 631 \ / 632 C 633 634 Since nothing depends on `A` or `B`, the root node guarantees that 635 these two verifiers will both be called and properly logged. 636 637 The root node is not directly accessible; however repair actions can 638 trigger on it by using `RepairStrategy.ROOT_TAG`. Additionally, the 639 node will be logged in `status.log` whenever `verify()` succeeds. 640 641 # The Repair Actions List 642 The list of repair actions is constructed from the second argument 643 passed to the passed to the `RepairStrategy` constructor. That 644 argument is an iterable consisting of four-element tuples in the 645 form `(constructor, tag, deps, triggers)`: 646 * The `constructor` value is a callable that creates a 647 `RepairAction` as for the interface of the class constructor. 648 For classes that inherit the default constructor from 649 `RepairAction`, this can be the class itself. 650 * The `tag` value is the tag to be associated with the constructed 651 repair action. 652 * The `deps` value is an iterable (e.g. list or tuple) of strings. 653 Each string corresponds to the `tag` member of a `Verifier` that 654 the repair action depends on. 655 * The `triggers` value is an iterable (e.g. list or tuple) of 656 strings. Each string corresponds to the `tag` member of a 657 `Verifier` that can trigger the repair action. 658 659 `RepairStrategy` deps and triggers can only refer to verifiers, 660 not to other repair actions. 661 """ 662 663 # This name is reserved; clients may not use it. 664 ROOT_TAG = 'PASS' 665 666 @staticmethod 667 def _add_verifier(verifiers, constructor, tag, dep_tags): 668 """ 669 Construct and remember a verifier. 670 671 Create a `Verifier` using `constructor` and `tag`. Dependencies 672 for construction are found by looking up `dep_tags` in the 673 `verifiers` dictionary. 674 675 After construction, the new verifier is added to `verifiers`. 676 677 @param verifiers Dictionary of verifiers, indexed by tag. 678 @param constructor Verifier construction function. 679 @param tag Tag parameter for the construction function. 680 @param dep_tags Tags of dependencies for the constructor, to 681 be found in `verifiers`. 682 """ 683 assert tag not in verifiers 684 deps = [verifiers[d] for d in dep_tags] 685 verifiers[tag] = constructor(tag, deps) 686 687 def __init__(self, verifier_data, repair_data): 688 """ 689 Construct a `RepairStrategy` from simplified DAG data. 690 691 The input `verifier_data` object describes how to construct 692 verify nodes and the dependencies that relate them, as detailed 693 above. 694 695 The input `repair_data` object describes how to construct repair 696 actions and their dependencies and triggers, as detailed above. 697 698 @param verifier_data Iterable value with constructors for the 699 elements of the verification DAG and their 700 dependencies. 701 @param repair_data Iterable value with constructors for the 702 elements of the repair action list, and 703 their dependencies and triggers. 704 """ 705 # Metrics - we report on 'actions' for every repair action 706 # we execute; we report on 'completions' for every complete 707 # repair operation. 708 self._completions_counter = metrics.Counter( 709 'chromeos/autotest/repair/completions') 710 self._actions_counter = metrics.Counter( 711 'chromeos/autotest/repair/actions') 712 # We use the `all_verifiers` list to guarantee that our root 713 # verifier will execute its dependencies in the order provided 714 # to us by our caller. 715 verifier_map = {} 716 all_tags = [] 717 dependencies = set() 718 for constructor, tag, deps in verifier_data: 719 self._add_verifier(verifier_map, constructor, tag, deps) 720 dependencies.update(deps) 721 all_tags.append(tag) 722 # Capture all the verifiers that have nothing depending on them. 723 root_tags = [t for t in all_tags if t not in dependencies] 724 self._add_verifier(verifier_map, _RootVerifier, 725 self.ROOT_TAG, root_tags) 726 self._verify_root = verifier_map[self.ROOT_TAG] 727 self._repair_actions = [] 728 for constructor, tag, deps, triggers in repair_data: 729 r = constructor(tag, 730 [verifier_map[d] for d in deps], 731 [verifier_map[t] for t in triggers]) 732 self._repair_actions.append(r) 733 734 def _count_completions(self, host, success): 735 try: 736 board = host.host_info_store.board or '' 737 except Exception: 738 board = '' 739 fields = {'success': success, 'board': board} 740 self._completions_counter.increment(fields=fields) 741 for ra in self._repair_actions: 742 fields = {'tag': ra.tag, 743 'status': ra.status, 744 'success': success, 745 'board': board} 746 self._actions_counter.increment(fields=fields) 747 748 def verify(self, host, silent=False): 749 """ 750 Run the verifier DAG on the given host. 751 752 @param host The target to be verified. 753 @param silent If true, don't log host status records. 754 """ 755 self._verify_root._reverify() 756 self._verify_root._verify_host(host, silent) 757 758 def repair(self, host, silent=False): 759 """ 760 Run the repair list on the given host. 761 762 @param host The target to be repaired. 763 @param silent If true, don't log host status records. 764 """ 765 self._verify_root._reverify() 766 for ra in self._repair_actions: 767 try: 768 ra._repair_host(host, silent) 769 except Exception as e: 770 # all logging and exception handling was done at 771 # lower levels 772 pass 773 try: 774 self._verify_root._verify_host(host, silent) 775 except: 776 self._count_completions(host, False) 777 raise 778 self._count_completions(host, True) 779
