1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import datetime 6 import logging 7 import time 8 import warnings 9 10 import common 11 12 from autotest_lib.client.common_lib import base_job 13 from autotest_lib.client.common_lib import error 14 from autotest_lib.client.common_lib import priorities 15 from autotest_lib.client.common_lib import time_utils 16 from autotest_lib.client.common_lib import utils 17 from autotest_lib.client.common_lib.cros import dev_server 18 from autotest_lib.server.cros import provision 19 from autotest_lib.server.cros.dynamic_suite import constants 20 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 21 from autotest_lib.server.cros.dynamic_suite.suite import ProvisionSuite 22 from autotest_lib.server.cros.dynamic_suite.suite import Suite 23 from autotest_lib.tko import utils as tko_utils 24 25 26 """CrOS dynamic test suite generation and execution module. 27 28 This module implements runtime-generated test suites for CrOS. 29 Design doc: http://goto.google.com/suitesv2 30 31 Individual tests can declare themselves as a part of one or more 32 suites, and the code here enables control files to be written 33 that can refer to these "dynamic suites" by name. We also provide 34 support for reimaging devices with a given build and running a 35 dynamic suite across all reimaged devices. 36 37 The public API for defining a suite includes one method: reimage_and_run(). 38 A suite control file can be written by importing this module and making 39 an appropriate call to this single method. In normal usage, this control 40 file will be run in a 'hostless' server-side autotest job, scheduling 41 sub-jobs to do the needed reimaging and test running. 42 43 Example control file: 44 45 import common 46 from autotest_lib.server.cros import provision 47 from autotest_lib.server.cros.dynamic_suite import dynamic_suite 48 49 dynamic_suite.reimage_and_run( 50 builds={provision.CROS_VERSION_PREFIX: build}, board=board, name='bvt', 51 job=job, pool=pool, check_hosts=check_hosts, add_experimental=True, 52 devserver_url=devserver_url) 53 54 This will -- at runtime -- find all control files that contain "bvt" in their 55 "SUITE=" clause, schedule jobs to reimage devices in the 56 specified pool of the specified board with the specified build and, upon 57 completion of those jobs, schedule and wait for jobs that run all the tests it 58 discovered. 59 60 Suites can be run by using the atest command-line tool: 61 atest suite create -b <board> -i <build/name> <suite> 62 e.g. 63 atest suite create -b x86-mario -i x86-mario/R20-2203.0.0 bvt 64 65 ------------------------------------------------------------------------- 66 Implementation details 67 68 A Suite instance represents a single test suite, defined by some predicate 69 run over all known control files. The simplest example is creating a Suite 70 by 'name'. 71 72 create_suite_job() takes the parameters needed to define a suite run (board, 73 build to test, machine pool, and which suite to run), ensures important 74 preconditions are met, finds the appropraite suite control file, and then 75 schedules the hostless job that will do the rest of the work. 76 77 Note that we have more than one Dev server in our test lab architecture. 78 We currently load balance per-build being tested, so one and only one dev 79 server is used by any given run through the reimaging/testing flow. 80 81 - create_suite_job() 82 The primary role of create_suite_job() is to ensure that the required 83 artifacts for the build to be tested are staged on the dev server. This 84 includes payloads required to autoupdate machines to the desired build, as 85 well as the autotest control files appropriate for that build. Then, the 86 RPC pulls the control file for the suite to be run from the dev server and 87 uses it to create the suite job with the autotest frontend. 88 89 +----------------+ 90 | Google Storage | Client 91 +----------------+ | 92 | ^ | create_suite_job() 93 payloads/ | | | 94 control files | | request | 95 V | V 96 +-------------+ download request +--------------------------+ 97 | |<----------------------| | 98 | Dev Server | | Autotest Frontend (AFE) | 99 | |---------------------->| | 100 +-------------+ suite control file +--------------------------+ 101 | 102 V 103 Suite Job (hostless) 104 105 - Reimage and Run 106 The overall process is to schedule all the tests, and then wait for the tests 107 to complete. 108 109 - The Reimaging Process 110 111 As an artifact of an old implementation, the number of machines to use 112 is called the 'sharding_factor', and the default is defined in the [CROS] 113 section of global_config.ini. 114 115 There used to be a 'num' parameter to control the maximum number of 116 machines, but it does not do anything any more. 117 118 A test control file can specify a list of DEPENDENCIES, which are really just 119 the set of labels a host needs to have in order for that test to be scheduled 120 on it. In the case of a dynamic_suite, many tests in the suite may have 121 DEPENDENCIES specified. All tests are scheduled with the DEPENDENCIES that 122 they specify, along with any suite dependencies that were specified, and the 123 scheduler will find and provision a host capable of running the test. 124 125 - Scheduling Suites 126 A Suite instance uses the labels specified in the suite dependencies to 127 schedule tests across all the hosts in the pool. It then waits for all these 128 jobs. As an optimization, the Dev server stages the payloads necessary to 129 run a suite in the background _after_ it has completed all the things 130 necessary for reimaging. Before running a suite, reimage_and_run() calls out 131 to the Dev server and blocks until it's completed staging all build artifacts 132 needed to run test suites. 133 134 Step by step: 135 0) At instantiation time, find all appropriate control files for this suite 136 that were included in the build to be tested. To do this, we consult the 137 Dev Server, where all these control files are staged. 138 139 +------------+ control files? +--------------------------+ 140 | |<----------------------| | 141 | Dev Server | | Autotest Frontend (AFE) | 142 | |---------------------->| [Suite Job] | 143 +------------+ control files! +--------------------------+ 144 145 1) Now that the Suite instance exists, it schedules jobs for every control 146 file it deemed appropriate, to be run on the hosts that were labeled 147 by the provisioning. We stuff keyvals into these jobs, indicating what 148 build they were testing and which suite they were for. 149 150 +--------------------------+ Job for VersLabel +--------+ 151 | |------------------------>| Host 1 | VersLabel 152 | Autotest Frontend (AFE) | +--------+ +--------+ 153 | [Suite Job] |----------->| Host 2 | 154 +--------------------------+ Job for +--------+ 155 | ^ VersLabel VersLabel 156 | | 157 +----------------+ 158 One job per test 159 {'build': build/name, 160 'suite': suite_name} 161 162 2) Now that all jobs are scheduled, they'll be doled out as labeled hosts 163 finish their assigned work and become available again. 164 165 - Waiting on Suites 166 0) As we clean up each test job, we check to see if any crashes occurred. If 167 they did, we look at the 'build' keyval in the job to see which build's debug 168 symbols we'll need to symbolicate the crash dump we just found. 169 170 1) Using this info, we tell a special Crash Server to stage the required debug 171 symbols. Once that's done, we ask the Crash Server to use those symbols to 172 symbolicate the crash dump in question. 173 174 +----------------+ 175 | Google Storage | 176 +----------------+ 177 | ^ 178 symbols! | | symbols? 179 V | 180 +------------+ stage symbols for build +--------------------------+ 181 | |<--------------------------| | 182 | Crash | | | 183 | Server | dump to symbolicate | Autotest Frontend (AFE) | 184 | |<--------------------------| [Suite Job] | 185 | |-------------------------->| | 186 +------------+ symbolicated dump +--------------------------+ 187 188 2) As jobs finish, we record their success or failure in the status of the suite 189 job. We also record a 'job keyval' in the suite job for each test, noting 190 the job ID and job owner. This can be used to refer to test logs later. 191 3) Once all jobs are complete, status is recorded for the suite job, and the 192 job_repo_url host attribute is removed from all hosts used by the suite. 193 194 """ 195 196 197 # Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. 198 199 class _SuiteSpec(object): 200 """This class contains the info that defines a suite run.""" 201 202 _REQUIRED_KEYWORDS = { 203 'board': str, 204 'builds': dict, 205 'name': str, 206 'job': base_job.base_job, 207 'devserver_url': str, 208 } 209 210 _VERSION_PREFIXES = frozenset(( 211 provision.CROS_VERSION_PREFIX, 212 provision.CROS_ANDROID_VERSION_PREFIX, 213 )) 214 215 def __init__( 216 self, 217 builds=None, 218 board=None, 219 name=None, 220 job=None, 221 devserver_url=None, 222 pool=None, 223 check_hosts=True, 224 add_experimental=True, 225 file_bugs=False, 226 max_runtime_mins=24*60, 227 timeout_mins=24*60, 228 suite_dependencies=None, 229 bug_template=None, 230 priority=priorities.Priority.DEFAULT, 231 predicate=None, 232 wait_for_results=True, 233 job_retry=False, 234 max_retries=None, 235 offload_failures_only=False, 236 test_source_build=None, 237 run_prod_code=False, 238 delay_minutes=0, 239 job_keyvals=None, 240 test_args=None, 241 child_dependencies=(), 242 **dargs): 243 """ 244 Vets arguments for reimage_and_run() and populates self with supplied 245 values. 246 247 Currently required args: 248 @param builds: the builds to install e.g. 249 {'cros-version:': 'x86-alex-release/R18-1655.0.0', 250 'fwrw-version:': 'x86-alex-firmware/R36-5771.50.0'} 251 @param board: which kind of devices to reimage. 252 @param name: a value of the SUITE control file variable to search for. 253 @param job: an instance of client.common_lib.base_job representing the 254 currently running suite job. 255 @param devserver_url: url to the selected devserver. 256 257 Currently supported optional args: 258 @param pool: the pool of machines to use for scheduling purposes. 259 @param check_hosts: require appropriate hosts to be available now. 260 @param add_experimental: schedule experimental tests as well, or not. 261 @param file_bugs: File bugs when tests in this suite fail. 262 @param max_runtime_mins: Max runtime in mins for each of the sub-jobs 263 this suite will run. 264 @param timeout_mins: Max lifetime in minutes for each of the sub-jobs 265 that this suite runs. 266 @param suite_dependencies: A list of strings of suite level 267 dependencies, which act just like test 268 dependencies and are appended to each test's 269 set of dependencies at job creation time. 270 A string of comma seperated labels is 271 accepted for backwards compatibility. 272 @param bug_template: A template dictionary specifying the default bug 273 filing options for failures in this suite. 274 @param priority: Integer priority level. Higher is more important. 275 @param predicate: Optional argument. If present, should be a function 276 mapping ControlData objects to True if they should be 277 included in suite. If argument is absent, suite 278 behavior will default to creating a suite of based 279 on the SUITE field of control files. 280 @param wait_for_results: Set to False to run the suite job without 281 waiting for test jobs to finish. 282 @param job_retry: Set to True to enable job-level retry. 283 @param max_retries: Maximum retry limit at suite level if not None. 284 Regardless how many times each individual test 285 has been retried, the total number of retries 286 happening in the suite can't exceed max_retries. 287 @param offload_failures_only: Only enable gs_offloading for failed 288 jobs. 289 @param test_source_build: Build that contains the server-side test code, 290 e.g., it can be the value of builds['cros-version:'] or 291 builds['fw-version:']. None uses the server-side test code from 292 builds['cros-version:']. 293 @param run_prod_code: If true, the suite will run the test code that 294 lives in prod aka the test code currently on the 295 lab servers. 296 @param delay_minutes: Delay the creation of test jobs for a given number 297 of minutes. 298 @param job_keyvals: General job keyvals to be inserted into keyval file 299 @param test_args: A dict of args passed all the way to each individual 300 test that will be actually ran. 301 @param child_dependencies: (optional) list of dependency strings 302 to be added as dependencies to child jobs. 303 @param **dargs: these arguments will be ignored. This allows us to 304 deprecate and remove arguments in ToT while not 305 breaking branch builds. 306 """ 307 self._check_init_params( 308 board=board, 309 builds=builds, 310 name=name, 311 job=job, 312 devserver_url=devserver_url) 313 314 self.board = 'board:%s' % board 315 self.builds = builds 316 self.name = name 317 self.job = job 318 self.pool = ('pool:%s' % pool) if pool else pool 319 self.check_hosts = check_hosts 320 self.add_experimental = add_experimental 321 self.file_bugs = file_bugs 322 self.dependencies = {'': []} 323 self.max_runtime_mins = max_runtime_mins 324 self.timeout_mins = timeout_mins 325 self.bug_template = {} if bug_template is None else bug_template 326 self.priority = priority 327 self.wait_for_results = wait_for_results 328 self.job_retry = job_retry 329 self.max_retries = max_retries 330 self.offload_failures_only = offload_failures_only 331 self.run_prod_code = run_prod_code 332 self.delay_minutes = delay_minutes 333 self.job_keyvals = job_keyvals 334 self.test_args = test_args 335 self.child_dependencies = child_dependencies 336 337 self._init_predicate(predicate) 338 self._init_suite_dependencies(suite_dependencies) 339 self._init_devserver(devserver_url) 340 self._init_test_source_build(test_source_build) 341 self._translate_builds() 342 self._add_builds_to_suite_deps() 343 344 for key, value in dargs.iteritems(): 345 warnings.warn('Ignored key %r was passed to suite with value %r' 346 % (key, value)) 347 348 def _check_init_params(self, **kwargs): 349 for key, expected_type in self._REQUIRED_KEYWORDS.iteritems(): 350 value = kwargs.get(key) 351 # TODO(ayatane): `not value` includes both the cases where value is 352 # None and where value is the correct type, but empty (e.g., empty 353 # dict). It looks like this is NOT the intended behavior, but I'm 354 # hesitant to remove it in case something is actually relying on 355 # this behavior. 356 if not value or not isinstance(value, expected_type): 357 raise error.SuiteArgumentException( 358 'reimage_and_run() needs %s=<%r>' 359 % (key, expected_type)) 360 361 def _init_predicate(self, predicate): 362 """Initialize predicate attribute.""" 363 if predicate is None: 364 self.predicate = Suite.name_in_tag_predicate(self.name) 365 else: 366 self.predicate = predicate 367 368 369 def _init_suite_dependencies(self, suite_dependencies): 370 """Initialize suite dependencies attribute.""" 371 if suite_dependencies is None: 372 self.suite_dependencies = [] 373 elif isinstance(suite_dependencies, str): 374 self.suite_dependencies = [dep.strip(' ') for dep 375 in suite_dependencies.split(',')] 376 else: 377 self.suite_dependencies = suite_dependencies 378 379 def _init_devserver(self, devserver_url): 380 """Initialize devserver attribute.""" 381 self.devserver = dev_server.ImageServer(devserver_url) 382 383 def _init_test_source_build(self, test_source_build): 384 """Initialize test_source_build attribute.""" 385 if test_source_build: 386 test_source_build = self.devserver.translate(test_source_build) 387 388 self.test_source_build = Suite.get_test_source_build( 389 self.builds, test_source_build=test_source_build) 390 391 def _translate_builds(self): 392 """Translate build names if they are in LATEST format.""" 393 for prefix in self._VERSION_PREFIXES: 394 if prefix in self.builds: 395 translated_build = self.devserver.translate( 396 self.builds[prefix]) 397 self.builds[prefix] = translated_build 398 399 def _add_builds_to_suite_deps(self): 400 """Add builds to suite_dependencies. 401 402 To support provision both CrOS and firmware, option builds are added to 403 _SuiteSpec, e.g., 404 405 builds = {'cros-version:': 'x86-alex-release/R18-1655.0.0', 406 'fwrw-version:': 'x86-alex-firmware/R36-5771.50.0'} 407 408 version_prefix+build should make it into each test as a DEPENDENCY. 409 The easiest way to do this is to tack it onto the suite_dependencies. 410 """ 411 self.suite_dependencies.extend( 412 provision.join(version_prefix, build) 413 for version_prefix, build in self.builds.iteritems() 414 ) 415 416 417 class _ProvisionSuiteSpec(_SuiteSpec): 418 419 def __init__(self, num_required, **kwargs): 420 self.num_required = num_required 421 super(_ProvisionSuiteSpec, self).__init__(**kwargs) 422 423 424 def run_provision_suite(**dargs): 425 """ 426 Run a provision suite. 427 428 Will re-image a number of devices (of the specified board) with the 429 provided builds by scheduling dummy_Pass. 430 431 @param job: an instance of client.common_lib.base_job representing the 432 currently running suite job. 433 434 @raises AsynchronousBuildFailure: if there was an issue finishing staging 435 from the devserver. 436 @raises MalformedDependenciesException: if the dependency_info file for 437 the required build fails to parse. 438 """ 439 spec = _ProvisionSuiteSpec(**dargs) 440 441 afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, 442 user=spec.job.user, debug=False) 443 tko = frontend_wrappers.RetryingTKO(timeout_min=30, delay_sec=10, 444 user=spec.job.user, debug=False) 445 446 try: 447 my_job_id = int(tko_utils.get_afe_job_id(spec.job.tag)) 448 logging.debug('Determined own job id: %d', my_job_id) 449 except (TypeError, ValueError): 450 my_job_id = None 451 logging.warning('Could not determine own job id.') 452 453 suite = ProvisionSuite( 454 tag=spec.name, 455 builds=spec.builds, 456 board=spec.board, 457 devserver=spec.devserver, 458 num_required=spec.num_required, 459 afe=afe, 460 tko=tko, 461 pool=spec.pool, 462 results_dir=spec.job.resultdir, 463 max_runtime_mins=spec.max_runtime_mins, 464 timeout_mins=spec.timeout_mins, 465 file_bugs=spec.file_bugs, 466 suite_job_id=my_job_id, 467 extra_deps=spec.suite_dependencies, 468 priority=spec.priority, 469 wait_for_results=spec.wait_for_results, 470 job_retry=spec.job_retry, 471 max_retries=spec.max_retries, 472 offload_failures_only=spec.offload_failures_only, 473 test_source_build=spec.test_source_build, 474 run_prod_code=spec.run_prod_code, 475 job_keyvals=spec.job_keyvals, 476 test_args=spec.test_args, 477 child_dependencies=spec.child_dependencies, 478 ) 479 480 _run_suite_with_spec(suite, spec) 481 482 logging.debug('Returning from dynamic_suite.run_provision_suite') 483 484 485 def reimage_and_run(**dargs): 486 """ 487 Backward-compatible API for dynamic_suite. 488 489 Will re-image a number of devices (of the specified board) with the 490 provided builds, and then run the indicated test suite on them. 491 Guaranteed to be compatible with any build from stable to dev. 492 493 @param dargs: Dictionary containing the arguments passed to _SuiteSpec(). 494 @raises AsynchronousBuildFailure: if there was an issue finishing staging 495 from the devserver. 496 @raises MalformedDependenciesException: if the dependency_info file for 497 the required build fails to parse. 498 """ 499 suite_spec = _SuiteSpec(**dargs) 500 501 afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, 502 user=suite_spec.job.user, debug=False) 503 tko = frontend_wrappers.RetryingTKO(timeout_min=30, delay_sec=10, 504 user=suite_spec.job.user, debug=False) 505 506 try: 507 my_job_id = int(tko_utils.get_afe_job_id(dargs['job'].tag)) 508 logging.debug('Determined own job id: %d', my_job_id) 509 except (TypeError, ValueError): 510 my_job_id = None 511 logging.warning('Could not determine own job id.') 512 513 _perform_reimage_and_run(suite_spec, afe, tko, suite_job_id=my_job_id) 514 515 logging.debug('Returning from dynamic_suite.reimage_and_run.') 516 517 518 def _perform_reimage_and_run(spec, afe, tko, suite_job_id=None): 519 """ 520 Do the work of reimaging hosts and running tests. 521 522 @param spec: a populated _SuiteSpec object. 523 @param afe: an instance of AFE as defined in server/frontend.py. 524 @param tko: an instance of TKO as defined in server/frontend.py. 525 @param suite_job_id: Job id that will act as parent id to all sub jobs. 526 Default: None 527 """ 528 # We can't create the suite until the devserver has finished downloading 529 # control_files and test_suites packages so that we can get the control 530 # files to schedule. 531 if not spec.run_prod_code: 532 _stage_artifacts_for_build(spec.devserver, spec.test_source_build) 533 suite = Suite.create_from_predicates( 534 predicates=[spec.predicate], 535 name=spec.name, 536 builds=spec.builds, 537 board=spec.board, 538 devserver=spec.devserver, 539 afe=afe, 540 tko=tko, 541 pool=spec.pool, 542 results_dir=spec.job.resultdir, 543 max_runtime_mins=spec.max_runtime_mins, 544 timeout_mins=spec.timeout_mins, 545 file_bugs=spec.file_bugs, 546 suite_job_id=suite_job_id, 547 extra_deps=spec.suite_dependencies, 548 priority=spec.priority, 549 wait_for_results=spec.wait_for_results, 550 job_retry=spec.job_retry, 551 max_retries=spec.max_retries, 552 offload_failures_only=spec.offload_failures_only, 553 test_source_build=spec.test_source_build, 554 run_prod_code=spec.run_prod_code, 555 job_keyvals=spec.job_keyvals, 556 test_args=spec.test_args, 557 child_dependencies=spec.child_dependencies, 558 ) 559 _run_suite_with_spec(suite, spec) 560 561 562 def _run_suite_with_spec(suite, spec): 563 """ 564 Do the work of reimaging hosts and running tests. 565 566 @param suite: _BaseSuite instance to run. 567 @param spec: a populated _SuiteSpec object. 568 """ 569 _run_suite( 570 suite=suite, 571 job=spec.job, 572 delay_minutes=spec.delay_minutes, 573 bug_template=spec.bug_template) 574 575 576 def _run_suite( 577 suite, 578 job, 579 delay_minutes, 580 bug_template): 581 """ 582 Run a suite. 583 584 @param suite: _BaseSuite instance. 585 @param job: an instance of client.common_lib.base_job representing the 586 currently running suite job. 587 @param delay_minutes: Delay the creation of test jobs for a given number 588 of minutes. 589 @param bug_template: A template dictionary specifying the default bug 590 filing options for failures in this suite. 591 """ 592 timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT) 593 utils.write_keyval( 594 job.resultdir, 595 {constants.ARTIFACT_FINISHED_TIME: timestamp}) 596 597 if delay_minutes: 598 logging.debug('delay_minutes is set. Sleeping %d minutes before ' 599 'creating test jobs.', delay_minutes) 600 time.sleep(delay_minutes*60) 601 logging.debug('Finished waiting for %d minutes before creating test ' 602 'jobs.', delay_minutes) 603 604 # Now we get to asychronously schedule tests. 605 suite.schedule(job.record_entry) 606 607 if suite.wait_for_results: 608 logging.debug('Waiting on suite.') 609 suite.wait(job.record_entry) 610 logging.debug('Finished waiting on suite. ' 611 'Returning from _perform_reimage_and_run.') 612 else: 613 logging.info('wait_for_results is set to False, suite job will exit ' 614 'without waiting for test jobs to finish.') 615 616 617 def _stage_artifacts_for_build(devserver, build): 618 """Stage artifacts for a suite job. 619 620 @param devserver: devserver to stage artifacts with. 621 @param build: image to stage artifacts for. 622 """ 623 try: 624 devserver.stage_artifacts( 625 image=build, 626 artifacts=['control_files', 'test_suites']) 627 except dev_server.DevServerException as e: 628 # If we can't get the control files, there's nothing to run. 629 raise error.AsynchronousBuildFailure(e) 630
