1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import datetime 6 import logging 7 import time 8 import warnings 9 10 import common 11 12 from autotest_lib.client.common_lib import base_job 13 from autotest_lib.client.common_lib import error 14 from autotest_lib.client.common_lib import priorities 15 from autotest_lib.client.common_lib import time_utils 16 from autotest_lib.client.common_lib import utils 17 from autotest_lib.client.common_lib.cros import dev_server 18 from autotest_lib.server.cros import provision 19 from autotest_lib.server.cros.dynamic_suite import constants 20 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 21 from autotest_lib.server.cros.dynamic_suite.suite import ProvisionSuite 22 from autotest_lib.server.cros.dynamic_suite.suite import Suite 23 from autotest_lib.tko import utils as tko_utils 24 25 26 """CrOS dynamic test suite generation and execution module. 27 28 This module implements runtime-generated test suites for CrOS. 29 Design doc: http://goto.google.com/suitesv2 30 31 Individual tests can declare themselves as a part of one or more 32 suites, and the code here enables control files to be written 33 that can refer to these "dynamic suites" by name. We also provide 34 support for reimaging devices with a given build and running a 35 dynamic suite across all reimaged devices. 36 37 The public API for defining a suite includes one method: reimage_and_run(). 38 A suite control file can be written by importing this module and making 39 an appropriate call to this single method. In normal usage, this control 40 file will be run in a 'hostless' server-side autotest job, scheduling 41 sub-jobs to do the needed reimaging and test running. 42 43 Example control file: 44 45 import common 46 from autotest_lib.server.cros import provision 47 from autotest_lib.server.cros.dynamic_suite import dynamic_suite 48 49 dynamic_suite.reimage_and_run( 50 builds={provision.CROS_VERSION_PREFIX: build}, board=board, name='bvt', 51 job=job, pool=pool, check_hosts=check_hosts, add_experimental=True, 52 devserver_url=devserver_url) 53 54 This will -- at runtime -- find all control files that contain "bvt" in their 55 "SUITE=" clause, schedule jobs to reimage devices in the 56 specified pool of the specified board with the specified build and, upon 57 completion of those jobs, schedule and wait for jobs that run all the tests it 58 discovered. 59 60 Suites can be run by using the atest command-line tool: 61 atest suite create -b <board> -i <build/name> <suite> 62 e.g. 63 atest suite create -b x86-mario -i x86-mario/R20-2203.0.0 bvt 64 65 ------------------------------------------------------------------------- 66 Implementation details 67 68 A Suite instance represents a single test suite, defined by some predicate 69 run over all known control files. The simplest example is creating a Suite 70 by 'name'. 71 72 create_suite_job() takes the parameters needed to define a suite run (board, 73 build to test, machine pool, and which suite to run), ensures important 74 preconditions are met, finds the appropraite suite control file, and then 75 schedules the hostless job that will do the rest of the work. 76 77 Note that we have more than one Dev server in our test lab architecture. 78 We currently load balance per-build being tested, so one and only one dev 79 server is used by any given run through the reimaging/testing flow. 80 81 - create_suite_job() 82 The primary role of create_suite_job() is to ensure that the required 83 artifacts for the build to be tested are staged on the dev server. This 84 includes payloads required to autoupdate machines to the desired build, as 85 well as the autotest control files appropriate for that build. Then, the 86 RPC pulls the control file for the suite to be run from the dev server and 87 uses it to create the suite job with the autotest frontend. 88 89 +----------------+ 90 | Google Storage | Client 91 +----------------+ | 92 | ^ | create_suite_job() 93 payloads/ | | | 94 control files | | request | 95 V | V 96 +-------------+ download request +--------------------------+ 97 | |<----------------------| | 98 | Dev Server | | Autotest Frontend (AFE) | 99 | |---------------------->| | 100 +-------------+ suite control file +--------------------------+ 101 | 102 V 103 Suite Job (hostless) 104 105 - Reimage and Run 106 The overall process is to schedule all the tests, and then wait for the tests 107 to complete. 108 109 - The Reimaging Process 110 111 As an artifact of an old implementation, the number of machines to use 112 is called the 'sharding_factor', and the default is defined in the [CROS] 113 section of global_config.ini. 114 115 There used to be a 'num' parameter to control the maximum number of 116 machines, but it does not do anything any more. 117 118 A test control file can specify a list of DEPENDENCIES, which are really just 119 the set of labels a host needs to have in order for that test to be scheduled 120 on it. In the case of a dynamic_suite, many tests in the suite may have 121 DEPENDENCIES specified. All tests are scheduled with the DEPENDENCIES that 122 they specify, along with any suite dependencies that were specified, and the 123 scheduler will find and provision a host capable of running the test. 124 125 - Scheduling Suites 126 A Suite instance uses the labels specified in the suite dependencies to 127 schedule tests across all the hosts in the pool. It then waits for all these 128 jobs. As an optimization, the Dev server stages the payloads necessary to 129 run a suite in the background _after_ it has completed all the things 130 necessary for reimaging. Before running a suite, reimage_and_run() calls out 131 to the Dev server and blocks until it's completed staging all build artifacts 132 needed to run test suites. 133 134 Step by step: 135 0) At instantiation time, find all appropriate control files for this suite 136 that were included in the build to be tested. To do this, we consult the 137 Dev Server, where all these control files are staged. 138 139 +------------+ control files? +--------------------------+ 140 | |<----------------------| | 141 | Dev Server | | Autotest Frontend (AFE) | 142 | |---------------------->| [Suite Job] | 143 +------------+ control files! +--------------------------+ 144 145 1) Now that the Suite instance exists, it schedules jobs for every control 146 file it deemed appropriate, to be run on the hosts that were labeled 147 by the provisioning. We stuff keyvals into these jobs, indicating what 148 build they were testing and which suite they were for. 149 150 +--------------------------+ Job for VersLabel +--------+ 151 | |------------------------>| Host 1 | VersLabel 152 | Autotest Frontend (AFE) | +--------+ +--------+ 153 | [Suite Job] |----------->| Host 2 | 154 +--------------------------+ Job for +--------+ 155 | ^ VersLabel VersLabel 156 | | 157 +----------------+ 158 One job per test 159 {'build': build/name, 160 'suite': suite_name} 161 162 2) Now that all jobs are scheduled, they'll be doled out as labeled hosts 163 finish their assigned work and become available again. 164 165 - Waiting on Suites 166 0) As we clean up each test job, we check to see if any crashes occurred. If 167 they did, we look at the 'build' keyval in the job to see which build's debug 168 symbols we'll need to symbolicate the crash dump we just found. 169 170 1) Using this info, we tell a special Crash Server to stage the required debug 171 symbols. Once that's done, we ask the Crash Server to use those symbols to 172 symbolicate the crash dump in question. 173 174 +----------------+ 175 | Google Storage | 176 +----------------+ 177 | ^ 178 symbols! | | symbols? 179 V | 180 +------------+ stage symbols for build +--------------------------+ 181 | |<--------------------------| | 182 | Crash | | | 183 | Server | dump to symbolicate | Autotest Frontend (AFE) | 184 | |<--------------------------| [Suite Job] | 185 | |-------------------------->| | 186 +------------+ symbolicated dump +--------------------------+ 187 188 2) As jobs finish, we record their success or failure in the status of the suite 189 job. We also record a 'job keyval' in the suite job for each test, noting 190 the job ID and job owner. This can be used to refer to test logs later. 191 3) Once all jobs are complete, status is recorded for the suite job, and the 192 job_repo_url host attribute is removed from all hosts used by the suite. 193 194 """ 195 196 197 # Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. 198 199 class _SuiteSpec(object): 200 """This class contains the info that defines a suite run.""" 201 202 _REQUIRED_KEYWORDS = { 203 'board': str, 204 'builds': dict, 205 'name': str, 206 'job': base_job.base_job, 207 'devserver_url': str, 208 } 209 210 _VERSION_PREFIXES = frozenset(( 211 provision.CROS_VERSION_PREFIX, 212 provision.CROS_ANDROID_VERSION_PREFIX, 213 provision.ANDROID_BUILD_VERSION_PREFIX, 214 )) 215 216 def __init__( 217 self, 218 builds=None, 219 board=None, 220 name=None, 221 job=None, 222 devserver_url=None, 223 pool=None, 224 check_hosts=True, 225 add_experimental=True, 226 file_bugs=False, 227 max_runtime_mins=24*60, 228 timeout_mins=24*60, 229 suite_dependencies=None, 230 bug_template=None, 231 priority=priorities.Priority.DEFAULT, 232 predicate=None, 233 wait_for_results=True, 234 job_retry=False, 235 max_retries=None, 236 offload_failures_only=False, 237 test_source_build=None, 238 run_prod_code=False, 239 delay_minutes=0, 240 job_keyvals=None, 241 test_args=None, 242 child_dependencies=(), 243 **dargs): 244 """ 245 Vets arguments for reimage_and_run() and populates self with supplied 246 values. 247 248 Currently required args: 249 @param builds: the builds to install e.g. 250 {'cros-version:': 'x86-alex-release/R18-1655.0.0', 251 'fwrw-version:': 'x86-alex-firmware/R36-5771.50.0'} 252 @param board: which kind of devices to reimage. 253 @param name: a value of the SUITE control file variable to search for. 254 @param job: an instance of client.common_lib.base_job representing the 255 currently running suite job. 256 @param devserver_url: url to the selected devserver. 257 258 Currently supported optional args: 259 @param pool: the pool of machines to use for scheduling purposes. 260 @param check_hosts: require appropriate hosts to be available now. 261 @param add_experimental: schedule experimental tests as well, or not. 262 @param file_bugs: File bugs when tests in this suite fail. 263 @param max_runtime_mins: Max runtime in mins for each of the sub-jobs 264 this suite will run. 265 @param timeout_mins: Max lifetime in minutes for each of the sub-jobs 266 that this suite runs. 267 @param suite_dependencies: A list of strings of suite level 268 dependencies, which act just like test 269 dependencies and are appended to each test's 270 set of dependencies at job creation time. 271 A string of comma seperated labels is 272 accepted for backwards compatibility. 273 @param bug_template: A template dictionary specifying the default bug 274 filing options for failures in this suite. 275 @param priority: Integer priority level. Higher is more important. 276 @param predicate: Optional argument. If present, should be a function 277 mapping ControlData objects to True if they should be 278 included in suite. If argument is absent, suite 279 behavior will default to creating a suite of based 280 on the SUITE field of control files. 281 @param wait_for_results: Set to False to run the suite job without 282 waiting for test jobs to finish. 283 @param job_retry: Set to True to enable job-level retry. 284 @param max_retries: Maximum retry limit at suite level if not None. 285 Regardless how many times each individual test 286 has been retried, the total number of retries 287 happening in the suite can't exceed max_retries. 288 @param offload_failures_only: Only enable gs_offloading for failed 289 jobs. 290 @param test_source_build: Build that contains the server-side test code, 291 e.g., it can be the value of builds['cros-version:'] or 292 builds['fw-version:']. None uses the server-side test code from 293 builds['cros-version:']. 294 @param run_prod_code: If true, the suite will run the test code that 295 lives in prod aka the test code currently on the 296 lab servers. 297 @param delay_minutes: Delay the creation of test jobs for a given number 298 of minutes. 299 @param job_keyvals: General job keyvals to be inserted into keyval file 300 @param test_args: A dict of args passed all the way to each individual 301 test that will be actually ran. 302 @param child_dependencies: (optional) list of dependency strings 303 to be added as dependencies to child jobs. 304 @param **dargs: these arguments will be ignored. This allows us to 305 deprecate and remove arguments in ToT while not 306 breaking branch builds. 307 """ 308 self._check_init_params( 309 board=board, 310 builds=builds, 311 name=name, 312 job=job, 313 devserver_url=devserver_url) 314 315 self.board = 'board:%s' % board 316 self.builds = builds 317 self.name = name 318 self.job = job 319 self.pool = ('pool:%s' % pool) if pool else pool 320 self.check_hosts = check_hosts 321 self.add_experimental = add_experimental 322 self.file_bugs = file_bugs 323 self.dependencies = {'': []} 324 self.max_runtime_mins = max_runtime_mins 325 self.timeout_mins = timeout_mins 326 self.bug_template = {} if bug_template is None else bug_template 327 self.priority = priority 328 self.wait_for_results = wait_for_results 329 self.job_retry = job_retry 330 self.max_retries = max_retries 331 self.offload_failures_only = offload_failures_only 332 self.run_prod_code = run_prod_code 333 self.delay_minutes = delay_minutes 334 self.job_keyvals = job_keyvals 335 self.test_args = test_args 336 self.child_dependencies = child_dependencies 337 338 self._init_predicate(predicate) 339 self._init_suite_dependencies(suite_dependencies) 340 self._init_devserver(devserver_url) 341 self._init_test_source_build(test_source_build) 342 self._translate_builds() 343 self._add_builds_to_suite_deps() 344 345 for key, value in dargs.iteritems(): 346 warnings.warn('Ignored key %r was passed to suite with value %r' 347 % (key, value)) 348 349 def _check_init_params(self, **kwargs): 350 for key, expected_type in self._REQUIRED_KEYWORDS.iteritems(): 351 value = kwargs.get(key) 352 # TODO(ayatane): `not value` includes both the cases where value is 353 # None and where value is the correct type, but empty (e.g., empty 354 # dict). It looks like this is NOT the intended behavior, but I'm 355 # hesitant to remove it in case something is actually relying on 356 # this behavior. 357 if not value or not isinstance(value, expected_type): 358 raise error.SuiteArgumentException( 359 'reimage_and_run() needs %s=<%r>' 360 % (key, expected_type)) 361 362 def _init_predicate(self, predicate): 363 """Initialize predicate attribute.""" 364 if predicate is None: 365 self.predicate = Suite.name_in_tag_predicate(self.name) 366 else: 367 self.predicate = predicate 368 369 370 def _init_suite_dependencies(self, suite_dependencies): 371 """Initialize suite dependencies attribute.""" 372 if suite_dependencies is None: 373 self.suite_dependencies = [] 374 elif isinstance(suite_dependencies, str): 375 self.suite_dependencies = [dep.strip(' ') for dep 376 in suite_dependencies.split(',')] 377 else: 378 self.suite_dependencies = suite_dependencies 379 380 def _init_devserver(self, devserver_url): 381 """Initialize devserver attribute.""" 382 if provision.ANDROID_BUILD_VERSION_PREFIX in self.builds: 383 self.devserver = dev_server.AndroidBuildServer(devserver_url) 384 else: 385 self.devserver = dev_server.ImageServer(devserver_url) 386 387 def _init_test_source_build(self, test_source_build): 388 """Initialize test_source_build attribute.""" 389 if test_source_build: 390 test_source_build = self.devserver.translate(test_source_build) 391 392 self.test_source_build = Suite.get_test_source_build( 393 self.builds, test_source_build=test_source_build) 394 395 def _translate_builds(self): 396 """Translate build names if they are in LATEST format.""" 397 for prefix in self._VERSION_PREFIXES: 398 if prefix in self.builds: 399 translated_build = self.devserver.translate( 400 self.builds[prefix]) 401 self.builds[prefix] = translated_build 402 403 def _add_builds_to_suite_deps(self): 404 """Add builds to suite_dependencies. 405 406 To support provision both CrOS and firmware, option builds are added to 407 _SuiteSpec, e.g., 408 409 builds = {'cros-version:': 'x86-alex-release/R18-1655.0.0', 410 'fwrw-version:': 'x86-alex-firmware/R36-5771.50.0'} 411 412 version_prefix+build should make it into each test as a DEPENDENCY. 413 The easiest way to do this is to tack it onto the suite_dependencies. 414 """ 415 self.suite_dependencies.extend( 416 provision.join(version_prefix, build) 417 for version_prefix, build in self.builds.iteritems() 418 ) 419 420 421 class _ProvisionSuiteSpec(_SuiteSpec): 422 423 def __init__(self, num_required, **kwargs): 424 self.num_required = num_required 425 super(_ProvisionSuiteSpec, self).__init__(**kwargs) 426 427 428 def run_provision_suite(**dargs): 429 """ 430 Run a provision suite. 431 432 Will re-image a number of devices (of the specified board) with the 433 provided builds by scheduling dummy_Pass. 434 435 @param job: an instance of client.common_lib.base_job representing the 436 currently running suite job. 437 438 @raises AsynchronousBuildFailure: if there was an issue finishing staging 439 from the devserver. 440 @raises MalformedDependenciesException: if the dependency_info file for 441 the required build fails to parse. 442 """ 443 spec = _ProvisionSuiteSpec(**dargs) 444 445 afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, 446 user=spec.job.user, debug=False) 447 tko = frontend_wrappers.RetryingTKO(timeout_min=30, delay_sec=10, 448 user=spec.job.user, debug=False) 449 450 try: 451 my_job_id = int(tko_utils.get_afe_job_id(spec.job.tag)) 452 logging.debug('Determined own job id: %d', my_job_id) 453 except ValueError: 454 my_job_id = None 455 logging.warning('Could not determine own job id.') 456 457 suite = ProvisionSuite( 458 tag=spec.name, 459 builds=spec.builds, 460 board=spec.board, 461 devserver=spec.devserver, 462 num_required=spec.num_required, 463 afe=afe, 464 tko=tko, 465 pool=spec.pool, 466 results_dir=spec.job.resultdir, 467 max_runtime_mins=spec.max_runtime_mins, 468 timeout_mins=spec.timeout_mins, 469 file_bugs=spec.file_bugs, 470 suite_job_id=my_job_id, 471 extra_deps=spec.suite_dependencies, 472 priority=spec.priority, 473 wait_for_results=spec.wait_for_results, 474 job_retry=spec.job_retry, 475 max_retries=spec.max_retries, 476 offload_failures_only=spec.offload_failures_only, 477 test_source_build=spec.test_source_build, 478 run_prod_code=spec.run_prod_code, 479 job_keyvals=spec.job_keyvals, 480 test_args=spec.test_args, 481 child_dependencies=spec.child_dependencies, 482 ) 483 484 _run_suite_with_spec(suite, spec) 485 486 logging.debug('Returning from dynamic_suite.run_provision_suite') 487 488 489 def reimage_and_run(**dargs): 490 """ 491 Backward-compatible API for dynamic_suite. 492 493 Will re-image a number of devices (of the specified board) with the 494 provided builds, and then run the indicated test suite on them. 495 Guaranteed to be compatible with any build from stable to dev. 496 497 @param dargs: Dictionary containing the arguments passed to _SuiteSpec(). 498 @raises AsynchronousBuildFailure: if there was an issue finishing staging 499 from the devserver. 500 @raises MalformedDependenciesException: if the dependency_info file for 501 the required build fails to parse. 502 """ 503 suite_spec = _SuiteSpec(**dargs) 504 505 afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, 506 user=suite_spec.job.user, debug=False) 507 tko = frontend_wrappers.RetryingTKO(timeout_min=30, delay_sec=10, 508 user=suite_spec.job.user, debug=False) 509 510 try: 511 my_job_id = int(tko_utils.get_afe_job_id(dargs['job'].tag)) 512 logging.debug('Determined own job id: %d', my_job_id) 513 except ValueError: 514 my_job_id = None 515 logging.warning('Could not determine own job id.') 516 517 _perform_reimage_and_run(suite_spec, afe, tko, suite_job_id=my_job_id) 518 519 logging.debug('Returning from dynamic_suite.reimage_and_run.') 520 521 522 def _perform_reimage_and_run(spec, afe, tko, suite_job_id=None): 523 """ 524 Do the work of reimaging hosts and running tests. 525 526 @param spec: a populated _SuiteSpec object. 527 @param afe: an instance of AFE as defined in server/frontend.py. 528 @param tko: an instance of TKO as defined in server/frontend.py. 529 @param suite_job_id: Job id that will act as parent id to all sub jobs. 530 Default: None 531 """ 532 # We can't create the suite until the devserver has finished downloading 533 # control_files and test_suites packages so that we can get the control 534 # files to schedule. 535 if not spec.run_prod_code: 536 _stage_artifacts_for_build(spec.devserver, spec.test_source_build) 537 suite = Suite.create_from_predicates( 538 predicates=[spec.predicate], 539 name=spec.name, 540 builds=spec.builds, 541 board=spec.board, 542 devserver=spec.devserver, 543 afe=afe, 544 tko=tko, 545 pool=spec.pool, 546 results_dir=spec.job.resultdir, 547 max_runtime_mins=spec.max_runtime_mins, 548 timeout_mins=spec.timeout_mins, 549 file_bugs=spec.file_bugs, 550 suite_job_id=suite_job_id, 551 extra_deps=spec.suite_dependencies, 552 priority=spec.priority, 553 wait_for_results=spec.wait_for_results, 554 job_retry=spec.job_retry, 555 max_retries=spec.max_retries, 556 offload_failures_only=spec.offload_failures_only, 557 test_source_build=spec.test_source_build, 558 run_prod_code=spec.run_prod_code, 559 job_keyvals=spec.job_keyvals, 560 test_args=spec.test_args, 561 child_dependencies=spec.child_dependencies, 562 ) 563 _run_suite_with_spec(suite, spec) 564 565 566 def _run_suite_with_spec(suite, spec): 567 """ 568 Do the work of reimaging hosts and running tests. 569 570 @param suite: _BaseSuite instance to run. 571 @param spec: a populated _SuiteSpec object. 572 """ 573 _run_suite( 574 suite=suite, 575 job=spec.job, 576 delay_minutes=spec.delay_minutes, 577 bug_template=spec.bug_template) 578 579 580 def _run_suite( 581 suite, 582 job, 583 delay_minutes, 584 bug_template): 585 """ 586 Run a suite. 587 588 @param suite: _BaseSuite instance. 589 @param job: an instance of client.common_lib.base_job representing the 590 currently running suite job. 591 @param delay_minutes: Delay the creation of test jobs for a given number 592 of minutes. 593 @param bug_template: A template dictionary specifying the default bug 594 filing options for failures in this suite. 595 """ 596 timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT) 597 utils.write_keyval( 598 job.resultdir, 599 {constants.ARTIFACT_FINISHED_TIME: timestamp}) 600 601 if delay_minutes: 602 logging.debug('delay_minutes is set. Sleeping %d minutes before ' 603 'creating test jobs.', delay_minutes) 604 time.sleep(delay_minutes*60) 605 logging.debug('Finished waiting for %d minutes before creating test ' 606 'jobs.', delay_minutes) 607 608 # Now we get to asychronously schedule tests. 609 suite.schedule(job.record_entry) 610 611 if suite.wait_for_results: 612 logging.debug('Waiting on suite.') 613 suite.wait(job.record_entry) 614 logging.debug('Finished waiting on suite. ' 615 'Returning from _perform_reimage_and_run.') 616 else: 617 logging.info('wait_for_results is set to False, suite job will exit ' 618 'without waiting for test jobs to finish.') 619 620 621 def _stage_artifacts_for_build(devserver, build): 622 """Stage artifacts for a suite job. 623 624 @param devserver: devserver to stage artifacts with. 625 @param build: image to stage artifacts for. 626 """ 627 try: 628 devserver.stage_artifacts( 629 image=build, 630 artifacts=['control_files', 'test_suites']) 631 except dev_server.DevServerException as e: 632 # If we can't get the control files, there's nothing to run. 633 raise error.AsynchronousBuildFailure(e) 634
