Home | History | Annotate | Download | only in dynamic_suite
      1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 
      6 import random
      7 import re
      8 
      9 import common
     10 
     11 from autotest_lib.client.common_lib import global_config
     12 
     13 
     14 _CONFIG = global_config.global_config
     15 
     16 
     17 # comments injected into the control file.
     18 _INJECT_BEGIN = '# INJECT_BEGIN - DO NOT DELETE THIS LINE'
     19 _INJECT_END = '# INJECT_END - DO NOT DELETE LINE'
     20 
     21 
     22 # The regex for an injected line in the control file with the format:
     23 # varable_name=varable_value
     24 _INJECT_VAR_RE = re.compile('^[_A-Za-z]\w*=.+$')
     25 
     26 
     27 def image_url_pattern():
     28     """Returns image_url_pattern from global_config."""
     29     return _CONFIG.get_config_value('CROS', 'image_url_pattern', type=str)
     30 
     31 
     32 def firmware_url_pattern():
     33     """Returns firmware_url_pattern from global_config."""
     34     return _CONFIG.get_config_value('CROS', 'firmware_url_pattern', type=str)
     35 
     36 
     37 def factory_image_url_pattern():
     38     """Returns path to factory image after it's been staged."""
     39     return _CONFIG.get_config_value('CROS', 'factory_image_url_pattern',
     40                                     type=str)
     41 
     42 
     43 def sharding_factor():
     44     """Returns sharding_factor from global_config."""
     45     return _CONFIG.get_config_value('CROS', 'sharding_factor', type=int)
     46 
     47 
     48 def infrastructure_user():
     49     """Returns infrastructure_user from global_config."""
     50     return _CONFIG.get_config_value('CROS', 'infrastructure_user', type=str)
     51 
     52 
     53 def package_url_pattern(is_launch_control_build=False):
     54     """Returns package_url_pattern from global_config.
     55 
     56     @param is_launch_control_build: True if the package url is for Launch
     57             Control build. Default is False.
     58     """
     59     if is_launch_control_build:
     60         return _CONFIG.get_config_value('ANDROID', 'package_url_pattern',
     61                                         type=str)
     62     else:
     63         return _CONFIG.get_config_value('CROS', 'package_url_pattern', type=str)
     64 
     65 
     66 def try_job_timeout_mins():
     67     """Returns try_job_timeout_mins from global_config."""
     68     return _CONFIG.get_config_value('SCHEDULER', 'try_job_timeout_mins',
     69                                     type=int, default=4*60)
     70 
     71 
     72 def get_package_url(devserver_url, build):
     73     """Returns the package url from the |devserver_url| and |build|.
     74 
     75     @param devserver_url: a string specifying the host to contact e.g.
     76         http://my_host:9090.
     77     @param build: the build/image string to use e.g. mario-release/R19-123.0.1.
     78     @return the url where you can find the packages for the build.
     79     """
     80     return package_url_pattern() % (devserver_url, build)
     81 
     82 
     83 def get_devserver_build_from_package_url(package_url,
     84                                          is_launch_control_build=False):
     85     """The inverse method of get_package_url.
     86 
     87     @param package_url: a string specifying the package url.
     88     @param is_launch_control_build: True if the package url is for Launch
     89                 Control build. Default is False.
     90 
     91     @return tuple containing the devserver_url, build.
     92     """
     93     pattern = package_url_pattern(is_launch_control_build)
     94     re_pattern = pattern.replace('%s', '(\S+)')
     95 
     96     devserver_build_tuple = re.search(re_pattern, package_url).groups()
     97 
     98     # TODO(beeps): This is a temporary hack around the fact that all
     99     # job_repo_urls in the database currently contain 'archive'. Remove
    100     # when all hosts have been reimaged at least once. Ref: crbug.com/214373.
    101     return (devserver_build_tuple[0],
    102             devserver_build_tuple[1].replace('archive/', ''))
    103 
    104 
    105 def get_build_from_image(image):
    106     """Get the build name from the image string.
    107 
    108     @param image: A string of image, can be the build name or a url to the
    109                   build, e.g.,
    110                   http://devserver/update/alex-release/R27-3837.0.0
    111 
    112     @return: Name of the build. Return None if fail to parse build name.
    113     """
    114     if not image.startswith('http://'):
    115         return image
    116     else:
    117         match = re.match('.*/([^/]+/R\d+-[^/]+)', image)
    118         if match:
    119             return match.group(1)
    120 
    121 
    122 def get_random_best_host(afe, host_list, require_usable_hosts=True):
    123     """
    124     Randomly choose the 'best' host from host_list, using fresh status.
    125 
    126     Hit the AFE to get latest status for the listed hosts.  Then apply
    127     the following heuristic to pick the 'best' set:
    128 
    129     Remove unusable hosts (not tools.is_usable()), then
    130     'Ready' > 'Running, Cleaning, Verifying, etc'
    131 
    132     If any 'Ready' hosts exist, return a random choice.  If not, randomly
    133     choose from the next tier.  If there are none of those either, None.
    134 
    135     @param afe: autotest front end that holds the hosts being managed.
    136     @param host_list: an iterable of Host objects, per server/frontend.py
    137     @param require_usable_hosts: only return hosts currently in a usable
    138                                  state.
    139     @return a Host object, or None if no appropriate host is found.
    140     """
    141     if not host_list:
    142         return None
    143     hostnames = [host.hostname for host in host_list]
    144     updated_hosts = afe.get_hosts(hostnames=hostnames)
    145     usable_hosts = [host for host in updated_hosts if is_usable(host)]
    146     ready_hosts = [host for host in usable_hosts if host.status == 'Ready']
    147     unusable_hosts = [h for h in updated_hosts if not is_usable(h)]
    148     if ready_hosts:
    149         return random.choice(ready_hosts)
    150     if usable_hosts:
    151         return random.choice(usable_hosts)
    152     if not require_usable_hosts and unusable_hosts:
    153         return random.choice(unusable_hosts)
    154     return None
    155 
    156 
    157 def remove_legacy_injection(control_file_in):
    158     """
    159     Removes the legacy injection part from a control file.
    160 
    161     @param control_file_in: the contents of a control file to munge.
    162 
    163     @return The modified control file string.
    164     """
    165     if not control_file_in:
    166         return control_file_in
    167 
    168     new_lines = []
    169     lines = control_file_in.strip().splitlines()
    170     remove_done = False
    171     for line in lines:
    172         if remove_done:
    173             new_lines.append(line)
    174         else:
    175             if not _INJECT_VAR_RE.match(line):
    176                 remove_done = True
    177                 new_lines.append(line)
    178     return '\n'.join(new_lines)
    179 
    180 
    181 def remove_injection(control_file_in):
    182     """
    183     Removes the injection part from a control file.
    184 
    185     @param control_file_in: the contents of a control file to munge.
    186 
    187     @return The modified control file string.
    188     """
    189     if not control_file_in:
    190         return control_file_in
    191 
    192     start = control_file_in.find(_INJECT_BEGIN)
    193     if start >=0:
    194         end = control_file_in.find(_INJECT_END, start)
    195     if start < 0 or end < 0:
    196         return remove_legacy_injection(control_file_in)
    197 
    198     end += len(_INJECT_END)
    199     ch = control_file_in[end]
    200     total_length = len(control_file_in)
    201     while end <= total_length and (
    202             ch == '\n' or ch == ' ' or ch == '\t'):
    203         end += 1
    204         if end < total_length:
    205           ch = control_file_in[end]
    206     return control_file_in[:start] + control_file_in[end:]
    207 
    208 
    209 def inject_vars(vars, control_file_in):
    210     """
    211     Inject the contents of |vars| into |control_file_in|.
    212 
    213     @param vars: a dict to shoehorn into the provided control file string.
    214     @param control_file_in: the contents of a control file to munge.
    215     @return the modified control file string.
    216     """
    217     control_file = ''
    218     control_file += _INJECT_BEGIN + '\n'
    219     for key, value in vars.iteritems():
    220         # None gets injected as 'None' without this check; same for digits.
    221         if isinstance(value, str):
    222             control_file += "%s=%s\n" % (key, repr(value))
    223         else:
    224             control_file += "%s=%r\n" % (key, value)
    225 
    226     args_dict_str = "%s=%s\n" % ('args_dict', repr(vars))
    227     return control_file + args_dict_str + _INJECT_END + '\n' + control_file_in
    228 
    229 
    230 def is_usable(host):
    231     """
    232     Given a host, determine if the host is usable right now.
    233 
    234     @param host: Host instance (as in server/frontend.py)
    235     @return True if host is alive and not incorrectly locked.  Else, False.
    236     """
    237     return alive(host) and not incorrectly_locked(host)
    238 
    239 
    240 def alive(host):
    241     """
    242     Given a host, determine if the host is alive.
    243 
    244     @param host: Host instance (as in server/frontend.py)
    245     @return True if host is not under, or in need of, repair.  Else, False.
    246     """
    247     return host.status not in ['Repair Failed', 'Repairing']
    248 
    249 
    250 def incorrectly_locked(host):
    251     """
    252     Given a host, determine if the host is locked by some user.
    253 
    254     If the host is unlocked, or locked by the test infrastructure,
    255     this will return False.  There is only one system user defined as part
    256     of the test infrastructure and is listed in global_config.ini under the
    257     [CROS] section in the 'infrastructure_user' field.
    258 
    259     @param host: Host instance (as in server/frontend.py)
    260     @return False if the host is not locked, or locked by the infra.
    261             True if the host is locked by the infra user.
    262     """
    263     return (host.locked and host.locked_by != infrastructure_user())
    264 
    265 
    266 def _testname_to_keyval_key(testname):
    267     """Make a test name acceptable as a keyval key.
    268 
    269     @param  testname Test name that must be converted.
    270     @return          A string with selected bad characters replaced
    271                      with allowable characters.
    272     """
    273     # Characters for keys in autotest keyvals are restricted; in
    274     # particular, '/' isn't allowed.  Alas, in the case of an
    275     # aborted job, the test name will be a path that includes '/'
    276     # characters.  We want to file bugs for aborted jobs, so we
    277     # apply a transform here to avoid trouble.
    278     return testname.replace('/', '_')
    279 
    280 
    281 _BUG_ID_KEYVAL = '-Bug_Id'
    282 _BUG_COUNT_KEYVAL = '-Bug_Count'
    283 
    284 
    285 def create_bug_keyvals(job_id, testname, bug_info):
    286     """Create keyvals to record a bug filed against a test failure.
    287 
    288     @param testname  Name of the test for which to record a bug.
    289     @param bug_info  Pair with the id of the bug and the count of
    290                      the number of times the bug has been seen.
    291     @param job_id    The afe job id of job which the test is associated to.
    292                      job_id will be a part of the key.
    293     @return          Keyvals to be recorded for the given test.
    294     """
    295     testname = _testname_to_keyval_key(testname)
    296     keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
    297     return {
    298         keyval_base + _BUG_ID_KEYVAL: bug_info[0],
    299         keyval_base + _BUG_COUNT_KEYVAL: bug_info[1]
    300     }
    301 
    302 
    303 def get_test_failure_bug_info(keyvals, job_id, testname):
    304     """Extract information about a bug filed against a test failure.
    305 
    306     This method tries to extract bug_id and bug_count from the keyvals
    307     of a suite. If for some reason it cannot retrieve the bug_id it will
    308     return (None, None) and there will be no link to the bug filed. We will
    309     instead link directly to the logs of the failed test.
    310 
    311     If it cannot retrieve the bug_count, it will return (int(bug_id), None)
    312     and this will result in a link to the bug filed, with an inline message
    313     saying we weren't able to determine how many times the bug occured.
    314 
    315     If it retrieved both the bug_id and bug_count, we return a tuple of 2
    316     integers and link to the bug filed, as well as mention how many times
    317     the bug has occured in the buildbot stages.
    318 
    319     @param keyvals  Keyvals associated with a suite job.
    320     @param job_id   The afe job id of the job that runs the test.
    321     @param testname Name of a test from the suite.
    322     @return         None if there is no bug info, or a pair with the
    323                     id of the bug, and the count of the number of
    324                     times the bug has been seen.
    325     """
    326     testname = _testname_to_keyval_key(testname)
    327     keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
    328     bug_id = keyvals.get(keyval_base + _BUG_ID_KEYVAL)
    329     if not bug_id:
    330         return None, None
    331     bug_id = int(bug_id)
    332     bug_count = keyvals.get(keyval_base + _BUG_COUNT_KEYVAL)
    333     bug_count = int(bug_count) if bug_count else None
    334     return bug_id, bug_count
    335 
    336 
    337 def create_job_name(build, suite, test_name):
    338     """Create the name of a test job based on given build, suite, and test_name.
    339 
    340     @param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
    341     @param suite: name of the suite, e.g., bvt.
    342     @param test_name: name of the test, e.g., dummy_Pass.
    343     @return: the test job's name, e.g.,
    344              lumpy-release/R31-1234.0.0/bvt/dummy_Pass.
    345     """
    346     return '/'.join([build, suite, test_name])
    347 
    348 
    349 def get_test_name(build, suite, test_job_name):
    350     """Get the test name from test job name.
    351 
    352     Name of test job may contain information like build and suite. This method
    353     strips these information and return only the test name.
    354 
    355     @param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
    356     @param suite: name of the suite, e.g., bvt.
    357     @param test_job_name: name of the test job, e.g.,
    358                           lumpy-release/R31-1234.0.0/bvt/dummy_Pass_SERVER_JOB.
    359     @return: the test name, e.g., dummy_Pass_SERVER_JOB.
    360     """
    361     # Do not change this naming convention without updating
    362     # site_utils.parse_job_name.
    363     return test_job_name.replace('%s/%s/' % (build, suite), '')
    364