Home | History | Annotate | Download | only in server
      1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import logging
      6 import os
      7 import tempfile
      8 import urllib2
      9 
     10 from autotest_lib.client.common_lib import error, global_config
     11 from autotest_lib.client.common_lib.cros import dev_server
     12 from autotest_lib.server import installable_object, autoserv_parser
     13 from autotest_lib.server import utils as server_utils
     14 from autotest_lib.server.cros.dynamic_suite import tools
     15 from autotest_lib.server.cros.dynamic_suite.constants import JOB_REPO_URL
     16 
     17 
     18 _CONFIG = global_config.global_config
     19 _PARSER = autoserv_parser.autoserv_parser
     20 
     21 
     22 class SiteAutotest(installable_object.InstallableObject):
     23     """Site implementation of Autotest."""
     24 
     25     def get(self, location=None):
     26         if not location:
     27             location = os.path.join(self.serverdir, '../client')
     28             location = os.path.abspath(location)
     29         installable_object.InstallableObject.get(self, location)
     30         self.got = True
     31 
     32 
     33     def _get_fetch_location_from_host_attribute(self):
     34         """Get repo to use for packages from host attribute, if possible.
     35 
     36         Hosts are tagged with an attribute containing the URL
     37         from which to source packages when running a test on that host.
     38         If self.host is set, attempt to look this attribute up by calling out
     39         to the AFE.
     40 
     41         @returns value of the 'job_repo_url' host attribute, if present.
     42         """
     43         try:
     44             from autotest_lib.server import frontend
     45             if self.host:
     46                 afe = frontend.AFE(debug=False)
     47                 hosts = afe.get_hosts(hostname=self.host.hostname)
     48                 if hosts and JOB_REPO_URL in hosts[0].attributes:
     49                     return hosts[0].attributes[JOB_REPO_URL]
     50                 logging.warning("No %s for %s", JOB_REPO_URL, self.host)
     51         except (ImportError, urllib2.URLError):
     52             logging.warning('Not attempting to look for %s', JOB_REPO_URL)
     53             pass
     54         return None
     55 
     56 
     57     def get_fetch_location(self):
     58         """Generate list of locations where autotest can look for packages.
     59 
     60         Old n' busted: Autotest packages are always stored at a URL that can
     61         be derived from the one passed via the voodoo magic --image argument.
     62         New hotness: Hosts are tagged with an attribute containing the URL
     63         from which to source packages when running a test on that host.
     64 
     65         @returns the list of candidate locations to check for packages.
     66         """
     67         repos = super(SiteAutotest, self).get_fetch_location()
     68 
     69         if _PARSER.options.image:
     70             image_opt = _PARSER.options.image
     71             if image_opt.startswith('http://'):
     72                 # A devserver HTTP url was specified, set that as the repo_url.
     73                 repos.append(image_opt.replace(
     74                     'update', 'static').rstrip('/') + '/autotest')
     75             else:
     76                 # An image_name like stumpy-release/R27-3437.0.0 was specified,
     77                 # set this as the repo_url for the host. If an AFE is not being
     78                 # run, this will ensure that the installed build uses the
     79                 # associated artifacts for the test specified when running
     80                 # autoserv with --image. However, any subsequent tests run on
     81                 # the host will no longer have the context of the image option
     82                 # and will revert back to utilizing test code/artifacts that are
     83                 # currently present in the users source checkout.
     84                 devserver_url = dev_server.ImageServer.resolve(image_opt).url()
     85                 repo_url = tools.get_package_url(devserver_url, image_opt)
     86                 repos.append(repo_url)
     87         elif not server_utils.is_inside_chroot():
     88             # Only try to get fetch location from host attribute if the test
     89             # is not running inside chroot.
     90             # No --image option was specified, look for the repo url via
     91             # the host attribute. If we are not running with a full AFE
     92             # autoserv will fall back to serving packages itself from whatever
     93             # source version it is sync'd to rather than using the proper
     94             # artifacts for the build on the host.
     95             found_repo = self._get_fetch_location_from_host_attribute()
     96             if found_repo is not None:
     97                 # Add our new repo to the end, the package manager will
     98                 # later reverse the list of repositories resulting in ours
     99                 # being first
    100                 repos.append(found_repo)
    101 
    102         return repos
    103 
    104 
    105     def install(self, host=None, autodir=None, use_packaging=True):
    106         """Install autotest.  If |host| is not None, stores it in |self.host|.
    107 
    108         @param host A Host instance on which autotest will be installed
    109         @param autodir Location on the remote host to install to
    110         @param use_packaging Enable install modes that use the packaging system.
    111 
    112         """
    113         if host:
    114             self.host = host
    115 
    116         super(SiteAutotest, self).install(host=host, autodir=autodir,
    117                                           use_packaging=use_packaging)
    118 
    119 
    120     def _install(self, host=None, autodir=None, use_autoserv=True,
    121                  use_packaging=True):
    122         """
    123         Install autotest.  If get() was not called previously, an
    124         attempt will be made to install from the autotest svn
    125         repository.
    126 
    127         @param host A Host instance on which autotest will be installed
    128         @param autodir Location on the remote host to install to
    129         @param use_autoserv Enable install modes that depend on the client
    130             running with the autoserv harness
    131         @param use_packaging Enable install modes that use the packaging system
    132 
    133         @exception AutoservError if a tarball was not specified and
    134             the target host does not have svn installed in its path
    135         """
    136         # TODO(milleral): http://crbug.com/258161
    137         super(SiteAutotest, self)._install(host, autodir, use_autoserv,
    138                                            use_packaging)
    139         # Send over the most recent global_config.ini after installation if one
    140         # is available.
    141         # This code is a bit duplicated from
    142         # _BaseRun._create_client_config_file, but oh well.
    143         if self.installed and self.source_material:
    144             logging.info('Installing updated global_config.ini.')
    145             destination = os.path.join(self.host.get_autodir(),
    146                                        'global_config.ini')
    147             with tempfile.NamedTemporaryFile() as client_config:
    148                 config = global_config.global_config
    149                 client_section = config.get_section_values('CLIENT')
    150                 client_section.write(client_config)
    151                 client_config.flush()
    152                 self.host.send_file(client_config.name, destination)
    153 
    154 
    155     def run_static_method(self, module, method, results_dir='.', host=None,
    156                           *args):
    157         """Runs a non-instance method with |args| from |module| on the client.
    158 
    159         This method runs a static/class/module autotest method on the client.
    160         For example:
    161           run_static_method("autotest_lib.client.cros.cros_ui", "reboot")
    162 
    163         Will run autotest_lib.client.cros.cros_ui.reboot() on the client.
    164 
    165         @param module: module name as you would refer to it when importing in a
    166             control file. e.g. autotest_lib.client.common_lib.module_name.
    167         @param method: the method you want to call.
    168         @param results_dir: A str path where the results should be stored
    169             on the local filesystem.
    170         @param host: A Host instance on which the control file should
    171             be run.
    172         @param args: args to pass to the method.
    173         """
    174         control = "\n".join(["import %s" % module,
    175                              "%s.%s(%s)\n" % (module, method,
    176                                               ','.join(map(repr, args)))])
    177         self.run(control, results_dir=results_dir, host=host)
    178 
    179 
    180 class SiteClientLogger(object):
    181     """Overrides default client logger to allow for using a local package cache.
    182     """
    183 
    184     def _process_line(self, line):
    185         """Returns the package checksum file if it exists."""
    186         logging.debug(line)
    187         fetch_package_match = self.fetch_package_parser.search(line)
    188         if fetch_package_match:
    189             pkg_name, dest_path, fifo_path = fetch_package_match.groups()
    190             serve_packages = _CONFIG.get_config_value(
    191                 "PACKAGES", "serve_packages_from_autoserv", type=bool)
    192             if serve_packages and pkg_name == 'packages.checksum':
    193                 try:
    194                     checksum_file = os.path.join(
    195                         self.job.pkgmgr.pkgmgr_dir, 'packages', pkg_name)
    196                     if os.path.exists(checksum_file):
    197                         self.host.send_file(checksum_file, dest_path)
    198                 except error.AutoservRunError:
    199                     msg = "Package checksum file not found, continuing anyway"
    200                     logging.exception(msg)
    201 
    202                 try:
    203                     # When fetching a package, the client expects to be
    204                     # notified when the fetching is complete. Autotest
    205                     # does this pushing a B to a fifo queue to the client.
    206                     self.host.run("echo B > %s" % fifo_path)
    207                 except error.AutoservRunError:
    208                     msg = "Checksum installation failed, continuing anyway"
    209                     logging.exception(msg)
    210                 finally:
    211                     return
    212 
    213         # Fall through to process the line using the default method.
    214         super(SiteClientLogger, self)._process_line(line)
    215 
    216 
    217     def _send_tarball(self, pkg_name, remote_dest):
    218         """Uses tarballs in package manager by default."""
    219         try:
    220             server_package = os.path.join(self.job.pkgmgr.pkgmgr_dir,
    221                                           'packages', pkg_name)
    222             if os.path.exists(server_package):
    223               self.host.send_file(server_package, remote_dest)
    224               return
    225 
    226         except error.AutoservRunError:
    227             msg = ("Package %s could not be sent from the package cache." %
    228                    pkg_name)
    229             logging.exception(msg)
    230 
    231         # Fall through to send tarball the default method.
    232         super(SiteClientLogger, self)._send_tarball(pkg_name, remote_dest)
    233 
    234 
    235 class _SiteRun(object):
    236     pass
    237