1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import tempfile 8 9 import common 10 from autotest_lib.client.bin import utils as common_utils 11 from autotest_lib.client.common_lib import error 12 from autotest_lib.client.common_lib.cros import dev_server 13 from autotest_lib.client.common_lib.cros import retry 14 from autotest_lib.server import utils as server_utils 15 from autotest_lib.site_utils.lxc import constants 16 17 try: 18 from chromite.lib import metrics 19 except ImportError: 20 metrics = common_utils.metrics_mock 21 22 23 def get_container_info(container_path, **filters): 24 """Get a collection of container information in the given container path. 25 26 This method parse the output of lxc-ls to get a list of container 27 information. The lxc-ls command output looks like: 28 NAME STATE IPV4 IPV6 AUTOSTART PID MEMORY RAM SWAP 29 -------------------------------------------------------------------------- 30 base STOPPED - - NO - - - - 31 test_123 RUNNING 10.0.3.27 - NO 8359 6.28MB 6.28MB 0.0MB 32 33 @param container_path: Path to look for containers. 34 @param filters: Key value to filter the containers, e.g., name='base' 35 36 @return: A list of dictionaries that each dictionary has the information of 37 a container. The keys are defined in ATTRIBUTES. 38 """ 39 cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path), 40 ','.join(constants.ATTRIBUTES)) 41 output = common_utils.run(cmd).stdout 42 info_collection = [] 43 44 for line in output.splitlines()[1:]: 45 # Only LXC 1.x has the second line of '-' as a separator. 46 if line.startswith('------'): 47 continue 48 info_collection.append(dict(zip(constants.ATTRIBUTES, line.split()))) 49 if filters: 50 filtered_collection = [] 51 for key, value in filters.iteritems(): 52 for info in info_collection: 53 if key in info and info[key] == value: 54 filtered_collection.append(info) 55 info_collection = filtered_collection 56 return info_collection 57 58 59 def download_extract(url, target, extract_dir): 60 """Download the file from given url and save it to the target, then extract. 61 62 @param url: Url to download the file. 63 @param target: Path of the file to save to. 64 @param extract_dir: Directory to extract the content of the file to. 65 """ 66 remote_url = dev_server.DevServer.get_server_url(url) 67 # This can be run in multiple threads, pick a unique tmp_file.name. 68 with tempfile.NamedTemporaryFile(prefix=os.path.basename(target) + '_', 69 delete=False) as tmp_file: 70 if remote_url in dev_server.ImageServerBase.servers(): 71 # TODO(xixuan): Better to only ssh to devservers in lab, and 72 # continue using curl for ganeti devservers. 73 _download_via_devserver(url, tmp_file.name) 74 else: 75 _download_via_curl(url, tmp_file.name) 76 common_utils.run('sudo mv %s %s' % (tmp_file.name, target)) 77 common_utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir)) 78 79 80 # Make sure retries only happen in the non-timeout case. 81 @retry.retry((error.CmdError), 82 blacklist=[error.CmdTimeoutError], 83 timeout_min=3*2, 84 delay_sec=10) 85 def _download_via_curl(url, target_file_path): 86 # We do not want to retry on CmdTimeoutError but still retry on 87 # CmdError. Hence we can't use curl --timeout=... 88 common_utils.run('sudo curl -s %s -o %s' % (url, target_file_path), 89 stderr_tee=common_utils.TEE_TO_LOGS, timeout=3*60) 90 91 92 # Make sure retries only happen in the non-timeout case. 93 @retry.retry((error.CmdError), 94 blacklist=[error.CmdTimeoutError], 95 timeout_min=(constants.DEVSERVER_CALL_TIMEOUT * 96 constants.DEVSERVER_CALL_RETRY / 60), 97 delay_sec=constants.DEVSERVER_CALL_DELAY) 98 def _download_via_devserver(url, target_file_path): 99 dev_server.ImageServerBase.download_file( 100 url, target_file_path, timeout=constants.DEVSERVER_CALL_TIMEOUT) 101 102 103 def _install_package_precheck(packages): 104 """If SSP is not enabled or the test is running in chroot (using test_that), 105 packages installation should be skipped. 106 107 The check does not raise exception so tests started by test_that or running 108 in an Autotest setup with SSP disabled can continue. That assume the running 109 environment, chroot or a machine, has the desired packages installed 110 already. 111 112 @param packages: A list of names of the packages to install. 113 114 @return: True if package installation can continue. False if it should be 115 skipped. 116 117 """ 118 if not constants.SSP_ENABLED and not common_utils.is_in_container(): 119 logging.info('Server-side packaging is not enabled. Install package %s ' 120 'is skipped.', packages) 121 return False 122 123 if server_utils.is_inside_chroot(): 124 logging.info('Test is running inside chroot. Install package %s is ' 125 'skipped.', packages) 126 return False 127 128 if not common_utils.is_in_container(): 129 raise error.ContainerError('Package installation is only supported ' 130 'when test is running inside container.') 131 132 return True 133 134 135 @metrics.SecondsTimerDecorator( 136 '%s/install_packages_duration' % constants.STATS_KEY) 137 @retry.retry(error.CmdError, timeout_min=30) 138 def install_packages(packages=[], python_packages=[], force_latest=False): 139 """Install the given package inside container. 140 141 !!! WARNING !!! 142 This call may introduce several minutes of delay in test run. The best way 143 to avoid such delay is to update the base container used for the test run. 144 File a bug for infra deputy to update the base container with the new 145 package a test requires. 146 147 @param packages: A list of names of the packages to install. 148 @param python_packages: A list of names of the python packages to install 149 using pip. 150 @param force_latest: True to force to install the latest version of the 151 package. Default to False, which means skip installing 152 the package if it's installed already, even with an old 153 version. 154 155 @raise error.ContainerError: If package is attempted to be installed outside 156 a container. 157 @raise error.CmdError: If the package doesn't exist or failed to install. 158 159 """ 160 if not _install_package_precheck(packages or python_packages): 161 return 162 163 # If force_latest is False, only install packages that are not already 164 # installed. 165 if not force_latest: 166 packages = [p for p in packages 167 if not common_utils.is_package_installed(p)] 168 python_packages = [p for p in python_packages 169 if not common_utils.is_python_package_installed(p)] 170 if not packages and not python_packages: 171 logging.debug('All packages are installed already, skip reinstall.') 172 return 173 174 # Always run apt-get update before installing any container. The base 175 # container may have outdated cache. 176 common_utils.run('sudo apt-get update') 177 # Make sure the lists are not None for iteration. 178 packages = [] if not packages else packages 179 if python_packages: 180 packages.extend(['python-pip', 'python-dev']) 181 if packages: 182 common_utils.run( 183 'sudo DEBIAN_FRONTEND=noninteractive apt-get install %s -y ' 184 '--force-yes' % ' '.join(packages)) 185 logging.debug('Packages are installed: %s.', packages) 186 187 target_setting = '' 188 # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/ 189 # is a readonly mount from the host. Therefore, new python modules have to 190 # be installed in /usr/lib/python2.7/dist-packages/ 191 # Containers created in Moblab does not have autotest/site-packages folder. 192 if not os.path.exists('/usr/local/autotest/site-packages'): 193 target_setting = '--target="/usr/lib/python2.7/dist-packages/"' 194 if python_packages: 195 common_utils.run('sudo pip install %s %s' % (target_setting, 196 ' '.join(python_packages))) 197 logging.debug('Python packages are installed: %s.', python_packages) 198 199 200 @retry.retry(error.CmdError, timeout_min=20) 201 def install_package(package): 202 """Install the given package inside container. 203 204 This function is kept for backwards compatibility reason. New code should 205 use function install_packages for better performance. 206 207 @param package: Name of the package to install. 208 209 @raise error.ContainerError: If package is attempted to be installed outside 210 a container. 211 @raise error.CmdError: If the package doesn't exist or failed to install. 212 213 """ 214 logging.warn('This function is obsoleted, please use install_packages ' 215 'instead.') 216 install_packages(packages=[package]) 217 218 219 @retry.retry(error.CmdError, timeout_min=20) 220 def install_python_package(package): 221 """Install the given python package inside container using pip. 222 223 This function is kept for backwards compatibility reason. New code should 224 use function install_packages for better performance. 225 226 @param package: Name of the python package to install. 227 228 @raise error.CmdError: If the package doesn't exist or failed to install. 229 """ 230 logging.warn('This function is obsoleted, please use install_packages ' 231 'instead.') 232 install_packages(python_packages=[package]) 233
