1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import tempfile 8 9 import common 10 from autotest_lib.client.bin import utils as common_utils 11 from autotest_lib.client.common_lib import error 12 from autotest_lib.client.common_lib.cros import dev_server 13 from autotest_lib.client.common_lib.cros import retry 14 from autotest_lib.server import utils as server_utils 15 from autotest_lib.site_utils.lxc import constants 16 from autotest_lib.site_utils.lxc import utils as lxc_utils 17 18 try: 19 from chromite.lib import metrics 20 except ImportError: 21 metrics = common_utils.metrics_mock 22 23 24 def _get_container_info_moblab(container_path, **filters): 25 """Get a collection of container information in the given container path 26 in a Moblab. 27 28 TODO(crbug.com/457496): remove this method once python 3 can be installed 29 in Moblab and lxc-ls command can use python 3 code. 30 31 When running in Moblab, lxc-ls behaves differently from a server with python 32 3 installed: 33 1. lxc-ls returns a list of containers installed under /etc/lxc, the default 34 lxc container directory. 35 2. lxc-ls --active lists all active containers, regardless where the 36 container is located. 37 For such differences, we have to special case Moblab to make the behavior 38 close to a server with python 3 installed. That is, 39 1. List only containers in a given folder. 40 2. Assume all active containers have state of RUNNING. 41 42 @param container_path: Path to look for containers. 43 @param filters: Key value to filter the containers, e.g., name='base' 44 45 @return: A list of dictionaries that each dictionary has the information of 46 a container. The keys are defined in ATTRIBUTES. 47 """ 48 info_collection = [] 49 active_containers = common_utils.run('sudo lxc-ls --active').stdout.split() 50 name_filter = filters.get('name', None) 51 state_filter = filters.get('state', None) 52 if filters and set(filters.keys()) - set(['name', 'state']): 53 raise error.ContainerError('When running in Moblab, container list ' 54 'filter only supports name and state.') 55 56 for name in os.listdir(container_path): 57 # Skip all files and folders without rootfs subfolder. 58 if (os.path.isfile(os.path.join(container_path, name)) or 59 not lxc_utils.path_exists(os.path.join(container_path, name, 60 'rootfs'))): 61 continue 62 info = {'name': name, 63 'state': 'RUNNING' if name in active_containers else 'STOPPED' 64 } 65 if ((name_filter and name_filter != info['name']) or 66 (state_filter and state_filter != info['state'])): 67 continue 68 69 info_collection.append(info) 70 return info_collection 71 72 73 def get_container_info(container_path, **filters): 74 """Get a collection of container information in the given container path. 75 76 This method parse the output of lxc-ls to get a list of container 77 information. The lxc-ls command output looks like: 78 NAME STATE IPV4 IPV6 AUTOSTART PID MEMORY RAM SWAP 79 -------------------------------------------------------------------------- 80 base STOPPED - - NO - - - - 81 test_123 RUNNING 10.0.3.27 - NO 8359 6.28MB 6.28MB 0.0MB 82 83 @param container_path: Path to look for containers. 84 @param filters: Key value to filter the containers, e.g., name='base' 85 86 @return: A list of dictionaries that each dictionary has the information of 87 a container. The keys are defined in ATTRIBUTES. 88 """ 89 if constants.IS_MOBLAB: 90 return _get_container_info_moblab(container_path, **filters) 91 92 cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path), 93 ','.join(constants.ATTRIBUTES)) 94 output = common_utils.run(cmd).stdout 95 info_collection = [] 96 97 for line in output.splitlines()[1:]: 98 # Only LXC 1.x has the second line of '-' as a separator. 99 if line.startswith('------'): 100 continue 101 info_collection.append(dict(zip(constants.ATTRIBUTES, line.split()))) 102 if filters: 103 filtered_collection = [] 104 for key, value in filters.iteritems(): 105 for info in info_collection: 106 if key in info and info[key] == value: 107 filtered_collection.append(info) 108 info_collection = filtered_collection 109 return info_collection 110 111 112 def download_extract(url, target, extract_dir): 113 """Download the file from given url and save it to the target, then extract. 114 115 @param url: Url to download the file. 116 @param target: Path of the file to save to. 117 @param extract_dir: Directory to extract the content of the file to. 118 """ 119 remote_url = dev_server.DevServer.get_server_url(url) 120 # This can be run in multiple threads, pick a unique tmp_file.name. 121 with tempfile.NamedTemporaryFile(prefix=os.path.basename(target) + '_', 122 delete=False) as tmp_file: 123 if remote_url in dev_server.ImageServerBase.servers(): 124 # TODO(xixuan): Better to only ssh to devservers in lab, and 125 # continue using curl for ganeti devservers. 126 _download_via_devserver(url, tmp_file.name) 127 else: 128 _download_via_curl(url, tmp_file.name) 129 common_utils.run('sudo mv %s %s' % (tmp_file.name, target)) 130 common_utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir)) 131 132 133 # Make sure retries only happen in the non-timeout case. 134 @retry.retry((error.CmdError), 135 blacklist=[error.CmdTimeoutError], 136 timeout_min=3*2, 137 delay_sec=10) 138 def _download_via_curl(url, target_file_path): 139 # We do not want to retry on CmdTimeoutError but still retry on 140 # CmdError. Hence we can't use curl --timeout=... 141 common_utils.run('sudo curl -s %s -o %s' % (url, target_file_path), 142 stderr_tee=common_utils.TEE_TO_LOGS, timeout=3*60) 143 144 145 # Make sure retries only happen in the non-timeout case. 146 @retry.retry((error.CmdError), 147 blacklist=[error.CmdTimeoutError], 148 timeout_min=(constants.DEVSERVER_CALL_TIMEOUT * 149 constants.DEVSERVER_CALL_RETRY / 60), 150 delay_sec=constants.DEVSERVER_CALL_DELAY) 151 def _download_via_devserver(url, target_file_path): 152 dev_server.ImageServerBase.download_file( 153 url, target_file_path, timeout=constants.DEVSERVER_CALL_TIMEOUT) 154 155 156 def _install_package_precheck(packages): 157 """If SSP is not enabled or the test is running in chroot (using test_that), 158 packages installation should be skipped. 159 160 The check does not raise exception so tests started by test_that or running 161 in an Autotest setup with SSP disabled can continue. That assume the running 162 environment, chroot or a machine, has the desired packages installed 163 already. 164 165 @param packages: A list of names of the packages to install. 166 167 @return: True if package installation can continue. False if it should be 168 skipped. 169 170 """ 171 if not constants.SSP_ENABLED and not common_utils.is_in_container(): 172 logging.info('Server-side packaging is not enabled. Install package %s ' 173 'is skipped.', packages) 174 return False 175 176 if server_utils.is_inside_chroot(): 177 logging.info('Test is running inside chroot. Install package %s is ' 178 'skipped.', packages) 179 return False 180 181 if not common_utils.is_in_container(): 182 raise error.ContainerError('Package installation is only supported ' 183 'when test is running inside container.') 184 185 return True 186 187 188 @metrics.SecondsTimerDecorator( 189 '%s/install_packages_duration' % constants.STATS_KEY) 190 @retry.retry(error.CmdError, timeout_min=30) 191 def install_packages(packages=[], python_packages=[], force_latest=False): 192 """Install the given package inside container. 193 194 !!! WARNING !!! 195 This call may introduce several minutes of delay in test run. The best way 196 to avoid such delay is to update the base container used for the test run. 197 File a bug for infra deputy to update the base container with the new 198 package a test requires. 199 200 @param packages: A list of names of the packages to install. 201 @param python_packages: A list of names of the python packages to install 202 using pip. 203 @param force_latest: True to force to install the latest version of the 204 package. Default to False, which means skip installing 205 the package if it's installed already, even with an old 206 version. 207 208 @raise error.ContainerError: If package is attempted to be installed outside 209 a container. 210 @raise error.CmdError: If the package doesn't exist or failed to install. 211 212 """ 213 if not _install_package_precheck(packages or python_packages): 214 return 215 216 # If force_latest is False, only install packages that are not already 217 # installed. 218 if not force_latest: 219 packages = [p for p in packages 220 if not common_utils.is_package_installed(p)] 221 python_packages = [p for p in python_packages 222 if not common_utils.is_python_package_installed(p)] 223 if not packages and not python_packages: 224 logging.debug('All packages are installed already, skip reinstall.') 225 return 226 227 # Always run apt-get update before installing any container. The base 228 # container may have outdated cache. 229 common_utils.run('sudo apt-get update') 230 # Make sure the lists are not None for iteration. 231 packages = [] if not packages else packages 232 if python_packages: 233 packages.extend(['python-pip', 'python-dev']) 234 if packages: 235 common_utils.run( 236 'sudo apt-get install %s -y --force-yes' % ' '.join(packages)) 237 logging.debug('Packages are installed: %s.', packages) 238 239 target_setting = '' 240 # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/ 241 # is a readonly mount from the host. Therefore, new python modules have to 242 # be installed in /usr/lib/python2.7/dist-packages/ 243 # Containers created in Moblab does not have autotest/site-packages folder. 244 if not os.path.exists('/usr/local/autotest/site-packages'): 245 target_setting = '--target="/usr/lib/python2.7/dist-packages/"' 246 if python_packages: 247 common_utils.run('sudo pip install %s %s' % (target_setting, 248 ' '.join(python_packages))) 249 logging.debug('Python packages are installed: %s.', python_packages) 250 251 252 @retry.retry(error.CmdError, timeout_min=20) 253 def install_package(package): 254 """Install the given package inside container. 255 256 This function is kept for backwards compatibility reason. New code should 257 use function install_packages for better performance. 258 259 @param package: Name of the package to install. 260 261 @raise error.ContainerError: If package is attempted to be installed outside 262 a container. 263 @raise error.CmdError: If the package doesn't exist or failed to install. 264 265 """ 266 logging.warn('This function is obsoleted, please use install_packages ' 267 'instead.') 268 install_packages(packages=[package]) 269 270 271 @retry.retry(error.CmdError, timeout_min=20) 272 def install_python_package(package): 273 """Install the given python package inside container using pip. 274 275 This function is kept for backwards compatibility reason. New code should 276 use function install_packages for better performance. 277 278 @param package: Name of the python package to install. 279 280 @raise error.CmdError: If the package doesn't exist or failed to install. 281 """ 282 logging.warn('This function is obsoleted, please use install_packages ' 283 'instead.') 284 install_packages(python_packages=[package]) 285
