1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import tempfile 8 9 import common 10 from autotest_lib.client.bin import utils as common_utils 11 from autotest_lib.client.common_lib import error 12 from autotest_lib.client.common_lib.cros import dev_server 13 from autotest_lib.client.common_lib.cros import retry 14 from autotest_lib.server import utils as server_utils 15 from autotest_lib.site_utils.lxc import constants 16 from autotest_lib.site_utils.lxc import utils as lxc_utils 17 18 try: 19 from chromite.lib import metrics 20 except ImportError: 21 metrics = common_utils.metrics_mock 22 23 24 def _get_container_info_moblab(container_path, **filters): 25 """Get a collection of container information in the given container path 26 in a Moblab. 27 28 TODO(crbug.com/457496): remove this method once python 3 can be installed 29 in Moblab and lxc-ls command can use python 3 code. 30 31 When running in Moblab, lxc-ls behaves differently from a server with python 32 3 installed: 33 1. lxc-ls returns a list of containers installed under /etc/lxc, the default 34 lxc container directory. 35 2. lxc-ls --active lists all active containers, regardless where the 36 container is located. 37 For such differences, we have to special case Moblab to make the behavior 38 close to a server with python 3 installed. That is, 39 1. List only containers in a given folder. 40 2. Assume all active containers have state of RUNNING. 41 42 @param container_path: Path to look for containers. 43 @param filters: Key value to filter the containers, e.g., name='base' 44 45 @return: A list of dictionaries that each dictionary has the information of 46 a container. The keys are defined in ATTRIBUTES. 47 """ 48 info_collection = [] 49 active_containers = common_utils.run('sudo lxc-ls --active').stdout.split() 50 name_filter = filters.get('name', None) 51 state_filter = filters.get('state', None) 52 if filters and set(filters.keys()) - set(['name', 'state']): 53 raise error.ContainerError('When running in Moblab, container list ' 54 'filter only supports name and state.') 55 56 for name in os.listdir(container_path): 57 # Skip all files and folders without rootfs subfolder. 58 if (os.path.isfile(os.path.join(container_path, name)) or 59 not lxc_utils.path_exists(os.path.join(container_path, name, 60 'rootfs'))): 61 continue 62 info = {'name': name, 63 'state': 'RUNNING' if name in active_containers else 'STOPPED' 64 } 65 if ((name_filter and name_filter != info['name']) or 66 (state_filter and state_filter != info['state'])): 67 continue 68 69 info_collection.append(info) 70 return info_collection 71 72 73 def get_container_info(container_path, **filters): 74 """Get a collection of container information in the given container path. 75 76 This method parse the output of lxc-ls to get a list of container 77 information. The lxc-ls command output looks like: 78 NAME STATE IPV4 IPV6 AUTOSTART PID MEMORY RAM SWAP 79 -------------------------------------------------------------------------- 80 base STOPPED - - NO - - - - 81 test_123 RUNNING 10.0.3.27 - NO 8359 6.28MB 6.28MB 0.0MB 82 83 @param container_path: Path to look for containers. 84 @param filters: Key value to filter the containers, e.g., name='base' 85 86 @return: A list of dictionaries that each dictionary has the information of 87 a container. The keys are defined in ATTRIBUTES. 88 """ 89 if constants.IS_MOBLAB: 90 return _get_container_info_moblab(container_path, **filters) 91 92 cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path), 93 ','.join(constants.ATTRIBUTES)) 94 output = common_utils.run(cmd).stdout 95 info_collection = [] 96 97 for line in output.splitlines()[1:]: 98 # Only LXC 1.x has the second line of '-' as a separator. 99 if line.startswith('------'): 100 continue 101 info_collection.append(dict(zip(constants.ATTRIBUTES, line.split()))) 102 if filters: 103 filtered_collection = [] 104 for key, value in filters.iteritems(): 105 for info in info_collection: 106 if key in info and info[key] == value: 107 filtered_collection.append(info) 108 info_collection = filtered_collection 109 return info_collection 110 111 112 # Make sure retries only happen in the non-timeout case. 113 @retry.retry((error.CmdError), 114 blacklist=[error.CmdTimeoutError], 115 timeout_min=(constants.DEVSERVER_CALL_TIMEOUT * 116 constants.DEVSERVER_CALL_RETRY / 60), 117 delay_sec=constants.DEVSERVER_CALL_DELAY) 118 def download_extract(url, target, extract_dir): 119 """Download the file from given url and save it to the target, then extract. 120 121 @param url: Url to download the file. 122 @param target: Path of the file to save to. 123 @param extract_dir: Directory to extract the content of the file to. 124 """ 125 remote_url = dev_server.DevServer.get_server_url(url) 126 # TODO(xixuan): Better to only ssh to devservers in lab, and continue using 127 # wget for ganeti devservers. 128 if remote_url in dev_server.ImageServerBase.servers(): 129 # This can be run in multiple threads, pick a unique tmp_file.name. 130 with tempfile.NamedTemporaryFile(prefix=os.path.basename(target) + '_', 131 delete=False) as tmp_file: 132 dev_server.ImageServerBase.download_file( 133 url, 134 tmp_file.name, 135 timeout=constants.DEVSERVER_CALL_TIMEOUT) 136 common_utils.run('sudo mv %s %s' % (tmp_file.name, target)) 137 else: 138 # We do not want to retry on CmdTimeoutError but still retry on 139 # CmdError. Hence we can't use wget --timeout=... 140 common_utils.run('sudo wget -nv %s -O %s' % (url, target), 141 stderr_tee=common_utils.TEE_TO_LOGS, 142 timeout=constants.DEVSERVER_CALL_TIMEOUT) 143 144 common_utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir)) 145 146 147 def _install_package_precheck(packages): 148 """If SSP is not enabled or the test is running in chroot (using test_that), 149 packages installation should be skipped. 150 151 The check does not raise exception so tests started by test_that or running 152 in an Autotest setup with SSP disabled can continue. That assume the running 153 environment, chroot or a machine, has the desired packages installed 154 already. 155 156 @param packages: A list of names of the packages to install. 157 158 @return: True if package installation can continue. False if it should be 159 skipped. 160 161 """ 162 if not constants.SSP_ENABLED and not common_utils.is_in_container(): 163 logging.info('Server-side packaging is not enabled. Install package %s ' 164 'is skipped.', packages) 165 return False 166 167 if server_utils.is_inside_chroot(): 168 logging.info('Test is running inside chroot. Install package %s is ' 169 'skipped.', packages) 170 return False 171 172 if not common_utils.is_in_container(): 173 raise error.ContainerError('Package installation is only supported ' 174 'when test is running inside container.') 175 176 return True 177 178 179 @metrics.SecondsTimerDecorator( 180 '%s/install_packages_duration' % constants.STATS_KEY) 181 @retry.retry(error.CmdError, timeout_min=30) 182 def install_packages(packages=[], python_packages=[], force_latest=False): 183 """Install the given package inside container. 184 185 !!! WARNING !!! 186 This call may introduce several minutes of delay in test run. The best way 187 to avoid such delay is to update the base container used for the test run. 188 File a bug for infra deputy to update the base container with the new 189 package a test requires. 190 191 @param packages: A list of names of the packages to install. 192 @param python_packages: A list of names of the python packages to install 193 using pip. 194 @param force_latest: True to force to install the latest version of the 195 package. Default to False, which means skip installing 196 the package if it's installed already, even with an old 197 version. 198 199 @raise error.ContainerError: If package is attempted to be installed outside 200 a container. 201 @raise error.CmdError: If the package doesn't exist or failed to install. 202 203 """ 204 if not _install_package_precheck(packages or python_packages): 205 return 206 207 # If force_latest is False, only install packages that are not already 208 # installed. 209 if not force_latest: 210 packages = [p for p in packages 211 if not common_utils.is_package_installed(p)] 212 python_packages = [p for p in python_packages 213 if not common_utils.is_python_package_installed(p)] 214 if not packages and not python_packages: 215 logging.debug('All packages are installed already, skip reinstall.') 216 return 217 218 # Always run apt-get update before installing any container. The base 219 # container may have outdated cache. 220 common_utils.run('sudo apt-get update') 221 # Make sure the lists are not None for iteration. 222 packages = [] if not packages else packages 223 if python_packages: 224 packages.extend(['python-pip', 'python-dev']) 225 if packages: 226 common_utils.run( 227 'sudo apt-get install %s -y --force-yes' % ' '.join(packages)) 228 logging.debug('Packages are installed: %s.', packages) 229 230 target_setting = '' 231 # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/ 232 # is a readonly mount from the host. Therefore, new python modules have to 233 # be installed in /usr/lib/python2.7/dist-packages/ 234 # Containers created in Moblab does not have autotest/site-packages folder. 235 if not os.path.exists('/usr/local/autotest/site-packages'): 236 target_setting = '--target="/usr/lib/python2.7/dist-packages/"' 237 if python_packages: 238 common_utils.run('sudo pip install %s %s' % (target_setting, 239 ' '.join(python_packages))) 240 logging.debug('Python packages are installed: %s.', python_packages) 241 242 243 @retry.retry(error.CmdError, timeout_min=20) 244 def install_package(package): 245 """Install the given package inside container. 246 247 This function is kept for backwards compatibility reason. New code should 248 use function install_packages for better performance. 249 250 @param package: Name of the package to install. 251 252 @raise error.ContainerError: If package is attempted to be installed outside 253 a container. 254 @raise error.CmdError: If the package doesn't exist or failed to install. 255 256 """ 257 logging.warn('This function is obsoleted, please use install_packages ' 258 'instead.') 259 install_packages(packages=[package]) 260 261 262 @retry.retry(error.CmdError, timeout_min=20) 263 def install_python_package(package): 264 """Install the given python package inside container using pip. 265 266 This function is kept for backwards compatibility reason. New code should 267 use function install_packages for better performance. 268 269 @param package: Name of the python package to install. 270 271 @raise error.CmdError: If the package doesn't exist or failed to install. 272 """ 273 logging.warn('This function is obsoleted, please use install_packages ' 274 'instead.') 275 install_packages(python_packages=[package]) 276
