1 # Copyright 2017 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import time 8 9 import common 10 from autotest_lib.client.bin import utils 11 from autotest_lib.client.common_lib import error 12 from autotest_lib.site_utils.lxc import Container 13 from autotest_lib.site_utils.lxc import config as lxc_config 14 from autotest_lib.site_utils.lxc import constants 15 from autotest_lib.site_utils.lxc import lxc 16 from autotest_lib.site_utils.lxc import utils as lxc_utils 17 from autotest_lib.site_utils.lxc.cleanup_if_fail import cleanup_if_fail 18 19 try: 20 from chromite.lib import metrics 21 except ImportError: 22 metrics = utils.metrics_mock 23 24 25 class ContainerBucket(object): 26 """A wrapper class to interact with containers in a specific container path. 27 """ 28 29 def __init__(self, 30 container_path=constants.DEFAULT_CONTAINER_PATH, 31 shared_host_path = constants.DEFAULT_SHARED_HOST_PATH): 32 """Initialize a ContainerBucket. 33 34 @param container_path: Path to the directory used to store containers. 35 Default is set to AUTOSERV/container_path in 36 global config. 37 """ 38 self.container_path = os.path.realpath(container_path) 39 self.shared_host_path = os.path.realpath(shared_host_path) 40 # Try to create the base container. 41 try: 42 base_container = Container.createFromExistingDir( 43 container_path, constants.BASE); 44 base_container.refresh_status() 45 self.base_container = base_container 46 except error.ContainerError: 47 self.base_container = None 48 49 50 def get_all(self): 51 """Get details of all containers. 52 53 @return: A dictionary of all containers with detailed attributes, 54 indexed by container name. 55 """ 56 info_collection = lxc.get_container_info(self.container_path) 57 containers = {} 58 for info in info_collection: 59 container = Container.createFromExistingDir(self.container_path, 60 **info) 61 containers[container.name] = container 62 return containers 63 64 65 def get(self, name): 66 """Get a container with matching name. 67 68 @param name: Name of the container. 69 70 @return: A container object with matching name. Returns None if no 71 container matches the given name. 72 """ 73 return self.get_all().get(name, None) 74 75 76 def exist(self, name): 77 """Check if a container exists with the given name. 78 79 @param name: Name of the container. 80 81 @return: True if the container with the given name exists, otherwise 82 returns False. 83 """ 84 return self.get(name) != None 85 86 87 def destroy_all(self): 88 """Destroy all containers, base must be destroyed at the last. 89 """ 90 containers = self.get_all().values() 91 for container in sorted( 92 containers, key=lambda n: 1 if n.name == constants.BASE else 0): 93 logging.info('Destroy container %s.', container.name) 94 container.destroy() 95 self._cleanup_shared_host_path() 96 97 98 @metrics.SecondsTimerDecorator( 99 '%s/create_from_base_duration' % constants.STATS_KEY) 100 def create_from_base(self, name, disable_snapshot_clone=False, 101 force_cleanup=False): 102 """Create a container from the base container. 103 104 @param name: Name of the container. 105 @param disable_snapshot_clone: Set to True to force to clone without 106 using snapshot clone even if the host supports that. 107 @param force_cleanup: Force to cleanup existing container. 108 109 @return: A Container object for the created container. 110 111 @raise ContainerError: If the container already exist. 112 @raise error.CmdError: If lxc-clone call failed for any reason. 113 """ 114 if self.exist(name) and not force_cleanup: 115 raise error.ContainerError('Container %s already exists.' % name) 116 117 use_snapshot = (constants.SUPPORT_SNAPSHOT_CLONE and not 118 disable_snapshot_clone) 119 120 try: 121 return Container.clone(src=self.base_container, 122 new_name=name, 123 new_path=self.container_path, 124 snapshot=use_snapshot, 125 cleanup=force_cleanup) 126 except error.CmdError: 127 logging.debug('Creating snapshot clone failed. Attempting without ' 128 'snapshot...') 129 if not use_snapshot: 130 raise 131 else: 132 # Snapshot clone failed, retry clone without snapshot. 133 container = Container.clone(src=self.base_container, 134 new_name=name, 135 new_path=self.container_path, 136 snapshot=False, 137 cleanup=force_cleanup) 138 return container 139 140 141 @cleanup_if_fail() 142 def setup_base(self, name=constants.BASE, force_delete=False): 143 """Setup base container. 144 145 @param name: Name of the base container, default to base. 146 @param force_delete: True to force to delete existing base container. 147 This action will destroy all running test 148 containers. Default is set to False. 149 """ 150 if not self.container_path: 151 raise error.ContainerError( 152 'You must set a valid directory to store containers in ' 153 'global config "AUTOSERV/ container_path".') 154 155 if not os.path.exists(self.container_path): 156 os.makedirs(self.container_path) 157 158 base_path = os.path.join(self.container_path, name) 159 if self.exist(name) and not force_delete: 160 logging.error( 161 'Base container already exists. Set force_delete to True ' 162 'to force to re-stage base container. Note that this ' 163 'action will destroy all running test containers') 164 # Set proper file permission. base container in moblab may have 165 # owner of not being root. Force to update the folder's owner. 166 # TODO(dshi): Change root to current user when test container can be 167 # unprivileged container. 168 utils.run('sudo chown -R root "%s"' % base_path) 169 utils.run('sudo chgrp -R root "%s"' % base_path) 170 return 171 172 # Destroy existing base container if exists. 173 if self.exist(name): 174 # TODO: We may need to destroy all snapshots created from this base 175 # container, not all container. 176 self.destroy_all() 177 178 # Download and untar the base container. 179 tar_path = os.path.join(self.container_path, '%s.tar.xz' % name) 180 path_to_cleanup = [tar_path, base_path] 181 for path in path_to_cleanup: 182 if os.path.exists(path): 183 utils.run('sudo rm -rf "%s"' % path) 184 container_url = constants.CONTAINER_BASE_URL_FMT % name 185 lxc.download_extract(container_url, tar_path, self.container_path) 186 # Remove the downloaded container tar file. 187 utils.run('sudo rm "%s"' % tar_path) 188 # Set proper file permission. 189 # TODO(dshi): Change root to current user when test container can be 190 # unprivileged container. 191 utils.run('sudo chown -R root "%s"' % base_path) 192 utils.run('sudo chgrp -R root "%s"' % base_path) 193 194 # Update container config with container_path from global config. 195 config_path = os.path.join(base_path, 'config') 196 rootfs_path = os.path.join(base_path, 'rootfs') 197 utils.run(('sudo sed ' 198 '-i "s|\(lxc\.rootfs[[:space:]]*=\).*$|\\1 {rootfs}|" ' 199 '"{config}"').format(rootfs=rootfs_path, 200 config=config_path)) 201 202 self.base_container = Container.createFromExistingDir( 203 self.container_path, name) 204 205 self._setup_shared_host_path() 206 207 208 def _setup_shared_host_path(self): 209 """Sets up the shared host directory.""" 210 # First, clear out the old shared host dir if it exists. 211 if lxc_utils.path_exists(self.shared_host_path): 212 self._cleanup_shared_host_path() 213 # Create the dir and set it up as a shared mount point. 214 utils.run(('sudo mkdir "{path}" && ' 215 'sudo mount --bind "{path}" "{path}" && ' 216 'sudo mount --make-unbindable "{path}" && ' 217 'sudo mount --make-shared "{path}"') 218 .format(path=self.shared_host_path)) 219 220 221 def _cleanup_shared_host_path(self): 222 """Removes the shared host directory. 223 224 This should only be called after all containers have been destroyed 225 (i.e. all host mounts have been disconnected and removed, so the shared 226 host directory should be empty). 227 """ 228 if not os.path.exists(self.shared_host_path): 229 return 230 231 if len(os.listdir(self.shared_host_path)) > 0: 232 raise RuntimeError('Attempting to clean up host dir before all ' 233 'hosts have been disconnected') 234 utils.run('sudo umount "{path}" && sudo rmdir "{path}"' 235 .format(path=self.shared_host_path)) 236 237 238 @metrics.SecondsTimerDecorator( 239 '%s/setup_test_duration' % constants.STATS_KEY) 240 @cleanup_if_fail() 241 def setup_test(self, name, job_id, server_package_url, result_path, 242 control=None, skip_cleanup=False, job_folder=None, 243 dut_name=None): 244 """Setup test container for the test job to run. 245 246 The setup includes: 247 1. Install autotest_server package from given url. 248 2. Copy over local shadow_config.ini. 249 3. Mount local site-packages. 250 4. Mount test result directory. 251 252 TODO(dshi): Setup also needs to include test control file for autoserv 253 to run in container. 254 255 @param name: Name of the container. 256 @param job_id: Job id for the test job to run in the test container. 257 @param server_package_url: Url to download autotest_server package. 258 @param result_path: Directory to be mounted to container to store test 259 results. 260 @param control: Path to the control file to run the test job. Default is 261 set to None. 262 @param skip_cleanup: Set to True to skip cleanup, used to troubleshoot 263 container failures. 264 @param job_folder: Folder name of the job, e.g., 123-debug_user. 265 @param dut_name: Name of the dut to run test, used as the hostname of 266 the container. Default is None. 267 @return: A Container object for the test container. 268 269 @raise ContainerError: If container does not exist, or not running. 270 """ 271 start_time = time.time() 272 273 if not os.path.exists(result_path): 274 raise error.ContainerError('Result directory does not exist: %s', 275 result_path) 276 result_path = os.path.abspath(result_path) 277 278 # Save control file to result_path temporarily. The reason is that the 279 # control file in drone_tmp folder can be deleted during scheduler 280 # restart. For test not using SSP, the window between test starts and 281 # control file being picked up by the test is very small (< 2 seconds). 282 # However, for tests using SSP, it takes around 1 minute before the 283 # container is setup. If scheduler is restarted during that period, the 284 # control file will be deleted, and the test will fail. 285 if control: 286 control_file_name = os.path.basename(control) 287 safe_control = os.path.join(result_path, control_file_name) 288 utils.run('cp %s %s' % (control, safe_control)) 289 290 # Create test container from the base container. 291 container = self.create_from_base(name) 292 293 # Update the hostname of the test container to be `dut-name`. 294 # Some TradeFed tests use hostname in test results, which is used to 295 # group test results in dashboard. The default container name is set to 296 # be the name of the folder, which is unique (as it is composed of job 297 # id and timestamp. For better result view, the container's hostname is 298 # set to be a string containing the dut hostname. 299 if dut_name: 300 container.set_hostname(dut_name.replace('.', '-')) 301 302 # Deploy server side package 303 container.install_ssp(server_package_url) 304 305 deploy_config_manager = lxc_config.DeployConfigManager(container) 306 deploy_config_manager.deploy_pre_start() 307 308 # Copy over control file to run the test job. 309 if control: 310 container.install_control_file(safe_control) 311 312 mount_entries = [(constants.SITE_PACKAGES_PATH, 313 constants.CONTAINER_SITE_PACKAGES_PATH, 314 True), 315 (os.path.join(common.autotest_dir, 'puppylab'), 316 os.path.join(constants.CONTAINER_AUTOTEST_DIR, 317 'puppylab'), 318 True), 319 (result_path, 320 os.path.join(constants.RESULT_DIR_FMT % job_folder), 321 False), 322 ] 323 324 # Update container config to mount directories. 325 for source, destination, readonly in mount_entries: 326 container.mount_dir(source, destination, readonly) 327 328 # Update file permissions. 329 # TODO(dshi): crbug.com/459344 Skip following action when test container 330 # can be unprivileged container. 331 autotest_path = os.path.join( 332 container.rootfs, 333 constants.CONTAINER_AUTOTEST_DIR.lstrip(os.path.sep)) 334 utils.run('sudo chown -R root "%s"' % autotest_path) 335 utils.run('sudo chgrp -R root "%s"' % autotest_path) 336 337 container.start(name) 338 deploy_config_manager.deploy_post_start() 339 340 container.modify_import_order() 341 342 container.verify_autotest_setup(job_folder) 343 344 logging.debug('Test container %s is set up.', name) 345 return container 346
