1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.targetprep; 18 19 import com.android.annotations.VisibleForTesting; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.command.remote.DeviceDescriptor; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 29 import com.android.tradefed.util.CommandResult; 30 import com.android.tradefed.util.CommandStatus; 31 import com.android.tradefed.util.FileUtil; 32 import com.android.tradefed.util.IRunUtil; 33 import com.android.tradefed.util.RunUtil; 34 import com.android.tradefed.util.VtsVendorConfigFileUtil; 35 36 import java.io.File; 37 import java.io.IOException; 38 import java.nio.file.FileVisitResult; 39 import java.nio.file.Files; 40 import java.nio.file.Path; 41 import java.nio.file.SimpleFileVisitor; 42 import java.nio.file.attribute.BasicFileAttributes; 43 import java.util.Arrays; 44 import java.util.Collection; 45 import java.util.NoSuchElementException; 46 import java.util.TreeSet; 47 48 /** 49 * Sets up a Python virtualenv on the host and installs packages. To activate it, the working 50 * directory is changed to the root of the virtualenv. 51 * 52 * This's a fork of PythonVirtualenvPreparer and is forked in order to simplify the change 53 * deployment process and reduce the deployment time, which are critical for VTS services. 54 * That means changes here will be upstreamed gradually. 55 */ 56 @OptionClass(alias = "python-venv") 57 public class VtsPythonVirtualenvPreparer implements IMultiTargetPreparer { 58 private static final String PIP = "pip"; 59 private static final String PATH = "PATH"; 60 private static final String OS_NAME = "os.name"; 61 private static final String WINDOWS = "Windows"; 62 private static final String LOCAL_PYPI_PATH_ENV_VAR_NAME = "VTS_PYPI_PATH"; 63 private static final String LOCAL_PYPI_PATH_KEY = "pypi_packages_path"; 64 protected static final String PYTHONPATH = "PYTHONPATH"; 65 protected static final String VIRTUAL_ENV_PATH = "VIRTUALENVPATH"; 66 private static final int BASE_TIMEOUT = 1000 * 60; 67 68 @Option(name = "venv-dir", description = "path of an existing virtualenv to use") 69 private File mVenvDir = null; 70 71 @Option(name = "requirements-file", description = "pip-formatted requirements file") 72 private File mRequirementsFile = null; 73 74 @Option(name = "script-file", description = "scripts which need to be executed in advance") 75 private Collection<String> mScriptFiles = new TreeSet<>(); 76 77 @Option(name = "dep-module", description = "modules which need to be installed by pip") 78 protected Collection<String> mDepModules = new TreeSet<>(); 79 80 @Option(name = "no-dep-module", description = "modules which should not be installed by pip") 81 private Collection<String> mNoDepModules = new TreeSet<>(Arrays.asList()); 82 83 @Option(name = "python-version", description = "The version of a Python interpreter to use.") 84 private String mPythonVersion = ""; 85 86 private IBuildInfo mBuildInfo = null; 87 private DeviceDescriptor mDescriptor = null; 88 private IRunUtil mRunUtil = new RunUtil(); 89 90 String mPip = PIP; 91 String mLocalPypiPath = null; 92 93 // Since we allow virtual env path to be reused during a test plan/module, only the preparer 94 // which created the directory should be the one to delete it. 95 private boolean mIsDirCreator = false; 96 97 // If the same object is used in multiple threads (in sharding mode), the class 98 // needs to know when it is safe to call the teardown method. 99 private int mNumOfInstances = 0; 100 101 /** 102 * {@inheritDoc} 103 */ 104 @Override 105 public synchronized void setUp(IInvocationContext context) 106 throws TargetSetupError, BuildError, DeviceNotAvailableException { 107 ++mNumOfInstances; 108 mBuildInfo = context.getBuildInfos().get(0); 109 if (mNumOfInstances == 1) { 110 ITestDevice device = context.getDevices().get(0); 111 mDescriptor = device.getDeviceDescriptor(); 112 createVirtualenv(mBuildInfo); 113 setLocalPypiPath(); 114 installDeps(); 115 } 116 addPathToBuild(mBuildInfo); 117 } 118 119 /** 120 * {@inheritDoc} 121 */ 122 @Override 123 public synchronized void tearDown(IInvocationContext context, Throwable e) 124 throws DeviceNotAvailableException { 125 --mNumOfInstances; 126 if (mNumOfInstances > 0) { 127 // Since this is a host side preparer, no need to repeat 128 return; 129 } 130 if (mVenvDir != null && mIsDirCreator) { 131 try { 132 recursiveDelete(mVenvDir.toPath()); 133 CLog.i("Deleted the virtual env's temp working dir, %s.", mVenvDir); 134 } catch (IOException exception) { 135 CLog.e("Failed to delete %s: %s", mVenvDir, exception); 136 } 137 mVenvDir = null; 138 } 139 } 140 141 /** 142 * This method sets mLocalPypiPath, the local PyPI package directory to 143 * install python packages from in the installDeps method. 144 */ 145 protected void setLocalPypiPath() { 146 VtsVendorConfigFileUtil configReader = new VtsVendorConfigFileUtil(); 147 if (configReader.LoadVendorConfig(mBuildInfo)) { 148 // First try to load local PyPI directory path from vendor config file 149 try { 150 String pypiPath = configReader.GetVendorConfigVariable(LOCAL_PYPI_PATH_KEY); 151 if (pypiPath.length() > 0 && dirExistsAndHaveReadAccess(pypiPath)) { 152 mLocalPypiPath = pypiPath; 153 CLog.i(String.format("Loaded %s: %s", LOCAL_PYPI_PATH_KEY, mLocalPypiPath)); 154 } 155 } catch (NoSuchElementException e) { 156 /* continue */ 157 } 158 } 159 160 // If loading path from vendor config file is unsuccessful, 161 // check local pypi path defined by LOCAL_PYPI_PATH_ENV_VAR_NAME 162 if (mLocalPypiPath == null) { 163 CLog.i("Checking whether local pypi packages directory exists"); 164 String pypiPath = System.getenv(LOCAL_PYPI_PATH_ENV_VAR_NAME); 165 if (pypiPath == null) { 166 CLog.i("Local pypi packages directory not specified by env var %s", 167 LOCAL_PYPI_PATH_ENV_VAR_NAME); 168 } else if (dirExistsAndHaveReadAccess(pypiPath)) { 169 mLocalPypiPath = pypiPath; 170 CLog.i("Set local pypi packages directory to %s", pypiPath); 171 } 172 } 173 174 if (mLocalPypiPath == null) { 175 CLog.i("Failed to set local pypi packages path. Therefore internet connection to " 176 + "https://pypi.python.org/simple/ must be available to run VTS tests."); 177 } 178 } 179 180 /** 181 * This method returns whether the given path is a dir that exists and the user has read access. 182 */ 183 private boolean dirExistsAndHaveReadAccess(String path) { 184 File pathDir = new File(path); 185 if (!pathDir.exists() || !pathDir.isDirectory()) { 186 CLog.i("Directory %s does not exist.", pathDir); 187 return false; 188 } 189 190 if (!isOnWindows()) { 191 CommandResult c = getRunUtil().runTimedCmd(BASE_TIMEOUT * 5, "ls", path); 192 if (c.getStatus() != CommandStatus.SUCCESS) { 193 CLog.i(String.format("Failed to read dir: %s. Result %s. stdout: %s, stderr: %s", 194 path, c.getStatus(), c.getStdout(), c.getStderr())); 195 return false; 196 } 197 return true; 198 } else { 199 try { 200 String[] pathDirList = pathDir.list(); 201 if (pathDirList == null) { 202 CLog.i("Failed to read dir: %s. Please check access permission.", pathDir); 203 return false; 204 } 205 } catch (SecurityException e) { 206 CLog.i(String.format( 207 "Failed to read dir %s with SecurityException %s", pathDir, e)); 208 return false; 209 } 210 return true; 211 } 212 } 213 214 protected void installDeps() throws TargetSetupError { 215 boolean hasDependencies = false; 216 if (!mScriptFiles.isEmpty()) { 217 for (String scriptFile : mScriptFiles) { 218 CLog.i("Attempting to execute a script, %s", scriptFile); 219 CommandResult c = getRunUtil().runTimedCmd(BASE_TIMEOUT * 5, scriptFile); 220 if (c.getStatus() != CommandStatus.SUCCESS) { 221 CLog.e("Executing script %s failed", scriptFile); 222 throw new TargetSetupError("Failed to source a script", mDescriptor); 223 } 224 } 225 } 226 if (mRequirementsFile != null) { 227 CommandResult c = getRunUtil().runTimedCmd( 228 BASE_TIMEOUT * 5, mPip, "install", "-r", mRequirementsFile.getAbsolutePath()); 229 if (!CommandStatus.SUCCESS.equals(c.getStatus())) { 230 CLog.e("Installing dependencies from %s failed with error: %s", 231 mRequirementsFile.getAbsolutePath(), c.getStderr()); 232 throw new TargetSetupError("Failed to install dependencies with pip", mDescriptor); 233 } 234 hasDependencies = true; 235 } 236 if (!mDepModules.isEmpty()) { 237 for (String dep : mDepModules) { 238 if (mNoDepModules.contains(dep)) { 239 continue; 240 } 241 CommandResult result = null; 242 if (mLocalPypiPath != null) { 243 CLog.i("Attempting installation of %s from local directory", dep); 244 result = getRunUtil().runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep, 245 "--no-index", "--find-links=" + mLocalPypiPath); 246 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 247 result.getStdout(), result.getStderr())); 248 if (result.getStatus() != CommandStatus.SUCCESS) { 249 CLog.e(String.format("Installing %s from %s failed", dep, mLocalPypiPath)); 250 } 251 } 252 if (mLocalPypiPath == null || result.getStatus() != CommandStatus.SUCCESS) { 253 CLog.i("Attempting installation of %s from PyPI", dep); 254 result = getRunUtil().runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep); 255 CLog.i("Result %s. stdout: %s, stderr: %s", result.getStatus(), 256 result.getStdout(), result.getStderr()); 257 if (result.getStatus() != CommandStatus.SUCCESS) { 258 CLog.e("Installing %s from PyPI failed.", dep); 259 CLog.i("Attempting to upgrade %s", dep); 260 result = getRunUtil().runTimedCmd( 261 BASE_TIMEOUT * 5, mPip, "install", "--upgrade", dep); 262 if (result.getStatus() != CommandStatus.SUCCESS) { 263 throw new TargetSetupError( 264 String.format("Failed to install dependencies with pip. " 265 + "Result %s. stdout: %s, stderr: %s", 266 result.getStatus(), result.getStdout(), 267 result.getStderr()), 268 mDescriptor); 269 } else { 270 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", 271 result.getStatus(), result.getStdout(), result.getStderr())); 272 } 273 } 274 } 275 hasDependencies = true; 276 } 277 } 278 if (!hasDependencies) { 279 CLog.i("No dependencies to install"); 280 } 281 } 282 283 /** 284 * Add PYTHONPATH and VIRTUAL_ENV_PATH to BuildInfo. 285 * @param buildInfo 286 */ 287 protected void addPathToBuild(IBuildInfo buildInfo) { 288 if (buildInfo.getFile(PYTHONPATH) == null) { 289 // make the install directory of new packages available to other classes that 290 // receive the build 291 buildInfo.setFile(PYTHONPATH, new File(mVenvDir, "local/lib/python2.7/site-packages"), 292 buildInfo.getBuildId()); 293 } 294 295 if (buildInfo.getFile(VIRTUAL_ENV_PATH) == null) { 296 buildInfo.setFile( 297 VIRTUAL_ENV_PATH, new File(mVenvDir.getAbsolutePath()), buildInfo.getBuildId()); 298 } 299 } 300 301 /** 302 * Create virtualenv directory by executing virtualenv command. 303 * @param buildInfo 304 * @throws TargetSetupError 305 */ 306 protected void createVirtualenv(IBuildInfo buildInfo) throws TargetSetupError { 307 if (mVenvDir == null) { 308 mVenvDir = buildInfo.getFile(VIRTUAL_ENV_PATH); 309 } 310 311 if (mVenvDir == null) { 312 CLog.i("Creating virtualenv"); 313 try { 314 mVenvDir = FileUtil.createTempDir(getMD5(buildInfo.getTestTag()) + "-virtualenv"); 315 mIsDirCreator = true; 316 String virtualEnvPath = mVenvDir.getAbsolutePath(); 317 CommandResult c; 318 if (mPythonVersion.length() == 0) { 319 c = getRunUtil().runTimedCmd(BASE_TIMEOUT, "virtualenv", virtualEnvPath); 320 } else { 321 String[] cmd = new String[] { 322 "virtualenv", "-p", "python" + mPythonVersion, virtualEnvPath}; 323 c = getRunUtil().runTimedCmd(BASE_TIMEOUT, cmd); 324 } 325 if (c.getStatus() != CommandStatus.SUCCESS) { 326 CLog.e(String.format("Failed to create virtualenv with : %s.", virtualEnvPath)); 327 throw new TargetSetupError("Failed to create virtualenv", mDescriptor); 328 } 329 } catch (IOException | RuntimeException e) { 330 CLog.e("Failed to create temp directory for virtualenv"); 331 throw new TargetSetupError("Error creating virtualenv", e, mDescriptor); 332 } 333 } 334 335 CLog.i("Python virtualenv path is: " + mVenvDir); 336 activate(); 337 } 338 339 /** 340 * This method returns a MD5 hash string for the given string. 341 */ 342 private String getMD5(String str) throws RuntimeException { 343 try { 344 java.security.MessageDigest md = java.security.MessageDigest.getInstance("MD5"); 345 byte[] array = md.digest(str.getBytes()); 346 StringBuffer sb = new StringBuffer(); 347 for (int i = 0; i < array.length; ++i) { 348 sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1, 3)); 349 } 350 return sb.toString(); 351 } catch (java.security.NoSuchAlgorithmException e) { 352 throw new RuntimeException("Error generating MD5 hash.", e); 353 } 354 } 355 356 protected void addDepModule(String module) { 357 mDepModules.add(module); 358 } 359 360 protected void setRequirementsFile(File f) { 361 mRequirementsFile = f; 362 } 363 364 /** 365 * Get an instance of {@link IRunUtil}. 366 */ 367 @VisibleForTesting 368 IRunUtil getRunUtil() { 369 if (mRunUtil == null) { 370 mRunUtil = new RunUtil(); 371 } 372 return mRunUtil; 373 } 374 375 /** 376 * This method recursively deletes a file tree without following symbolic links. 377 * 378 * @param rootPath the path to delete. 379 * @throws IOException if fails to traverse or delete the files. 380 */ 381 private static void recursiveDelete(Path rootPath) throws IOException { 382 Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() { 383 @Override 384 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) 385 throws IOException { 386 Files.delete(file); 387 return FileVisitResult.CONTINUE; 388 } 389 @Override 390 public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { 391 if (e != null) { 392 throw e; 393 } 394 Files.delete(dir); 395 return FileVisitResult.CONTINUE; 396 } 397 }); 398 } 399 400 /** 401 * This method returns whether the OS is Windows. 402 */ 403 private static boolean isOnWindows() { 404 return System.getProperty(OS_NAME).contains(WINDOWS); 405 } 406 407 private void activate() { 408 File binDir = new File(mVenvDir, isOnWindows() ? "Scripts" : "bin"); 409 getRunUtil().setWorkingDir(binDir); 410 String path = System.getenv(PATH); 411 getRunUtil().setEnvVariable(PATH, binDir + File.pathSeparator + path); 412 File pipFile = new File(binDir, PIP); 413 pipFile.setExecutable(true); 414 mPip = pipFile.getAbsolutePath(); 415 } 416 } 417