1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.build.IFolderBuildInfo; 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.log.LogUtil.CLog; 21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 22 import com.android.tradefed.result.FileInputStreamSource; 23 import com.android.tradefed.result.ITestInvocationListener; 24 import com.android.tradefed.result.InputStreamSource; 25 import com.android.tradefed.result.LogDataType; 26 import com.android.tradefed.result.TestDescription; 27 import com.android.tradefed.util.CommandResult; 28 import com.android.tradefed.util.CommandStatus; 29 import com.android.tradefed.util.FileUtil; 30 import com.android.tradefed.util.HprofAllocSiteParser; 31 import com.android.tradefed.util.RunUtil; 32 import com.android.tradefed.util.StreamUtil; 33 import com.android.tradefed.util.SystemUtil.EnvVariable; 34 import com.android.tradefed.util.proto.TfMetricProtoUtil; 35 import com.android.tradefed.util.TarUtil; 36 37 import com.google.common.annotations.VisibleForTesting; 38 39 import java.io.File; 40 import java.io.IOException; 41 import java.io.InputStream; 42 import java.util.ArrayList; 43 import java.util.Arrays; 44 import java.util.HashMap; 45 import java.util.List; 46 import java.util.Map; 47 import java.util.regex.Pattern; 48 49 /** 50 * A {@link IRemoteTest} for running unit or functional tests against a separate TF installation. 51 * <p/> 52 * Launches an external java process to run the tests. Used for running the TF unit or 53 * functional tests continuously. 54 */ 55 public class TfTestLauncher extends SubprocessTfLauncher { 56 57 private static final long COVERAGE_REPORT_TIMEOUT_MS = 2 * 60 * 1000; 58 59 @Option(name = "jacoco-code-coverage", description = "Enable jacoco code coverage on the java " 60 + "sub process. Run will be slightly slower because of the overhead.") 61 private boolean mEnableCoverage = false; 62 63 @Option( 64 name = "hprof-heap-memory", 65 description = 66 "Enable hprof agent while running the java" 67 + "sub process. Run will be slightly slower because of the overhead." 68 ) 69 private boolean mEnableHprof = false; 70 71 @Option(name = "ant-config-res", description = "The name of the ant resource configuration to " 72 + "transform the results in readable format.") 73 private String mAntConfigResource = "/jacoco/ant-tf-coverage.xml"; 74 75 @Option(name = "sub-branch", description = "The branch to be provided to the sub invocation, " 76 + "if null, the branch in build info will be used.") 77 private String mSubBranch = null; 78 79 @Option(name = "sub-build-flavor", description = "The build flavor to be provided to the " 80 + "sub invocation, if null, the build flavor in build info will be used.") 81 private String mSubBuildFlavor = null; 82 83 @Option(name = "sub-build-id", description = "The build id that the sub invocation will try " 84 + "to use in case where it needs its own device.") 85 private String mSubBuildId = null; 86 87 @Option(name = "use-virtual-device", description = 88 "Flag if the subprocess is going to need to instantiate a virtual device to run.") 89 private boolean mUseVirtualDevice = false; 90 91 @Option(name = "sub-apk-path", description = "The name of all the Apks that needs to be " 92 + "installed by the subprocess invocation. Apk need to be inside the downloaded zip. " 93 + "Can be repeated.") 94 private List<String> mSubApkPath = new ArrayList<String>(); 95 96 // The regex pattern of temp files to be found in the temporary dir of the subprocess. 97 // Any file not matching the patterns, or multiple files in the temporary dir match the same 98 // pattern, is considered as test failure. 99 private static final String[] EXPECTED_TMP_FILE_PATTERNS = { 100 "inv_.*", "tradefed_global_log_.*", "lc_cache", "stage-android-build-api", 101 }; 102 103 // A destination file where the report will be put. 104 private File mDestCoverageFile = null; 105 // A destination file where the hprof report will be put. 106 private File mHprofFile = null; 107 // A {@link File} pointing to the jacoco args jar file extracted from the resources 108 private File mAgent = null; 109 110 /** {@inheritDoc} */ 111 @Override 112 protected void addJavaArguments(List<String> args) { 113 super.addJavaArguments(args); 114 try { 115 if (mEnableCoverage) { 116 mDestCoverageFile = FileUtil.createTempFile("coverage", ".exec"); 117 mAgent = extractJacocoAgent(); 118 addCoverageArgs(mAgent, args, mDestCoverageFile); 119 } 120 if (mEnableHprof) { 121 mHprofFile = FileUtil.createTempFile("java.hprof", ".txt"); 122 // verbose=n to avoid dump in stderr 123 // cutoff the min value we look at. 124 String hprofAgent = 125 String.format( 126 "-agentlib:hprof=heap=sites,cutoff=0.01,depth=16,verbose=n,file=%s", 127 mHprofFile.getAbsolutePath()); 128 args.add(hprofAgent); 129 } 130 } catch (IOException e) { 131 throw new RuntimeException(e); 132 } 133 } 134 135 /** {@inheritDoc} */ 136 @Override 137 protected void preRun() { 138 super.preRun(); 139 140 if (!mUseVirtualDevice) { 141 mCmdArgs.add("-n"); 142 } else { 143 // if it needs a device we also enable more logs 144 mCmdArgs.add("--log-level"); 145 mCmdArgs.add("VERBOSE"); 146 mCmdArgs.add("--log-level-display"); 147 mCmdArgs.add("VERBOSE"); 148 } 149 mCmdArgs.add("--test-tag"); 150 mCmdArgs.add(mBuildInfo.getTestTag()); 151 mCmdArgs.add("--build-id"); 152 if (mSubBuildId != null) { 153 mCmdArgs.add(mSubBuildId); 154 } else { 155 mCmdArgs.add(mBuildInfo.getBuildId()); 156 } 157 mCmdArgs.add("--branch"); 158 if (mSubBranch != null) { 159 mCmdArgs.add(mSubBranch); 160 } else if (mBuildInfo.getBuildBranch() != null) { 161 mCmdArgs.add(mBuildInfo.getBuildBranch()); 162 } else { 163 throw new RuntimeException("Branch option is required for the sub invocation."); 164 } 165 mCmdArgs.add("--build-flavor"); 166 if (mSubBuildFlavor != null) { 167 mCmdArgs.add(mSubBuildFlavor); 168 } else if (mBuildInfo.getBuildFlavor() != null) { 169 mCmdArgs.add(mBuildInfo.getBuildFlavor()); 170 } else { 171 throw new RuntimeException("Build flavor option is required for the sub invocation."); 172 } 173 174 for (String apk : mSubApkPath) { 175 mCmdArgs.add("--apk-path"); 176 String apkPath = 177 String.format( 178 "%s%s%s", 179 ((IFolderBuildInfo) mBuildInfo).getRootDir().getAbsolutePath(), 180 File.separator, 181 apk); 182 mCmdArgs.add(apkPath); 183 } 184 // Unset potential build environment to ensure they do not affect the unit tests 185 getRunUtil().unsetEnvVariable(EnvVariable.ANDROID_HOST_OUT_TESTCASES.name()); 186 getRunUtil().unsetEnvVariable(EnvVariable.ANDROID_TARGET_OUT_TESTCASES.name()); 187 } 188 189 /** {@inheritDoc} */ 190 @Override 191 protected void postRun(ITestInvocationListener listener, boolean exception, long elapsedTime) { 192 super.postRun(listener, exception, elapsedTime); 193 reportMetrics(elapsedTime, listener); 194 FileUtil.deleteFile(mAgent); 195 196 // Evaluate coverage from the subprocess 197 if (mEnableCoverage) { 198 InputStreamSource coverage = null; 199 File xmlResult = null; 200 try { 201 xmlResult = processExecData(mDestCoverageFile, mRootDir); 202 coverage = new FileInputStreamSource(xmlResult); 203 listener.testLog("coverage_xml", LogDataType.JACOCO_XML, coverage); 204 } catch (IOException e) { 205 if (exception) { 206 // If exception was thrown above, we only log this one since it's most 207 // likely related to it. 208 CLog.e(e); 209 } else { 210 throw new RuntimeException(e); 211 } 212 } finally { 213 FileUtil.deleteFile(mDestCoverageFile); 214 StreamUtil.cancel(coverage); 215 FileUtil.deleteFile(xmlResult); 216 } 217 } 218 if (mEnableHprof) { 219 logHprofResults(mHprofFile, listener); 220 } 221 222 if (mTmpDir != null) { 223 testTmpDirClean(mTmpDir, listener); 224 } 225 cleanTmpFile(); 226 } 227 228 @VisibleForTesting 229 void cleanTmpFile() { 230 FileUtil.deleteFile(mHprofFile); 231 FileUtil.deleteFile(mDestCoverageFile); 232 FileUtil.deleteFile(mAgent); 233 } 234 235 /** 236 * Helper to add arguments required for code coverage collection. 237 * 238 * @param jacocoAgent the jacoco args file to run the coverage. 239 * @param args list of arguments that will be run in the subprocess. 240 * @param destfile destination file where the report will be put. 241 */ 242 private void addCoverageArgs(File jacocoAgent, List<String> args, File destfile) { 243 String javaagent = String.format("-javaagent:%s=destfile=%s," 244 + "includes=com.android.tradefed*:com.google.android.tradefed*", 245 jacocoAgent.getAbsolutePath(), 246 destfile.getAbsolutePath()); 247 args.add(javaagent); 248 } 249 250 /** 251 * Returns a {@link File} pointing to the jacoco args jar file extracted from the resources. 252 */ 253 private File extractJacocoAgent() throws IOException { 254 String jacocoAgentRes = "/jacoco/jacocoagent.jar"; 255 InputStream jacocoAgentStream = getClass().getResourceAsStream(jacocoAgentRes); 256 if (jacocoAgentStream == null) { 257 throw new IOException("Could not find " + jacocoAgentRes); 258 } 259 File jacocoAgent = FileUtil.createTempFile("jacocoagent", ".jar"); 260 FileUtil.writeToFile(jacocoAgentStream, jacocoAgent); 261 return jacocoAgent; 262 } 263 264 /** 265 * Helper to process the execution data into user readable format (xml) that can easily be 266 * parsed. 267 * 268 * @param executionData output files of the java args jacoco. 269 * @param rootDir base directory of downloaded TF 270 * @return a {@link File} pointing to the human readable xml result file. 271 */ 272 private File processExecData(File executionData, String rootDir) throws IOException { 273 File xmlReport = FileUtil.createTempFile("coverage_xml", ".xml"); 274 InputStream template = getClass().getResourceAsStream(mAntConfigResource); 275 if (template == null) { 276 throw new IOException("Could not find " + mAntConfigResource); 277 } 278 String jacocoAntRes = "/jacoco/jacocoant.jar"; 279 InputStream jacocoAntStream = getClass().getResourceAsStream(jacocoAntRes); 280 if (jacocoAntStream == null) { 281 throw new IOException("Could not find " + jacocoAntRes); 282 } 283 File antConfig = FileUtil.createTempFile("ant-merge_", ".xml"); 284 File jacocoAnt = FileUtil.createTempFile("jacocoant", ".jar"); 285 try { 286 FileUtil.writeToFile(template, antConfig); 287 FileUtil.writeToFile(jacocoAntStream, jacocoAnt); 288 String[] cmd = {"ant", "-f", antConfig.getPath(), 289 "-Djacocoant.path=" + jacocoAnt.getAbsolutePath(), 290 "-Dexecution.files=" + executionData.getAbsolutePath(), 291 "-Droot.dir=" + rootDir, 292 "-Ddest.file=" + xmlReport.getAbsolutePath()}; 293 CommandResult result = RunUtil.getDefault().runTimedCmd(COVERAGE_REPORT_TIMEOUT_MS, 294 cmd); 295 CLog.d(result.getStdout()); 296 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 297 throw new IOException(result.getStderr()); 298 } 299 return xmlReport; 300 } finally { 301 FileUtil.deleteFile(antConfig); 302 FileUtil.deleteFile(jacocoAnt); 303 } 304 } 305 306 /** 307 * Report an elapsed-time metric to keep track of it. 308 * 309 * @param elapsedTime time it took the subprocess to run. 310 * @param listener the {@link ITestInvocationListener} where to report the metric. 311 */ 312 private void reportMetrics(long elapsedTime, ITestInvocationListener listener) { 313 if (elapsedTime == -1l) { 314 return; 315 } 316 listener.testRunStarted("elapsed-time", 1); 317 TestDescription tid = new TestDescription("elapsed-time", "run-elapsed-time"); 318 listener.testStarted(tid); 319 HashMap<String, Metric> runMetrics = new HashMap<>(); 320 runMetrics.put( 321 "elapsed-time", TfMetricProtoUtil.stringToMetric(Long.toString(elapsedTime))); 322 listener.testEnded(tid, runMetrics); 323 listener.testRunEnded(elapsedTime, runMetrics); 324 } 325 326 /** 327 * Extra test to ensure no files are created by the unit tests in the subprocess and not 328 * cleaned. 329 * 330 * @param tmpDir the temporary dir of the subprocess. 331 * @param listener the {@link ITestInvocationListener} where to report the test. 332 */ 333 @VisibleForTesting 334 protected void testTmpDirClean(File tmpDir, ITestInvocationListener listener) { 335 listener.testRunStarted("temporaryFiles", 1); 336 TestDescription tid = new TestDescription("temporary-files", "testIfClean"); 337 listener.testStarted(tid); 338 String[] listFiles = tmpDir.list(); 339 List<String> unmatchedFiles = new ArrayList<String>(); 340 List<String> patterns = new ArrayList<String>(Arrays.asList(EXPECTED_TMP_FILE_PATTERNS)); 341 for (String file : Arrays.asList(listFiles)) { 342 Boolean matchFound = false; 343 for (String pattern : patterns) { 344 if (Pattern.matches(pattern, file)) { 345 patterns.remove(pattern); 346 matchFound = true; 347 break; 348 } 349 } 350 if (!matchFound) { 351 unmatchedFiles.add(file); 352 } 353 } 354 if (unmatchedFiles.size() > 0) { 355 String trace = String.format("Found '%d' unexpected temporary files: %s.\nOnly " 356 + "expected files are: %s. And each should appears only once.", 357 unmatchedFiles.size(), unmatchedFiles, 358 Arrays.asList(EXPECTED_TMP_FILE_PATTERNS)); 359 listener.testFailed(tid, trace); 360 } 361 listener.testEnded(tid, new HashMap<String, Metric>()); 362 listener.testRunEnded(0, new HashMap<String, Metric>()); 363 } 364 365 /** 366 * Helper to log and report as metric the hprof data. 367 * 368 * @param hprofFile file containing the Hprof report 369 * @param listener the {@link ITestInvocationListener} where to report the test. 370 */ 371 private void logHprofResults(File hprofFile, ITestInvocationListener listener) { 372 if (hprofFile == null) { 373 CLog.w("Hprof file was null. Skipping parsing."); 374 return; 375 } 376 if (!hprofFile.exists()) { 377 CLog.w("Hprof file %s was not found. Skipping parsing.", hprofFile.getAbsolutePath()); 378 return; 379 } 380 InputStreamSource memory = null; 381 File tmpGzip = null; 382 try { 383 tmpGzip = TarUtil.gzip(hprofFile); 384 memory = new FileInputStreamSource(tmpGzip); 385 listener.testLog("hprof", LogDataType.GZIP, memory); 386 } catch (IOException e) { 387 CLog.e(e); 388 return; 389 } finally { 390 StreamUtil.cancel(memory); 391 FileUtil.deleteFile(tmpGzip); 392 } 393 HprofAllocSiteParser parser = new HprofAllocSiteParser(); 394 try { 395 Map<String, String> results = parser.parse(hprofFile); 396 if (results.isEmpty()) { 397 CLog.d("No allocation site found from hprof file"); 398 return; 399 } 400 listener.testRunStarted("hprofAllocSites", 1); 401 TestDescription tid = new TestDescription("hprof", "allocationSites"); 402 listener.testStarted(tid); 403 listener.testEnded(tid, TfMetricProtoUtil.upgradeConvert(results)); 404 listener.testRunEnded(0, new HashMap<String, Metric>()); 405 } catch (IOException e) { 406 throw new RuntimeException(e); 407 } 408 } 409 } 410
