1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.testtype; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.compatibility.common.util.AbiUtils; 21 import com.android.ddmlib.IShellOutputReceiver; 22 import com.android.ddmlib.Log; 23 import com.android.ddmlib.Log.LogLevel; 24 import com.android.ddmlib.MultiLineReceiver; 25 import com.android.ddmlib.testrunner.TestIdentifier; 26 import com.android.tradefed.build.IBuildInfo; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.config.OptionCopier; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.device.ITestDevice; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.testtype.IAbi; 33 import com.android.tradefed.testtype.IAbiReceiver; 34 import com.android.tradefed.testtype.IBuildReceiver; 35 import com.android.tradefed.testtype.IDeviceTest; 36 import com.android.tradefed.testtype.IRemoteTest; 37 import com.android.tradefed.testtype.IRuntimeHintProvider; 38 import com.android.tradefed.testtype.IShardableTest; 39 import com.android.tradefed.testtype.ITestCollector; 40 import com.android.tradefed.testtype.ITestFileFilterReceiver; 41 import com.android.tradefed.testtype.ITestFilterReceiver; 42 import com.android.tradefed.util.ArrayUtil; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.TimeVal; 45 import com.google.common.base.Splitter; 46 47 import vogar.ExpectationStore; 48 import vogar.ModeId; 49 50 import java.io.BufferedReader; 51 import java.io.File; 52 import java.io.FilenameFilter; 53 import java.io.FileReader; 54 import java.io.IOException; 55 import java.io.PrintWriter; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.Collection; 59 import java.util.Collections; 60 import java.util.HashSet; 61 import java.util.List; 62 import java.util.Set; 63 import java.util.concurrent.TimeUnit; 64 65 /** 66 * A wrapper to run tests against Dalvik. 67 */ 68 public class DalvikTest implements IAbiReceiver, IBuildReceiver, IDeviceTest, IRemoteTest, 69 IRuntimeHintProvider, IShardableTest, ITestCollector, ITestFileFilterReceiver, 70 ITestFilterReceiver { 71 72 private static final String TAG = DalvikTest.class.getSimpleName(); 73 74 /** 75 * TEST_PACKAGES is a Set containing the names of packages on the classpath known to contain 76 * tests to be run under DalvikTest. The TEST_PACKAGES set is used to shard DalvikTest into 77 * multiple DalvikTests, each responsible for running one of these packages' tests. 78 */ 79 private static final Set<String> TEST_PACKAGES = new HashSet<>(); 80 private static final String JDWP_PACKAGE_BASE = "org.apache.harmony.jpda.tests.jdwp.%s"; 81 static { 82 // Though uppercase, these are package names, not class names 83 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayReference")); 84 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayType")); 85 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassLoaderReference")); 86 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassObjectReference")); 87 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassType")); 88 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "DebuggerOnDemand")); 89 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Deoptimization")); 90 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "EventModifiers")); 91 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Events")); 92 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "InterfaceType")); 93 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Method")); 94 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "MultiSession")); 95 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ObjectReference")); 96 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ReferenceType")); 97 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StackFrame")); 98 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StringReference")); 99 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadGroupReference")); 100 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadReference")); 101 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "VirtualMachine")); 102 } 103 104 private static final String EXPECTATIONS_EXT = ".expectations"; 105 // Command to run the VM, args are bitness, classpath, dalvik-args, abi, runner-args, 106 // include and exclude filters, and exclude filters file. 107 private static final String COMMAND = "dalvikvm%s -classpath %s %s " 108 + "com.android.compatibility.dalvik.DalvikTestRunner --abi=%s %s %s %s %s %s %s"; 109 private static final String INCLUDE_FILE = "/data/local/tmp/dalvik/includes"; 110 private static final String EXCLUDE_FILE = "/data/local/tmp/dalvik/excludes"; 111 private static String START_RUN = "start-run"; 112 private static String END_RUN = "end-run"; 113 private static String START_TEST = "start-test"; 114 private static String END_TEST = "end-test"; 115 private static String FAILURE = "failure"; 116 117 @Option(name = "run-name", description = "The name to use when reporting results") 118 private String mRunName; 119 120 @Option(name = "classpath", description = "Holds the paths to search when loading tests") 121 private List<String> mClasspath = new ArrayList<>(); 122 123 @Option(name = "dalvik-arg", description = "Holds arguments to pass to Dalvik") 124 private List<String> mDalvikArgs = new ArrayList<>(); 125 126 @Option(name = "runner-arg", 127 description = "Holds arguments to pass to the device-side test runner") 128 private List<String> mRunnerArgs = new ArrayList<>(); 129 130 @Option(name = "include-filter", 131 description = "The include filters of the test name to run.") 132 private List<String> mIncludeFilters = new ArrayList<>(); 133 134 @Option(name = "exclude-filter", 135 description = "The exclude filters of the test name to run.") 136 private List<String> mExcludeFilters = new ArrayList<>(); 137 138 @Option(name = "test-file-include-filter", 139 description="A file containing a list of line separated test classes and optionally" 140 + " methods to include") 141 private File mIncludeTestFile = null; 142 143 @Option(name = "test-file-exclude-filter", 144 description="A file containing a list of line separated test classes and optionally" 145 + " methods to exclude") 146 private File mExcludeTestFile = null; 147 148 @Option(name = "runtime-hint", 149 isTimeVal = true, 150 description="The hint about the test's runtime.") 151 private long mRuntimeHint = 60000;// 1 minute 152 153 @Option(name = "known-failures", 154 description = "Comma-separated list of files specifying known-failures to be skipped") 155 private String mKnownFailures; 156 157 @Option(name = "collect-tests-only", 158 description = "Only invoke the instrumentation to collect list of applicable test " 159 + "cases. All test run callbacks will be triggered, but test execution will " 160 + "not be actually carried out.") 161 private boolean mCollectTestsOnly = false; 162 163 @Option(name = "per-test-timeout", 164 description = "The maximum amount of time during which the DalvikTestRunner may " 165 + "yield no output. Because the runner outputs results for each test, this " 166 + "is essentially a per-test timeout") 167 private long mPerTestTimeout = 10; // 10 minutes 168 169 private IAbi mAbi; 170 private CompatibilityBuildHelper mBuildHelper; 171 private ITestDevice mDevice; 172 173 /** 174 * {@inheritDoc} 175 */ 176 @Override 177 public void setAbi(IAbi abi) { 178 mAbi = abi; 179 } 180 181 /** 182 * {@inheritDoc} 183 */ 184 @Override 185 public void setBuild(IBuildInfo build) { 186 mBuildHelper = new CompatibilityBuildHelper(build); 187 } 188 189 /** 190 * {@inheritDoc} 191 */ 192 @Override 193 public void setDevice(ITestDevice device) { 194 mDevice = device; 195 } 196 197 /** 198 * {@inheritDoc} 199 */ 200 @Override 201 public ITestDevice getDevice() { 202 return mDevice; 203 } 204 205 /** 206 * {@inheritDoc} 207 */ 208 @Override 209 public void addIncludeFilter(String filter) { 210 mIncludeFilters.add(filter); 211 } 212 213 /** 214 * {@inheritDoc} 215 */ 216 @Override 217 public void addAllIncludeFilters(Set<String> filters) { 218 mIncludeFilters.addAll(filters); 219 } 220 221 /** 222 * {@inheritDoc} 223 */ 224 @Override 225 public void addExcludeFilter(String filter) { 226 mExcludeFilters.add(filter); 227 } 228 229 /** 230 * {@inheritDoc} 231 */ 232 @Override 233 public void addAllExcludeFilters(Set<String> filters) { 234 mExcludeFilters.addAll(filters); 235 } 236 237 /** 238 * {@inheritDoc} 239 */ 240 @Override 241 public void setIncludeTestFile(File testFile) { 242 mIncludeTestFile = testFile; 243 } 244 245 /** 246 * {@inheritDoc} 247 */ 248 @Override 249 public void setExcludeTestFile(File testFile) { 250 mExcludeTestFile = testFile; 251 } 252 253 /** 254 * {@inheritDoc} 255 */ 256 @Override 257 public long getRuntimeHint() { 258 return mRuntimeHint; 259 } 260 261 /** 262 * {@inheritDoc} 263 */ 264 @Override 265 public void setCollectTestsOnly(boolean shouldCollectTest) { 266 mCollectTestsOnly = shouldCollectTest; 267 } 268 269 /** 270 * {@inheritDoc} 271 */ 272 @Override 273 public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException { 274 String abiName = mAbi.getName(); 275 String bitness = AbiUtils.getBitness(abiName); 276 277 File tmpExcludeFile = null; 278 try { 279 // push one file of exclude filters to the device 280 tmpExcludeFile = getExcludeFile(); 281 if (!mDevice.pushFile(tmpExcludeFile, EXCLUDE_FILE)) { 282 Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + tmpExcludeFile); 283 } 284 } catch (IOException e) { 285 throw new RuntimeException("Failed to parse expectations", e); 286 } finally { 287 FileUtil.deleteFile(tmpExcludeFile); 288 } 289 290 // push one file of include filters to the device, if file exists 291 if (mIncludeTestFile != null) { 292 String path = mIncludeTestFile.getAbsolutePath(); 293 if (!mIncludeTestFile.isFile() || !mIncludeTestFile.canRead()) { 294 throw new RuntimeException(String.format("Failed to read include file %s", path)); 295 } 296 if (!mDevice.pushFile(mIncludeTestFile, INCLUDE_FILE)) { 297 Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + path); 298 } 299 } 300 301 302 // Create command 303 mDalvikArgs.add("-Duser.name=shell"); 304 mDalvikArgs.add("-Duser.language=en"); 305 mDalvikArgs.add("-Duser.region=US"); 306 mDalvikArgs.add("-Xcheck:jni"); 307 mDalvikArgs.add("-Xjnigreflimit:2000"); 308 309 String dalvikArgs = ArrayUtil.join(" ", mDalvikArgs); 310 dalvikArgs = dalvikArgs.replace("|#ABI#|", bitness); 311 312 String runnerArgs = ArrayUtil.join(" ", mRunnerArgs); 313 // Filters 314 StringBuilder includeFilters = new StringBuilder(); 315 if (!mIncludeFilters.isEmpty()) { 316 includeFilters.append("--include-filter="); 317 includeFilters.append(ArrayUtil.join(",", mIncludeFilters)); 318 } 319 StringBuilder excludeFilters = new StringBuilder(); 320 if (!mExcludeFilters.isEmpty()) { 321 excludeFilters.append("--exclude-filter="); 322 excludeFilters.append(ArrayUtil.join(",", mExcludeFilters)); 323 } 324 // Filter files 325 String includeFile = String.format("--include-filter-file=%s", INCLUDE_FILE); 326 String excludeFile = String.format("--exclude-filter-file=%s", EXCLUDE_FILE); 327 // Communicate with DalvikTestRunner if tests should only be collected 328 String collectTestsOnlyString = (mCollectTestsOnly) ? "--collect-tests-only" : ""; 329 final String command = String.format(COMMAND, bitness, 330 ArrayUtil.join(File.pathSeparator, mClasspath), 331 dalvikArgs, abiName, runnerArgs, 332 includeFilters, excludeFilters, includeFile, excludeFile, collectTestsOnlyString); 333 IShellOutputReceiver receiver = new MultiLineReceiver() { 334 private TestIdentifier test; 335 336 @Override 337 public boolean isCancelled() { 338 return false; 339 } 340 341 @Override 342 public void processNewLines(String[] lines) { 343 for (String line : lines) { 344 String[] parts = line.split(":", 2); 345 String tag = parts[0]; 346 if (tag.equals(START_RUN)) { 347 listener.testRunStarted(mRunName, Integer.parseInt(parts[1])); 348 Log.logAndDisplay(LogLevel.INFO, TAG, command); 349 Log.logAndDisplay(LogLevel.INFO, TAG, line); 350 } else if (tag.equals(END_RUN)) { 351 listener.testRunEnded(Integer.parseInt(parts[1]), 352 Collections.<String, String>emptyMap()); 353 Log.logAndDisplay(LogLevel.INFO, TAG, line); 354 } else if (tag.equals(START_TEST)) { 355 test = getTestIdentifier(parts[1]); 356 listener.testStarted(test); 357 } else if (tag.equals(FAILURE)) { 358 listener.testFailed(test, parts[1]); 359 } else if (tag.equals(END_TEST)) { 360 listener.testEnded(getTestIdentifier(parts[1]), 361 Collections.<String, String>emptyMap()); 362 } else { 363 Log.logAndDisplay(LogLevel.INFO, TAG, line); 364 } 365 } 366 } 367 368 private TestIdentifier getTestIdentifier(String name) { 369 String[] parts = name.split("#"); 370 String className = parts[0]; 371 String testName = ""; 372 if (parts.length > 1) { 373 testName = parts[1]; 374 } 375 return new TestIdentifier(className, testName); 376 } 377 378 }; 379 mDevice.executeShellCommand(command, receiver, mPerTestTimeout, TimeUnit.MINUTES, 1); 380 } 381 382 /* 383 * Due to known failures, there are typically too many excludes to pass via command line. 384 * Collect excludes from .expectation files in the testcases directory, from files in the 385 * module's resources directory, and from mExcludeTestFile, if set. 386 */ 387 private File getExcludeFile() throws IOException { 388 File excludeFile = null; 389 PrintWriter out = null; 390 391 try { 392 excludeFile = File.createTempFile("excludes", "txt"); 393 out = new PrintWriter(excludeFile); 394 // create expectation store from set of expectation files found in testcases dir 395 Set<File> expectationFiles = new HashSet<>(); 396 for (File f : mBuildHelper.getTestsDir().listFiles( 397 new ExpectationFileFilter(mRunName))) { 398 expectationFiles.add(f); 399 } 400 ExpectationStore testsDirStore = 401 ExpectationStore.parse(expectationFiles, ModeId.DEVICE); 402 // create expectation store from expectation files found in module resources dir 403 ExpectationStore resourceStore = null; 404 if (mKnownFailures != null) { 405 Splitter splitter = Splitter.on(',').trimResults(); 406 Set<String> knownFailuresFiles = 407 new HashSet<>(splitter.splitToList(mKnownFailures)); 408 resourceStore = ExpectationStore.parseResources( 409 getClass(), knownFailuresFiles, ModeId.DEVICE); 410 } 411 // Add expectations from testcases dir 412 for (String exclude : testsDirStore.getAllFailures().keySet()) { 413 out.println(exclude); 414 } 415 for (String exclude : testsDirStore.getAllOutComes().keySet()) { 416 out.println(exclude); 417 } 418 // Add expectations from resources dir 419 if (resourceStore != null) { 420 for (String exclude : resourceStore.getAllFailures().keySet()) { 421 out.println(exclude); 422 } 423 for (String exclude : resourceStore.getAllOutComes().keySet()) { 424 out.println(exclude); 425 } 426 } 427 // Add excludes from test-file-exclude-filter option 428 for (String exclude : getFiltersFromFile(mExcludeTestFile)) { 429 out.println(exclude); 430 } 431 out.flush(); 432 } finally { 433 if (out != null) { 434 out.close(); 435 } 436 } 437 return excludeFile; 438 } 439 440 441 /* 442 * Helper method that reads filters from a file into a set. 443 * Returns an empty set given a null file 444 */ 445 private static Set<String> getFiltersFromFile(File f) throws IOException { 446 Set<String> filters = new HashSet<String>(); 447 if (f != null) { 448 BufferedReader reader = new BufferedReader(new FileReader(f)); 449 String filter = null; 450 while ((filter = reader.readLine()) != null) { 451 filters.add(filter); 452 } 453 reader.close(); 454 } 455 return filters; 456 } 457 458 /** 459 * {@inheritDoc} 460 */ 461 @Override 462 public Collection<IRemoteTest> split() { 463 List<IRemoteTest> shards = new ArrayList<>(); 464 // A DalvikTest to run any tests not contained in packages from TEST_PACKAGES, may be empty 465 DalvikTest catchAll = new DalvikTest(); 466 OptionCopier.copyOptionsNoThrow(this, catchAll); 467 shards.add(catchAll); 468 // estimate catchAll's runtime to be that of a single package in TEST_PACKAGES 469 long runtimeHint = mRuntimeHint / TEST_PACKAGES.size(); 470 catchAll.mRuntimeHint = runtimeHint; 471 for (String packageName: TEST_PACKAGES) { 472 catchAll.addExcludeFilter(packageName); 473 // create one shard for package 'packageName' 474 DalvikTest test = new DalvikTest(); 475 OptionCopier.copyOptionsNoThrow(this, test); 476 test.addIncludeFilter(packageName); 477 test.mRuntimeHint = runtimeHint; 478 shards.add(test); 479 } 480 // return a shard for each package in TEST_PACKAGE, plus a shard for any other tests 481 return shards; 482 } 483 484 /** 485 * A {@link FilenameFilter} to find all the expectation files in a directory. 486 */ 487 public static class ExpectationFileFilter implements FilenameFilter { 488 489 private String mName; 490 491 public ExpectationFileFilter(String name) { 492 mName = name; 493 } 494 /** 495 * {@inheritDoc} 496 */ 497 @Override 498 public boolean accept(File dir, String name) { 499 return name.startsWith(mName) && name.endsWith(EXPECTATIONS_EXT); 500 } 501 } 502 } 503