1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts.bitstreams; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.targetprep.MediaPreparer; 20 import com.android.compatibility.common.util.MetricsReportLog; 21 import com.android.compatibility.common.util.ResultType; 22 import com.android.compatibility.common.util.ResultUnit; 23 import com.android.tradefed.build.IBuildInfo; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.config.OptionClass; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.testtype.IAbi; 30 import com.android.tradefed.testtype.IAbiReceiver; 31 import com.android.tradefed.testtype.IBuildReceiver; 32 import com.android.tradefed.testtype.IDeviceTest; 33 import com.android.tradefed.util.FileUtil; 34 import java.io.ByteArrayOutputStream; 35 import java.io.File; 36 import java.io.FileFilter; 37 import java.io.IOException; 38 import java.io.InputStream; 39 import java.io.OutputStream; 40 import java.io.PrintStream; 41 import java.nio.file.Files; 42 import java.util.ArrayDeque; 43 import java.util.ArrayList; 44 import java.util.Collection; 45 import java.util.Collections; 46 import java.util.Deque; 47 import java.util.HashMap; 48 import java.util.Iterator; 49 import java.util.LinkedHashSet; 50 import java.util.List; 51 import java.util.Map; 52 import java.util.Map.Entry; 53 import java.util.Set; 54 import java.util.concurrent.ConcurrentHashMap; 55 import java.util.concurrent.ConcurrentMap; 56 import org.junit.Assert; 57 import org.junit.Ignore; 58 import org.junit.Test; 59 import org.xmlpull.v1.XmlPullParser; 60 import org.xmlpull.v1.XmlPullParserException; 61 import org.xmlpull.v1.XmlPullParserFactory; 62 63 /** 64 * Test that verifies video bitstreams decode pixel perfectly 65 */ 66 @OptionClass(alias="media-bitstreams-test") 67 public abstract class MediaBitstreamsTest implements IDeviceTest, IBuildReceiver, IAbiReceiver { 68 69 @Option(name = MediaBitstreams.OPT_HOST_BITSTREAMS_PATH, 70 description = "Absolute path of Ittiam bitstreams (host)", 71 mandatory = true) 72 private File mHostBitstreamsPath = getDefaultBitstreamsDir(); 73 74 @Option(name = MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, 75 description = "Absolute path of Ittiam bitstreams (device)") 76 private String mDeviceBitstreamsPath = MediaBitstreams.DEFAULT_DEVICE_BITSTEAMS_PATH; 77 78 @Option(name = MediaBitstreams.OPT_DOWNLOAD_BITSTREAMS, 79 description = "Whether to download the bitstreams files") 80 private boolean mDownloadBitstreams = false; 81 82 @Option(name = MediaBitstreams.OPT_UTILIZATION_RATE, 83 description = "Percentage of external storage space used for test") 84 private int mUtilizationRate = 80; 85 86 @Option(name = MediaBitstreams.OPT_NUM_BATCHES, 87 description = "Number of batches to test;" 88 + " each batch uses external storage up to utilization rate") 89 private int mNumBatches = Integer.MAX_VALUE; 90 91 @Option(name = MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, 92 description = "Whether to debug target device under test") 93 private boolean mDebugTargetDevice = false; 94 95 @Option(name = MediaBitstreams.OPT_BITSTREAMS_PREFIX, 96 description = "Only test bitstreams in this sub-directory") 97 private String mPrefix = ""; 98 99 private String mPath = ""; 100 101 private static ConcurrentMap<String, List<ConformanceEntry>> mResults = new ConcurrentHashMap<>(); 102 103 /** 104 * Which subset of bitstreams to test 105 */ 106 enum BitstreamPackage { 107 STANDARD, 108 FULL, 109 } 110 111 private BitstreamPackage mPackage = BitstreamPackage.FULL; 112 private BitstreamPackage mPackageToRun = BitstreamPackage.STANDARD; 113 private boolean mEnforce = false; 114 115 static class ConformanceEntry { 116 final String mPath, mCodecName, mStatus; 117 ConformanceEntry(String path, String codecName, String status) { 118 mPath = path; 119 mCodecName = codecName; 120 mStatus = status; 121 } 122 @Override 123 public String toString() { 124 return String.format("%s,%s,%s", mPath, mCodecName, mStatus); 125 } 126 } 127 128 /** 129 * A helper to access resources in the build. 130 */ 131 private CompatibilityBuildHelper mBuildHelper; 132 133 private IAbi mAbi; 134 private ITestDevice mDevice; 135 136 static File getDefaultBitstreamsDir() { 137 File mediaDir = MediaPreparer.getDefaultMediaDir(); 138 File[] subDirs = mediaDir.listFiles(new FileFilter() { 139 @Override 140 public boolean accept(File child) { 141 return child.isDirectory(); 142 } 143 }); 144 if (subDirs != null && subDirs.length == 1) { 145 File parent = new File(mediaDir, subDirs[0].getName()); 146 return new File(parent, MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 147 } else { 148 return new File(MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 149 } 150 } 151 152 static Collection<Object[]> bitstreams(String prefix, BitstreamPackage packageToRun) { 153 final String dynConfXml = new File("/", MediaBitstreams.DYNAMIC_CONFIG_XML).toString(); 154 try (InputStream is = MediaBitstreamsTest.class.getResourceAsStream(dynConfXml)) { 155 List<Object[]> entries = new ArrayList<>(); 156 XmlPullParser parser = XmlPullParserFactory.newInstance().newPullParser(); 157 parser.setInput(is, null); 158 parser.nextTag(); 159 parser.require(XmlPullParser.START_TAG, null, MediaBitstreams.DYNAMIC_CONFIG); 160 while (parser.next() != XmlPullParser.END_DOCUMENT) { 161 if (parser.getEventType() != XmlPullParser.START_TAG 162 || !MediaBitstreams.DYNAMIC_CONFIG_ENTRY.equals(parser.getName())) { 163 continue; 164 } 165 final String key = MediaBitstreams.DYNAMIC_CONFIG_KEY; 166 String bitstream = parser.getAttributeValue(null, key); 167 if (!bitstream.startsWith(prefix)) { 168 continue; 169 } 170 while (parser.next() != XmlPullParser.END_DOCUMENT) { 171 if (parser.getEventType() != XmlPullParser.START_TAG) { 172 continue; 173 } 174 if (MediaBitstreams.DYNAMIC_CONFIG_VALUE.equals(parser.getName())) { 175 parser.next(); 176 break; 177 } 178 } 179 String format = parser.getText(); 180 String[] kvPairs = format.split(","); 181 BitstreamPackage curPackage = BitstreamPackage.FULL; 182 boolean enforce = false; 183 for (String kvPair : kvPairs) { 184 String[] kv = kvPair.split("="); 185 if (MediaBitstreams.DYNAMIC_CONFIG_PACKAGE.equals(kv[0])) { 186 String packageName = kv[1]; 187 try { 188 curPackage = BitstreamPackage.valueOf(packageName.toUpperCase()); 189 } catch (Exception e) { 190 CLog.w(e); 191 } 192 } else if (MediaBitstreams.DYNAMIC_CONFIG_ENFORCE.equals(kv[0])) { 193 enforce = "true".equals(kv[1]); 194 } 195 } 196 if (curPackage.compareTo(packageToRun) <= 0) { 197 entries.add(new Object[] {prefix, bitstream, curPackage, packageToRun, enforce}); 198 } 199 } 200 return entries; 201 } catch (XmlPullParserException | IOException e) { 202 CLog.e(e); 203 return Collections.emptyList(); 204 } 205 } 206 207 public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun 208 ) { 209 this(prefix, path, pkg, packageToRun, false); 210 } 211 212 public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun, 213 boolean enforce) { 214 mPrefix = prefix; 215 mPath = path; 216 mPackage = pkg; 217 mPackageToRun = packageToRun; 218 mEnforce = enforce; 219 } 220 221 @Override 222 public void setBuild(IBuildInfo buildInfo) { 223 // Get the build, this is used to access the APK. 224 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 225 } 226 227 @Override 228 public void setAbi(IAbi abi) { 229 mAbi = abi; 230 } 231 232 @Override 233 public void setDevice(ITestDevice device) { 234 mDevice = device; 235 } 236 237 @Override 238 public ITestDevice getDevice() { 239 return mDevice; 240 } 241 242 /* 243 * Returns true if all necessary media files exist on the device, and false otherwise. 244 * 245 * This method is exposed for unit testing. 246 */ 247 private boolean bitstreamsExistOnDevice(ITestDevice device) 248 throws DeviceNotAvailableException { 249 return device.doesFileExist(mDeviceBitstreamsPath) 250 && device.isDirectory(mDeviceBitstreamsPath); 251 } 252 253 private String getCurrentMethod() { 254 return Thread.currentThread().getStackTrace()[2].getMethodName(); 255 } 256 257 private MetricsReportLog createReport(String methodName) { 258 String className = MediaBitstreamsTest.class.getCanonicalName(); 259 MetricsReportLog report = new MetricsReportLog( 260 mBuildHelper.getBuildInfo(), mAbi.getName(), 261 String.format("%s#%s", className, methodName), 262 MediaBitstreams.K_MODULE + "." + this.getClass().getSimpleName(), 263 "media_bitstreams_conformance", true); 264 return report; 265 } 266 267 /** 268 * @param method test method name in the form class#method 269 * @param p path to bitstream 270 * @param d decoder name 271 * @param s test status: unsupported, true, false, crash, or timeout. 272 */ 273 private void addConformanceEntry(String method, String p, String d, String s) { 274 MetricsReportLog report = createReport(method); 275 report.addValue(MediaBitstreams.KEY_PATH, p, ResultType.NEUTRAL, ResultUnit.NONE); 276 report.addValue(MediaBitstreams.KEY_CODEC_NAME, d, ResultType.NEUTRAL, ResultUnit.NONE); 277 report.addValue(MediaBitstreams.KEY_STATUS, s, ResultType.NEUTRAL, ResultUnit.NONE); 278 report.submit(); 279 280 ConformanceEntry ce = new ConformanceEntry(p, d, s); 281 mResults.putIfAbsent(p, new ArrayList<>()); 282 mResults.get(p).add(ce); 283 } 284 285 Map<String, String> getArgs() { 286 Map<String, String> args = new HashMap<>(); 287 args.put(MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, Boolean.toString(mDebugTargetDevice)); 288 args.put(MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, mDeviceBitstreamsPath); 289 return args; 290 } 291 292 private class ProcessBitstreamsFormats extends ReportProcessor { 293 294 @Override 295 void setUp(ITestDevice device) throws DeviceNotAvailableException { 296 if (mDownloadBitstreams || !bitstreamsExistOnDevice(device)) { 297 device.pushDir(mHostBitstreamsPath, mDeviceBitstreamsPath); 298 } 299 } 300 301 @Override 302 Map<String, String> getArgs() { 303 return MediaBitstreamsTest.this.getArgs(); 304 } 305 306 @Override 307 void process(ITestDevice device, String reportPath) 308 throws DeviceNotAvailableException, IOException { 309 File dynamicConfigFile = mBuildHelper.getTestFile(MediaBitstreams.K_MODULE + ".dynamic"); 310 device.pullFile(reportPath, dynamicConfigFile); 311 CLog.i("Pulled bitstreams formats to %s", dynamicConfigFile.getPath()); 312 } 313 314 } 315 316 private class ProcessBitstreamsValidation extends ReportProcessor { 317 318 Set<String> mBitstreams; 319 Deque<String> mProcessedBitstreams = new ArrayDeque<>(); 320 private final String mMethodName; 321 private final String mBitstreamsListTxt = new File( 322 mDeviceBitstreamsPath, 323 MediaBitstreams.K_BITSTREAMS_LIST_TXT).toString(); 324 private String mLastCrash; 325 326 ProcessBitstreamsValidation(Set<String> bitstreams, String methodName) { 327 mBitstreams = bitstreams; 328 mMethodName = methodName; 329 } 330 331 private String getBitstreamsListString() { 332 OutputStream baos = new ByteArrayOutputStream(); 333 PrintStream ps = new PrintStream(baos, true); 334 try { 335 for (String b : mBitstreams) { 336 ps.println(b); 337 } 338 return baos.toString(); 339 } finally { 340 ps.close(); 341 } 342 } 343 344 private void pushBitstreams(ITestDevice device) 345 throws IOException, DeviceNotAvailableException { 346 File tmp = null; 347 try { 348 CLog.i("Pushing %d bitstream(s) from %s to %s", 349 mBitstreams.size(), 350 mHostBitstreamsPath, 351 mDeviceBitstreamsPath); 352 tmp = Files.createTempDirectory(null).toFile(); 353 for (String b : mBitstreams) { 354 String m = MediaBitstreams.getMd5Path(b); 355 for (String f : new String[] {m, b}) { 356 File tmpf = new File(tmp, f); 357 new File(tmpf.getParent()).mkdirs(); 358 FileUtil.copyFile(new File(mHostBitstreamsPath, f), tmpf); 359 } 360 } 361 device.executeShellCommand(String.format("rm -rf %s", mDeviceBitstreamsPath)); 362 device.pushDir(tmp, mDeviceBitstreamsPath); 363 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 364 } finally { 365 FileUtil.recursiveDelete(tmp); 366 } 367 } 368 369 @Override 370 void setUp(ITestDevice device) throws DeviceNotAvailableException, IOException { 371 pushBitstreams(device); 372 } 373 374 @Override 375 Map<String, String> getArgs() { 376 Map<String, String> args = MediaBitstreamsTest.this.getArgs(); 377 if (mLastCrash != null) { 378 args.put(MediaBitstreams.OPT_LAST_CRASH, mLastCrash); 379 } 380 return args; 381 } 382 383 private void parse(ITestDevice device, String reportPath) 384 throws DeviceNotAvailableException { 385 String[] lines = getReportLines(device, reportPath); 386 mProcessedBitstreams.clear(); 387 for (int i = 0; i < lines.length;) { 388 389 String path = lines[i++]; 390 mProcessedBitstreams.add(path); 391 String errMsg; 392 393 boolean failedEarly; 394 if (i < lines.length) { 395 failedEarly = Boolean.parseBoolean(lines[i++]); 396 errMsg = failedEarly ? lines[i++] : ""; 397 } else { 398 failedEarly = true; 399 errMsg = MediaBitstreams.K_NATIVE_CRASH; 400 mLastCrash = MediaBitstreams.generateCrashSignature(path, ""); 401 mProcessedBitstreams.removeLast(); 402 } 403 404 if (failedEarly) { 405 addConformanceEntry(mMethodName, path, null, errMsg); 406 continue; 407 } 408 409 int n = Integer.parseInt(lines[i++]); 410 for (int j = 0; j < n && i < lines.length; j++) { 411 String decoderName = lines[i++]; 412 String result; 413 if (i < lines.length) { 414 result = lines[i++]; 415 } else { 416 result = MediaBitstreams.K_NATIVE_CRASH; 417 mLastCrash = MediaBitstreams.generateCrashSignature(path, decoderName); 418 mProcessedBitstreams.removeLast(); 419 } 420 addConformanceEntry(mMethodName, path, decoderName, result); 421 } 422 423 424 } 425 } 426 427 @Override 428 void process(ITestDevice device, String reportPath) 429 throws DeviceNotAvailableException, IOException { 430 parse(device, reportPath); 431 } 432 433 @Override 434 boolean recover(ITestDevice device, String reportPath) 435 throws DeviceNotAvailableException, IOException { 436 try { 437 parse(device, reportPath); 438 mBitstreams.removeAll(mProcessedBitstreams); 439 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 440 return true; 441 } catch (RuntimeException e) { 442 File hostFile = reportPath == null ? null : device.pullFile(reportPath); 443 CLog.e("Error parsing report; saving report to %s", hostFile); 444 CLog.e(e); 445 return false; 446 } 447 } 448 449 } 450 451 @Ignore 452 @Test 453 public void testGetBitstreamsFormats() throws DeviceNotAvailableException, IOException { 454 ReportProcessor processor = new ProcessBitstreamsFormats(); 455 processor.processDeviceReport( 456 getDevice(), 457 getCurrentMethod(), 458 MediaBitstreams.KEY_BITSTREAMS_FORMATS_XML); 459 } 460 461 @Test 462 public void testBitstreamsConformance() { 463 File bitstreamFile = new File(mHostBitstreamsPath, mPath); 464 if (!bitstreamFile.exists()) { 465 // todo(b/65165250): throw Exception once MediaPreparer can auto-download 466 CLog.w(bitstreamFile + " not found; skipping"); 467 return; 468 } 469 470 if (!mResults.containsKey(mPath)) { 471 try { 472 testBitstreamsConformance(mPrefix); 473 } catch (DeviceNotAvailableException | IOException e) { 474 String curMethod = getCurrentMethod(); 475 addConformanceEntry(curMethod, mPath, MediaBitstreams.K_UNAVAILABLE, e.toString()); 476 } 477 } 478 479 if (mEnforce) { 480 if (!mResults.containsKey(mPath)) { 481 Assert.fail("no results captured for " + mPath); 482 } 483 List<ConformanceEntry> entries = mResults.get(mPath); 484 for (ConformanceEntry ce : entries) { 485 if (!"true".equals(ce.mStatus) && !"unsupported".equals(ce.mStatus)) { 486 Assert.fail(ce.toString()); 487 } 488 } 489 } 490 491 } 492 493 private void testBitstreamsConformance(String prefix) 494 throws DeviceNotAvailableException, IOException { 495 496 ITestDevice device = getDevice(); 497 SupportedBitstreamsProcessor preparer; 498 preparer = new SupportedBitstreamsProcessor(prefix, mDebugTargetDevice); 499 preparer.processDeviceReport( 500 device, 501 MediaBitstreams.K_TEST_GET_SUPPORTED_BITSTREAMS, 502 MediaBitstreams.KEY_SUPPORTED_BITSTREAMS_TXT); 503 Collection<Object[]> bitstreams = bitstreams(mPrefix, mPackageToRun); 504 Set<String> supportedBitstreams = preparer.getSupportedBitstreams(); 505 CLog.i("%d supported bitstreams under %s", supportedBitstreams.size(), prefix); 506 507 int n = 0; 508 long size = 0; 509 long limit = device.getExternalStoreFreeSpace() * mUtilizationRate * 1024 / 100; 510 511 String curMethod = getCurrentMethod(); 512 Set<String> toPush = new LinkedHashSet<>(); 513 Iterator<Object[]> iter = bitstreams.iterator(); 514 515 for (int i = 0; i < bitstreams.size(); i++) { 516 517 if (n >= mNumBatches) { 518 break; 519 } 520 521 String p = (String) iter.next()[1]; 522 Map<String, Boolean> decoderCapabilities; 523 decoderCapabilities = preparer.getDecoderCapabilitiesForPath(p); 524 if (decoderCapabilities.isEmpty()) { 525 addConformanceEntry( 526 curMethod, p, 527 MediaBitstreams.K_UNAVAILABLE, 528 MediaBitstreams.K_UNSUPPORTED); 529 } 530 for (Entry<String, Boolean> entry : decoderCapabilities.entrySet()) { 531 Boolean supported = entry.getValue(); 532 if (supported) { 533 File bitstreamFile = new File(mHostBitstreamsPath, p); 534 String md5Path = MediaBitstreams.getMd5Path(p); 535 File md5File = new File(mHostBitstreamsPath, md5Path); 536 if (md5File.exists() && bitstreamFile.exists() && toPush.add(p)) { 537 size += md5File.length(); 538 size += bitstreamFile.length(); 539 } 540 } else { 541 String d = entry.getKey(); 542 addConformanceEntry(curMethod, p, d, MediaBitstreams.K_UNSUPPORTED); 543 } 544 } 545 546 if (size > limit || i + 1 == bitstreams.size()) { 547 ReportProcessor processor; 548 processor = new ProcessBitstreamsValidation(toPush, curMethod); 549 processor.processDeviceReport( 550 device, 551 curMethod, 552 MediaBitstreams.KEY_BITSTREAMS_VALIDATION_TXT); 553 toPush.clear(); 554 size = 0; 555 n++; 556 } 557 558 } 559 560 } 561 562 563 } 564
