1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts.bitstreams; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.targetprep.MediaPreparer; 20 import com.android.compatibility.common.util.MetricsReportLog; 21 import com.android.compatibility.common.util.ResultType; 22 import com.android.compatibility.common.util.ResultUnit; 23 import com.android.tradefed.build.IBuildInfo; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.config.OptionClass; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.testtype.IAbi; 30 import com.android.tradefed.testtype.IAbiReceiver; 31 import com.android.tradefed.testtype.IBuildReceiver; 32 import com.android.tradefed.testtype.IDeviceTest; 33 import com.android.tradefed.util.FileUtil; 34 import java.io.ByteArrayOutputStream; 35 import java.io.File; 36 import java.io.FileFilter; 37 import java.io.IOException; 38 import java.io.InputStream; 39 import java.io.OutputStream; 40 import java.io.PrintStream; 41 import java.nio.file.Files; 42 import java.util.ArrayDeque; 43 import java.util.ArrayList; 44 import java.util.Collection; 45 import java.util.Collections; 46 import java.util.Deque; 47 import java.util.HashMap; 48 import java.util.Iterator; 49 import java.util.LinkedHashSet; 50 import java.util.List; 51 import java.util.Map; 52 import java.util.Map.Entry; 53 import java.util.Set; 54 import java.util.concurrent.ConcurrentHashMap; 55 import java.util.concurrent.ConcurrentMap; 56 import org.junit.Ignore; 57 import org.junit.Test; 58 import org.xmlpull.v1.XmlPullParser; 59 import org.xmlpull.v1.XmlPullParserException; 60 import org.xmlpull.v1.XmlPullParserFactory; 61 62 /** 63 * Test that verifies video bitstreams decode pixel perfectly 64 */ 65 @OptionClass(alias="media-bitstreams-test") 66 public abstract class MediaBitstreamsTest implements IDeviceTest, IBuildReceiver, IAbiReceiver { 67 68 @Option(name = MediaBitstreams.OPT_HOST_BITSTREAMS_PATH, 69 description = "Absolute path of Ittiam bitstreams (host)", 70 mandatory = true) 71 private File mHostBitstreamsPath = getDefaultBitstreamsDir(); 72 73 @Option(name = MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, 74 description = "Absolute path of Ittiam bitstreams (device)") 75 private String mDeviceBitstreamsPath = MediaBitstreams.DEFAULT_DEVICE_BITSTEAMS_PATH; 76 77 @Option(name = MediaBitstreams.OPT_DOWNLOAD_BITSTREAMS, 78 description = "Whether to download the bitstreams files") 79 private boolean mDownloadBitstreams = false; 80 81 @Option(name = MediaBitstreams.OPT_UTILIZATION_RATE, 82 description = "Percentage of external storage space used for test") 83 private int mUtilizationRate = 80; 84 85 @Option(name = MediaBitstreams.OPT_NUM_BATCHES, 86 description = "Number of batches to test;" 87 + " each batch uses external storage up to utilization rate") 88 private int mNumBatches = Integer.MAX_VALUE; 89 90 @Option(name = MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, 91 description = "Whether to debug target device under test") 92 private boolean mDebugTargetDevice = false; 93 94 @Option(name = MediaBitstreams.OPT_BITSTREAMS_PREFIX, 95 description = "Only test bitstreams in this sub-directory") 96 private String mPrefix = ""; 97 98 private String mPath = ""; 99 100 private static ConcurrentMap<String, List<ConformanceEntry>> mResults = new ConcurrentHashMap<>(); 101 102 /** 103 * Which subset of bitstreams to test 104 */ 105 enum BitstreamPackage { 106 STANDARD, 107 FULL, 108 } 109 110 private BitstreamPackage mPackage = BitstreamPackage.FULL; 111 private BitstreamPackage mPackageToRun = BitstreamPackage.STANDARD; 112 113 static class ConformanceEntry { 114 final String mPath, mCodecName, mStatus; 115 ConformanceEntry(String path, String codecName, String status) { 116 mPath = path; 117 mCodecName = codecName; 118 mStatus = status; 119 } 120 @Override 121 public String toString() { 122 return String.format("%s,%s,%s", mPath, mCodecName, mStatus); 123 } 124 } 125 126 /** 127 * A helper to access resources in the build. 128 */ 129 private CompatibilityBuildHelper mBuildHelper; 130 131 private IAbi mAbi; 132 private ITestDevice mDevice; 133 134 static File getDefaultBitstreamsDir() { 135 File mediaDir = MediaPreparer.getDefaultMediaDir(); 136 File[] subDirs = mediaDir.listFiles(new FileFilter() { 137 @Override 138 public boolean accept(File child) { 139 return child.isDirectory(); 140 } 141 }); 142 if (subDirs != null && subDirs.length == 1) { 143 File parent = new File(mediaDir, subDirs[0].getName()); 144 return new File(parent, MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 145 } else { 146 return new File(MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 147 } 148 } 149 150 static Collection<Object[]> bitstreams(String prefix, BitstreamPackage packageToRun) { 151 final String dynConfXml = new File("/", MediaBitstreams.DYNAMIC_CONFIG_XML).toString(); 152 try (InputStream is = MediaBitstreamsTest.class.getResourceAsStream(dynConfXml)) { 153 List<Object[]> entries = new ArrayList<>(); 154 XmlPullParser parser = XmlPullParserFactory.newInstance().newPullParser(); 155 parser.setInput(is, null); 156 parser.nextTag(); 157 parser.require(XmlPullParser.START_TAG, null, MediaBitstreams.DYNAMIC_CONFIG); 158 while (parser.next() != XmlPullParser.END_DOCUMENT) { 159 if (parser.getEventType() != XmlPullParser.START_TAG 160 || !MediaBitstreams.DYNAMIC_CONFIG_ENTRY.equals(parser.getName())) { 161 continue; 162 } 163 final String key = MediaBitstreams.DYNAMIC_CONFIG_KEY; 164 String bitstream = parser.getAttributeValue(null, key); 165 if (!bitstream.startsWith(prefix)) { 166 continue; 167 } 168 while (parser.next() != XmlPullParser.END_DOCUMENT) { 169 if (parser.getEventType() != XmlPullParser.START_TAG) { 170 continue; 171 } 172 if (MediaBitstreams.DYNAMIC_CONFIG_VALUE.equals(parser.getName())) { 173 parser.next(); 174 break; 175 } 176 } 177 String format = parser.getText(); 178 String[] kvPairs = format.split(","); 179 BitstreamPackage curPackage = BitstreamPackage.FULL; 180 for (String kvPair : kvPairs) { 181 String[] kv = kvPair.split("="); 182 if (MediaBitstreams.DYNAMIC_CONFIG_PACKAGE.equals(kv[0])) { 183 String packageName = kv[1]; 184 try { 185 curPackage = BitstreamPackage.valueOf(packageName.toUpperCase()); 186 } catch (Exception e) { 187 CLog.w(e); 188 } 189 } 190 } 191 if (curPackage.compareTo(packageToRun) <= 0) { 192 entries.add(new Object[] {prefix, bitstream, curPackage, packageToRun}); 193 } 194 } 195 return entries; 196 } catch (XmlPullParserException | IOException e) { 197 CLog.e(e); 198 return Collections.emptyList(); 199 } 200 } 201 202 public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun 203 ) { 204 mPrefix = prefix; 205 mPath = path; 206 mPackage = pkg; 207 mPackageToRun = packageToRun; 208 } 209 210 @Override 211 public void setBuild(IBuildInfo buildInfo) { 212 // Get the build, this is used to access the APK. 213 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 214 } 215 216 @Override 217 public void setAbi(IAbi abi) { 218 mAbi = abi; 219 } 220 221 @Override 222 public void setDevice(ITestDevice device) { 223 mDevice = device; 224 } 225 226 @Override 227 public ITestDevice getDevice() { 228 return mDevice; 229 } 230 231 /* 232 * Returns true if all necessary media files exist on the device, and false otherwise. 233 * 234 * This method is exposed for unit testing. 235 */ 236 private boolean bitstreamsExistOnDevice(ITestDevice device) 237 throws DeviceNotAvailableException { 238 return device.doesFileExist(mDeviceBitstreamsPath) 239 && device.isDirectory(mDeviceBitstreamsPath); 240 } 241 242 private String getCurrentMethod() { 243 return Thread.currentThread().getStackTrace()[2].getMethodName(); 244 } 245 246 private MetricsReportLog createReport(String methodName) { 247 String className = MediaBitstreamsTest.class.getCanonicalName(); 248 MetricsReportLog report = new MetricsReportLog( 249 mBuildHelper.getBuildInfo(), mAbi.getName(), 250 String.format("%s#%s", className, methodName), 251 MediaBitstreams.K_MODULE + "." + this.getClass().getSimpleName(), 252 "media_bitstreams_conformance", true); 253 return report; 254 } 255 256 /** 257 * @param method test method name in the form class#method 258 * @param p path to bitstream 259 * @param d decoder name 260 * @param s test status: unsupported, true, false, crash, or timeout. 261 */ 262 private void addConformanceEntry(String method, String p, String d, String s) { 263 MetricsReportLog report = createReport(method); 264 report.addValue(MediaBitstreams.KEY_PATH, p, ResultType.NEUTRAL, ResultUnit.NONE); 265 report.addValue(MediaBitstreams.KEY_CODEC_NAME, d, ResultType.NEUTRAL, ResultUnit.NONE); 266 report.addValue(MediaBitstreams.KEY_STATUS, s, ResultType.NEUTRAL, ResultUnit.NONE); 267 report.submit(); 268 269 ConformanceEntry ce = new ConformanceEntry(p, d, s); 270 mResults.putIfAbsent(p, new ArrayList<>()); 271 mResults.get(p).add(ce); 272 } 273 274 Map<String, String> getArgs() { 275 Map<String, String> args = new HashMap<>(); 276 args.put(MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, Boolean.toString(mDebugTargetDevice)); 277 args.put(MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, mDeviceBitstreamsPath); 278 return args; 279 } 280 281 private class ProcessBitstreamsFormats extends ReportProcessor { 282 283 @Override 284 void setUp(ITestDevice device) throws DeviceNotAvailableException { 285 if (mDownloadBitstreams || !bitstreamsExistOnDevice(device)) { 286 device.pushDir(mHostBitstreamsPath, mDeviceBitstreamsPath); 287 } 288 } 289 290 @Override 291 Map<String, String> getArgs() { 292 return MediaBitstreamsTest.this.getArgs(); 293 } 294 295 @Override 296 void process(ITestDevice device, String reportPath) 297 throws DeviceNotAvailableException, IOException { 298 File dynamicConfigFile = mBuildHelper.getTestFile(MediaBitstreams.K_MODULE + ".dynamic"); 299 device.pullFile(reportPath, dynamicConfigFile); 300 CLog.i("Pulled bitstreams formats to %s", dynamicConfigFile.getPath()); 301 } 302 303 } 304 305 private class ProcessBitstreamsValidation extends ReportProcessor { 306 307 Set<String> mBitstreams; 308 Deque<String> mProcessedBitstreams = new ArrayDeque<>(); 309 private final String mMethodName; 310 private final String mBitstreamsListTxt = new File( 311 mDeviceBitstreamsPath, 312 MediaBitstreams.K_BITSTREAMS_LIST_TXT).toString(); 313 private String mLastCrash; 314 315 ProcessBitstreamsValidation(Set<String> bitstreams, String methodName) { 316 mBitstreams = bitstreams; 317 mMethodName = methodName; 318 } 319 320 private String getBitstreamsListString() { 321 OutputStream baos = new ByteArrayOutputStream(); 322 PrintStream ps = new PrintStream(baos, true); 323 try { 324 for (String b : mBitstreams) { 325 ps.println(b); 326 } 327 return baos.toString(); 328 } finally { 329 ps.close(); 330 } 331 } 332 333 private void pushBitstreams(ITestDevice device) 334 throws IOException, DeviceNotAvailableException { 335 File tmp = null; 336 try { 337 CLog.i("Pushing %d bitstream(s) from %s to %s", 338 mBitstreams.size(), 339 mHostBitstreamsPath, 340 mDeviceBitstreamsPath); 341 tmp = Files.createTempDirectory(null).toFile(); 342 for (String b : mBitstreams) { 343 String m = MediaBitstreams.getMd5Path(b); 344 for (String f : new String[] {m, b}) { 345 File tmpf = new File(tmp, f); 346 new File(tmpf.getParent()).mkdirs(); 347 FileUtil.copyFile(new File(mHostBitstreamsPath, f), tmpf); 348 } 349 } 350 device.executeShellCommand(String.format("rm -rf %s", mDeviceBitstreamsPath)); 351 device.pushDir(tmp, mDeviceBitstreamsPath); 352 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 353 } finally { 354 FileUtil.recursiveDelete(tmp); 355 } 356 } 357 358 @Override 359 void setUp(ITestDevice device) throws DeviceNotAvailableException, IOException { 360 pushBitstreams(device); 361 } 362 363 @Override 364 Map<String, String> getArgs() { 365 Map<String, String> args = MediaBitstreamsTest.this.getArgs(); 366 if (mLastCrash != null) { 367 args.put(MediaBitstreams.OPT_LAST_CRASH, mLastCrash); 368 } 369 return args; 370 } 371 372 private void parse(ITestDevice device, String reportPath) 373 throws DeviceNotAvailableException { 374 String[] lines = getReportLines(device, reportPath); 375 mProcessedBitstreams.clear(); 376 for (int i = 0; i < lines.length;) { 377 378 String path = lines[i++]; 379 mProcessedBitstreams.add(path); 380 String errMsg; 381 382 boolean failedEarly; 383 if (i < lines.length) { 384 failedEarly = Boolean.parseBoolean(lines[i++]); 385 errMsg = failedEarly ? lines[i++] : ""; 386 } else { 387 failedEarly = true; 388 errMsg = MediaBitstreams.K_NATIVE_CRASH; 389 mLastCrash = MediaBitstreams.generateCrashSignature(path, ""); 390 mProcessedBitstreams.removeLast(); 391 } 392 393 if (failedEarly) { 394 addConformanceEntry(mMethodName, path, null, errMsg); 395 continue; 396 } 397 398 int n = Integer.parseInt(lines[i++]); 399 for (int j = 0; j < n && i < lines.length; j++) { 400 String decoderName = lines[i++]; 401 String result; 402 if (i < lines.length) { 403 result = lines[i++]; 404 } else { 405 result = MediaBitstreams.K_NATIVE_CRASH; 406 mLastCrash = MediaBitstreams.generateCrashSignature(path, decoderName); 407 mProcessedBitstreams.removeLast(); 408 } 409 addConformanceEntry(mMethodName, path, decoderName, result); 410 } 411 412 413 } 414 } 415 416 @Override 417 void process(ITestDevice device, String reportPath) 418 throws DeviceNotAvailableException, IOException { 419 parse(device, reportPath); 420 } 421 422 @Override 423 boolean recover(ITestDevice device, String reportPath) 424 throws DeviceNotAvailableException, IOException { 425 try { 426 parse(device, reportPath); 427 mBitstreams.removeAll(mProcessedBitstreams); 428 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 429 return true; 430 } catch (RuntimeException e) { 431 File hostFile = reportPath == null ? null : device.pullFile(reportPath); 432 CLog.e("Error parsing report; saving report to %s", hostFile); 433 CLog.e(e); 434 return false; 435 } 436 } 437 438 } 439 440 @Ignore 441 @Test 442 public void testGetBitstreamsFormats() throws DeviceNotAvailableException, IOException { 443 ReportProcessor processor = new ProcessBitstreamsFormats(); 444 processor.processDeviceReport( 445 getDevice(), 446 getCurrentMethod(), 447 MediaBitstreams.KEY_BITSTREAMS_FORMATS_XML); 448 } 449 450 @Test 451 public void testBitstreamsConformance() { 452 File bitstreamFile = new File(mHostBitstreamsPath, mPath); 453 if (!bitstreamFile.exists()) { 454 // todo(b/65165250): throw Exception once MediaPreparer can auto-download 455 CLog.w(bitstreamFile + " not found; skipping"); 456 return; 457 } 458 459 if (!mResults.containsKey(mPath)) { 460 try { 461 testBitstreamsConformance(mPrefix); 462 } catch (DeviceNotAvailableException | IOException e) { 463 String curMethod = getCurrentMethod(); 464 addConformanceEntry(curMethod, mPath, MediaBitstreams.K_UNAVAILABLE, e.toString()); 465 } 466 } 467 // todo(robertshih): lookup conformance entry; pass/fail based on lookup result 468 } 469 470 private void testBitstreamsConformance(String prefix) 471 throws DeviceNotAvailableException, IOException { 472 473 ITestDevice device = getDevice(); 474 SupportedBitstreamsProcessor preparer; 475 preparer = new SupportedBitstreamsProcessor(prefix, mDebugTargetDevice); 476 preparer.processDeviceReport( 477 device, 478 MediaBitstreams.K_TEST_GET_SUPPORTED_BITSTREAMS, 479 MediaBitstreams.KEY_SUPPORTED_BITSTREAMS_TXT); 480 Collection<Object[]> bitstreams = bitstreams(mPrefix, mPackageToRun); 481 Set<String> supportedBitstreams = preparer.getSupportedBitstreams(); 482 CLog.i("%d supported bitstreams under %s", supportedBitstreams.size(), prefix); 483 484 int n = 0; 485 long size = 0; 486 long limit = device.getExternalStoreFreeSpace() * mUtilizationRate * 1024 / 100; 487 488 String curMethod = getCurrentMethod(); 489 Set<String> toPush = new LinkedHashSet<>(); 490 Iterator<Object[]> iter = bitstreams.iterator(); 491 492 for (int i = 0; i < bitstreams.size(); i++) { 493 494 if (n >= mNumBatches) { 495 break; 496 } 497 498 String p = (String) iter.next()[1]; 499 Map<String, Boolean> decoderCapabilities; 500 decoderCapabilities = preparer.getDecoderCapabilitiesForPath(p); 501 if (decoderCapabilities.isEmpty()) { 502 addConformanceEntry( 503 curMethod, p, 504 MediaBitstreams.K_UNAVAILABLE, 505 MediaBitstreams.K_UNSUPPORTED); 506 } 507 for (Entry<String, Boolean> entry : decoderCapabilities.entrySet()) { 508 Boolean supported = entry.getValue(); 509 if (supported) { 510 File bitstreamFile = new File(mHostBitstreamsPath, p); 511 String md5Path = MediaBitstreams.getMd5Path(p); 512 File md5File = new File(mHostBitstreamsPath, md5Path); 513 if (md5File.exists() && bitstreamFile.exists() && toPush.add(p)) { 514 size += md5File.length(); 515 size += bitstreamFile.length(); 516 } 517 } else { 518 String d = entry.getKey(); 519 addConformanceEntry(curMethod, p, d, MediaBitstreams.K_UNSUPPORTED); 520 } 521 } 522 523 if (size > limit || i + 1 == bitstreams.size()) { 524 ReportProcessor processor; 525 processor = new ProcessBitstreamsValidation(toPush, curMethod); 526 processor.processDeviceReport( 527 device, 528 curMethod, 529 MediaBitstreams.KEY_BITSTREAMS_VALIDATION_TXT); 530 toPush.clear(); 531 size = 0; 532 n++; 533 } 534 535 } 536 537 } 538 539 540 } 541
