1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.media.tests; 17 18 import com.android.media.tests.AudioLoopbackImageAnalyzer.Result; 19 import com.android.tradefed.device.DeviceNotAvailableException; 20 import com.android.tradefed.device.ITestDevice; 21 import com.android.tradefed.log.LogUtil.CLog; 22 import com.android.tradefed.util.Pair; 23 24 import com.google.common.io.Files; 25 26 import java.io.BufferedReader; 27 import java.io.File; 28 import java.io.FileNotFoundException; 29 import java.io.IOException; 30 import java.io.PrintWriter; 31 import java.io.UnsupportedEncodingException; 32 import java.nio.charset.StandardCharsets; 33 import java.time.Instant; 34 import java.util.ArrayList; 35 import java.util.Arrays; 36 import java.util.Collections; 37 import java.util.Comparator; 38 import java.util.HashMap; 39 import java.util.List; 40 import java.util.Map; 41 42 /** Helper class for AudioLoopbackTest. It keeps runtime data, analytics, */ 43 public class AudioLoopbackTestHelper { 44 45 private StatisticsData mLatencyStats = null; 46 private StatisticsData mConfidenceStats = null; 47 private ArrayList<ResultData> mAllResults; 48 private ArrayList<ResultData> mGoodResults = new ArrayList<ResultData>(); 49 private ArrayList<ResultData> mBadResults = new ArrayList<ResultData>(); 50 private ArrayList<Map<String, String>> mResultDictionaries = 51 new ArrayList<Map<String, String>>(); 52 53 // Controls acceptable tolerance in ms around median latency 54 private static final double TOLERANCE = 2.0; 55 56 //=================================================================== 57 // ENUMS 58 //=================================================================== 59 public enum LogFileType { 60 RESULT, 61 WAVE, 62 GRAPH, 63 PLAYER_BUFFER, 64 PLAYER_BUFFER_HISTOGRAM, 65 PLAYER_BUFFER_PERIOD_TIMES, 66 RECORDER_BUFFER, 67 RECORDER_BUFFER_HISTOGRAM, 68 RECORDER_BUFFER_PERIOD_TIMES, 69 GLITCHES_MILLIS, 70 HEAT_MAP, 71 LOGCAT 72 } 73 74 //=================================================================== 75 // INNER CLASSES 76 //=================================================================== 77 private class StatisticsData { 78 double mMin = 0; 79 double mMax = 0; 80 double mMean = 0; 81 double mMedian = 0; 82 83 @Override 84 public String toString() { 85 return String.format( 86 "min = %1$f, max = %2$f, median=%3$f, mean = %4$f", mMin, mMax, mMedian, mMean); 87 } 88 } 89 90 /** ResultData is an inner class that holds results and logfile info from each test run */ 91 public static class ResultData { 92 private Float mLatencyMs; 93 private Float mLatencyConfidence; 94 private Integer mAudioLevel; 95 private Integer mIteration; 96 private Long mDeviceTestStartTime; 97 private boolean mIsTimedOut = false; 98 private HashMap<LogFileType, String> mLogs = new HashMap<LogFileType, String>(); 99 private Result mImageAnalyzerResult = Result.UNKNOWN; 100 private String mFailureReason = null; 101 102 // Optional 103 private Float mPeriodConfidence = null; 104 private Float mRms = null; 105 private Float mRmsAverage = null; 106 private Integer mBblockSize = null; 107 108 public float getLatency() { 109 return mLatencyMs.floatValue(); 110 } 111 112 public void setLatency(float latencyMs) { 113 this.mLatencyMs = Float.valueOf(latencyMs); 114 } 115 116 public boolean hasLatency() { 117 return mLatencyMs != null; 118 } 119 120 public float getConfidence() { 121 return mLatencyConfidence.floatValue(); 122 } 123 124 public void setConfidence(float latencyConfidence) { 125 this.mLatencyConfidence = Float.valueOf(latencyConfidence); 126 } 127 128 public boolean hasConfidence() { 129 return mLatencyConfidence != null; 130 } 131 132 public float getPeriodConfidence() { 133 return mPeriodConfidence.floatValue(); 134 } 135 136 public void setPeriodConfidence(float periodConfidence) { 137 this.mPeriodConfidence = Float.valueOf(periodConfidence); 138 } 139 140 public boolean hasPeriodConfidence() { 141 return mPeriodConfidence != null; 142 } 143 144 public float getRMS() { 145 return mRms.floatValue(); 146 } 147 148 public void setRMS(float rms) { 149 this.mRms = Float.valueOf(rms); 150 } 151 152 public boolean hasRMS() { 153 return mRms != null; 154 } 155 156 public float getRMSAverage() { 157 return mRmsAverage.floatValue(); 158 } 159 160 public void setRMSAverage(float rmsAverage) { 161 this.mRmsAverage = Float.valueOf(rmsAverage); 162 } 163 164 public boolean hasRMSAverage() { 165 return mRmsAverage != null; 166 } 167 168 public int getAudioLevel() { 169 return mAudioLevel.intValue(); 170 } 171 172 public void setAudioLevel(int audioLevel) { 173 this.mAudioLevel = Integer.valueOf(audioLevel); 174 } 175 176 public boolean hasAudioLevel() { 177 return mAudioLevel != null; 178 } 179 180 public int getBlockSize() { 181 return mBblockSize.intValue(); 182 } 183 184 public void setBlockSize(int blockSize) { 185 this.mBblockSize = Integer.valueOf(blockSize); 186 } 187 188 public boolean hasBlockSize() { 189 return mBblockSize != null; 190 } 191 192 public int getIteration() { 193 return mIteration.intValue(); 194 } 195 196 public void setIteration(int iteration) { 197 this.mIteration = Integer.valueOf(iteration); 198 } 199 200 public long getDeviceTestStartTime() { 201 return mDeviceTestStartTime.longValue(); 202 } 203 204 public void setDeviceTestStartTime(long deviceTestStartTime) { 205 this.mDeviceTestStartTime = Long.valueOf(deviceTestStartTime); 206 } 207 208 public Result getImageAnalyzerResult() { 209 return mImageAnalyzerResult; 210 } 211 212 public void setImageAnalyzerResult(Result imageAnalyzerResult) { 213 this.mImageAnalyzerResult = imageAnalyzerResult; 214 } 215 216 public String getFailureReason() { 217 return mFailureReason; 218 } 219 220 public void setFailureReason(String failureReason) { 221 this.mFailureReason = failureReason; 222 } 223 224 public boolean isTimedOut() { 225 return mIsTimedOut; 226 } 227 228 public void setIsTimedOut(boolean isTimedOut) { 229 this.mIsTimedOut = isTimedOut; 230 } 231 232 public String getLogFile(LogFileType log) { 233 return mLogs.get(log); 234 } 235 236 public void setLogFile(LogFileType log, String filename) { 237 CLog.i("setLogFile: type=" + log.name() + ", filename=" + filename); 238 if (!mLogs.containsKey(log) && filename != null && !filename.isEmpty()) { 239 mLogs.put(log, filename); 240 } 241 } 242 243 public boolean hasBadResults() { 244 return hasTimedOut() 245 || hasNoTestResults() 246 || !hasLatency() 247 || !hasConfidence() 248 || mImageAnalyzerResult == Result.FAIL; 249 } 250 251 public boolean hasTimedOut() { 252 return mIsTimedOut; 253 } 254 255 public boolean hasLogFile(LogFileType log) { 256 return mLogs.containsKey(log); 257 } 258 259 public boolean hasNoTestResults() { 260 return !hasConfidence() && !hasLatency(); 261 } 262 263 public static Comparator<ResultData> latencyComparator = 264 new Comparator<ResultData>() { 265 @Override 266 public int compare(ResultData o1, ResultData o2) { 267 return o1.mLatencyMs.compareTo(o2.mLatencyMs); 268 } 269 }; 270 271 public static Comparator<ResultData> confidenceComparator = 272 new Comparator<ResultData>() { 273 @Override 274 public int compare(ResultData o1, ResultData o2) { 275 return o1.mLatencyConfidence.compareTo(o2.mLatencyConfidence); 276 } 277 }; 278 279 public static Comparator<ResultData> iteratorComparator = 280 new Comparator<ResultData>() { 281 @Override 282 public int compare(ResultData o1, ResultData o2) { 283 return Integer.compare(o1.mIteration, o2.mIteration); 284 } 285 }; 286 287 @Override 288 public String toString() { 289 final String NL = "\n"; 290 final StringBuilder sb = new StringBuilder(512); 291 sb.append("{").append(NL); 292 sb.append("{\nlatencyMs=").append(mLatencyMs).append(NL); 293 sb.append("latencyConfidence=").append(mLatencyConfidence).append(NL); 294 sb.append("isTimedOut=").append(mIsTimedOut).append(NL); 295 sb.append("iteration=").append(mIteration).append(NL); 296 sb.append("logs=").append(Arrays.toString(mLogs.values().toArray())).append(NL); 297 sb.append("audioLevel=").append(mAudioLevel).append(NL); 298 sb.append("deviceTestStartTime=").append(mDeviceTestStartTime).append(NL); 299 sb.append("rms=").append(mRms).append(NL); 300 sb.append("rmsAverage=").append(mRmsAverage).append(NL); 301 sb.append("}").append(NL); 302 return sb.toString(); 303 } 304 } 305 306 public AudioLoopbackTestHelper(int iterations) { 307 mAllResults = new ArrayList<ResultData>(iterations); 308 } 309 310 public void addTestData(ResultData data, 311 Map<String, 312 String> resultDictionary, 313 boolean useImageAnalyzer) { 314 mResultDictionaries.add(data.getIteration(), resultDictionary); 315 mAllResults.add(data); 316 317 if (useImageAnalyzer && data.hasLogFile(LogFileType.GRAPH)) { 318 // Analyze captured screenshot to see if wave form is within reason 319 final String screenshot = data.getLogFile(LogFileType.GRAPH); 320 final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot); 321 data.setImageAnalyzerResult(result.first); 322 data.setFailureReason(result.second); 323 } 324 } 325 326 public final List<ResultData> getAllTestData() { 327 return mAllResults; 328 } 329 330 public Map<String, String> getResultDictionaryForIteration(int i) { 331 return mResultDictionaries.get(i); 332 } 333 334 /** 335 * Returns a list of the worst test result objects, up to maxNrOfWorstResults 336 * 337 * <p> 338 * 339 * <ol> 340 * <li> Tests in the bad results list are added first 341 * <li> If still space, add test results based on low confidence and then tests that are 342 * outside tolerance boundaries 343 * </ol> 344 * 345 * @param maxNrOfWorstResults 346 * @return list of worst test result objects 347 */ 348 public List<ResultData> getWorstResults(int maxNrOfWorstResults) { 349 int counter = 0; 350 final ArrayList<ResultData> worstResults = new ArrayList<ResultData>(maxNrOfWorstResults); 351 352 for (final ResultData data : mBadResults) { 353 if (counter < maxNrOfWorstResults) { 354 worstResults.add(data); 355 counter++; 356 } 357 } 358 359 for (final ResultData data : mGoodResults) { 360 if (counter < maxNrOfWorstResults) { 361 boolean failed = false; 362 if (data.getConfidence() < 1.0f) { 363 data.setFailureReason("Low confidence"); 364 failed = true; 365 } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) 366 || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { 367 data.setFailureReason("Latency not within tolerance from median"); 368 failed = true; 369 } 370 371 if (failed) { 372 worstResults.add(data); 373 counter++; 374 } 375 } 376 } 377 378 return worstResults; 379 } 380 381 public static Map<String, String> parseKeyValuePairFromFile( 382 File result, 383 final Map<String, String> dictionary, 384 final String resultKeyPrefix, 385 final String splitOn, 386 final String keyValueFormat) 387 throws IOException { 388 389 final Map<String, String> resultMap = new HashMap<String, String>(); 390 final BufferedReader br = Files.newReader(result, StandardCharsets.UTF_8); 391 392 try { 393 String line = br.readLine(); 394 while (line != null) { 395 line = line.trim().replaceAll(" +", " "); 396 final String[] tokens = line.split(splitOn); 397 if (tokens.length >= 2) { 398 final String key = tokens[0].trim(); 399 final String value = tokens[1].trim(); 400 if (dictionary.containsKey(key)) { 401 CLog.i(String.format(keyValueFormat, key, value)); 402 resultMap.put(resultKeyPrefix + dictionary.get(key), value); 403 } 404 } 405 line = br.readLine(); 406 } 407 } finally { 408 br.close(); 409 } 410 return resultMap; 411 } 412 413 public int processTestData() { 414 415 // Collect statistics about the test run 416 int nrOfValidResults = 0; 417 double sumLatency = 0; 418 double sumConfidence = 0; 419 420 final int totalNrOfTests = mAllResults.size(); 421 mLatencyStats = new StatisticsData(); 422 mConfidenceStats = new StatisticsData(); 423 mBadResults = new ArrayList<ResultData>(); 424 mGoodResults = new ArrayList<ResultData>(totalNrOfTests); 425 426 // Copy all results into Good results list 427 mGoodResults.addAll(mAllResults); 428 429 for (final ResultData data : mAllResults) { 430 if (data.hasBadResults()) { 431 mBadResults.add(data); 432 continue; 433 } 434 // Get mean values 435 sumLatency += data.getLatency(); 436 sumConfidence += data.getConfidence(); 437 } 438 439 if (!mBadResults.isEmpty()) { 440 analyzeBadResults(mBadResults, mAllResults.size()); 441 } 442 443 // Remove bad runs from result array 444 mGoodResults.removeAll(mBadResults); 445 446 // Fail test immediately if we don't have ANY good results 447 if (mGoodResults.isEmpty()) { 448 return 0; 449 } 450 451 nrOfValidResults = mGoodResults.size(); 452 453 // ---- LATENCY: Get Median, Min and Max values ---- 454 Collections.sort(mGoodResults, ResultData.latencyComparator); 455 456 mLatencyStats.mMin = mGoodResults.get(0).mLatencyMs; 457 mLatencyStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyMs; 458 mLatencyStats.mMean = sumLatency / nrOfValidResults; 459 // Is array even or odd numbered 460 if (nrOfValidResults % 2 == 0) { 461 final int middle = nrOfValidResults / 2; 462 final float middleLeft = mGoodResults.get(middle - 1).mLatencyMs; 463 final float middleRight = mGoodResults.get(middle).mLatencyMs; 464 mLatencyStats.mMedian = (middleLeft + middleRight) / 2.0f; 465 } else { 466 // It's and odd numbered array, just grab the middle value 467 mLatencyStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyMs; 468 } 469 470 // ---- CONFIDENCE: Get Median, Min and Max values ---- 471 Collections.sort(mGoodResults, ResultData.confidenceComparator); 472 473 mConfidenceStats.mMin = mGoodResults.get(0).mLatencyConfidence; 474 mConfidenceStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyConfidence; 475 mConfidenceStats.mMean = sumConfidence / nrOfValidResults; 476 // Is array even or odd numbered 477 if (nrOfValidResults % 2 == 0) { 478 final int middle = nrOfValidResults / 2; 479 final float middleLeft = mGoodResults.get(middle - 1).mLatencyConfidence; 480 final float middleRight = mGoodResults.get(middle).mLatencyConfidence; 481 mConfidenceStats.mMedian = (middleLeft + middleRight) / 2.0f; 482 } else { 483 // It's and odd numbered array, just grab the middle value 484 mConfidenceStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyConfidence; 485 } 486 487 for (final ResultData data : mGoodResults) { 488 // Check if within Latency Tolerance 489 if (data.getConfidence() < 1.0f) { 490 data.setFailureReason("Low confidence"); 491 } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) 492 || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { 493 data.setFailureReason("Latency not within tolerance from median"); 494 } 495 } 496 497 // Create histogram 498 // Strategy: Create buckets based on whole ints, like 16 ms, 17 ms, 18 ms etc. Count how 499 // many tests fall into each bucket. Just cast the float to an int, no rounding up/down 500 // required. 501 final int[] histogram = new int[(int) mLatencyStats.mMax + 1]; 502 for (final ResultData rd : mGoodResults) { 503 // Increase value in bucket 504 histogram[(int) (rd.mLatencyMs.floatValue())]++; 505 } 506 507 CLog.i("========== VALID RESULTS ============================================"); 508 CLog.i(String.format("Valid tests: %1$d of %2$d", nrOfValidResults, totalNrOfTests)); 509 CLog.i("Latency: " + mLatencyStats.toString()); 510 CLog.i("Confidence: " + mConfidenceStats.toString()); 511 CLog.i("========== HISTOGRAM ================================================"); 512 for (int i = 0; i < histogram.length; i++) { 513 if (histogram[i] > 0) { 514 CLog.i(String.format("%1$01d ms => %2$d", i, histogram[i])); 515 } 516 } 517 518 // VERIFY the good results by running image analysis on the 519 // screenshot of the incoming audio waveform 520 521 return nrOfValidResults; 522 } 523 524 public void writeAllResultsToCSVFile(File csvFile, ITestDevice device) 525 throws DeviceNotAvailableException, FileNotFoundException, 526 UnsupportedEncodingException { 527 528 final String deviceType = device.getProperty("ro.build.product"); 529 final String buildId = device.getBuildAlias(); 530 final String serialNumber = device.getSerialNumber(); 531 532 // Sort data on iteration 533 Collections.sort(mAllResults, ResultData.iteratorComparator); 534 535 final StringBuilder sb = new StringBuilder(256); 536 final PrintWriter writer = new PrintWriter(csvFile, StandardCharsets.UTF_8.name()); 537 final String SEPARATOR = ","; 538 539 // Write column labels 540 writer.println( 541 "Device Time,Device Type,Build Id,Serial Number,Iteration,Latency," 542 + "Confidence,Period Confidence,Block Size,Audio Level,RMS,RMS Average," 543 + "Image Analysis,Failure Reason"); 544 for (final ResultData data : mAllResults) { 545 final Instant instant = Instant.ofEpochSecond(data.mDeviceTestStartTime); 546 547 sb.append(instant).append(SEPARATOR); 548 sb.append(deviceType).append(SEPARATOR); 549 sb.append(buildId).append(SEPARATOR); 550 sb.append(serialNumber).append(SEPARATOR); 551 sb.append(data.getIteration()).append(SEPARATOR); 552 sb.append(data.hasLatency() ? data.getLatency() : "").append(SEPARATOR); 553 sb.append(data.hasConfidence() ? data.getConfidence() : "").append(SEPARATOR); 554 sb.append(data.hasPeriodConfidence() ? data.getPeriodConfidence() : "").append(SEPARATOR); 555 sb.append(data.hasBlockSize() ? data.getBlockSize() : "").append(SEPARATOR); 556 sb.append(data.hasAudioLevel() ? data.getAudioLevel() : "").append(SEPARATOR); 557 sb.append(data.hasRMS() ? data.getRMS() : "").append(SEPARATOR); 558 sb.append(data.hasRMSAverage() ? data.getRMSAverage() : "").append(SEPARATOR); 559 sb.append(data.getImageAnalyzerResult().name()).append(SEPARATOR); 560 sb.append(data.getFailureReason()); 561 562 writer.println(sb.toString()); 563 564 sb.setLength(0); 565 } 566 writer.close(); 567 } 568 569 private void analyzeBadResults(ArrayList<ResultData> badResults, int totalNrOfTests) { 570 int testNoData = 0; 571 int testTimeoutCounts = 0; 572 int testResultsNotFoundCounts = 0; 573 int testWithoutLatencyResultCount = 0; 574 int testWithoutConfidenceResultCount = 0; 575 576 for (final ResultData data : badResults) { 577 if (data.hasTimedOut()) { 578 testTimeoutCounts++; 579 testNoData++; 580 continue; 581 } 582 583 if (data.hasNoTestResults()) { 584 testResultsNotFoundCounts++; 585 testNoData++; 586 continue; 587 } 588 589 if (!data.hasLatency()) { 590 testWithoutLatencyResultCount++; 591 testNoData++; 592 continue; 593 } 594 595 if (!data.hasConfidence()) { 596 testWithoutConfidenceResultCount++; 597 testNoData++; 598 continue; 599 } 600 } 601 602 CLog.i("========== BAD RESULTS ============================================"); 603 CLog.i(String.format("No Data: %1$d of %2$d", testNoData, totalNrOfTests)); 604 CLog.i(String.format("Timed out: %1$d of %2$d", testTimeoutCounts, totalNrOfTests)); 605 CLog.i( 606 String.format( 607 "No results: %1$d of %2$d", testResultsNotFoundCounts, totalNrOfTests)); 608 CLog.i( 609 String.format( 610 "No Latency results: %1$d of %2$d", 611 testWithoutLatencyResultCount, totalNrOfTests)); 612 CLog.i( 613 String.format( 614 "No Confidence results: %1$d of %2$d", 615 testWithoutConfidenceResultCount, totalNrOfTests)); 616 } 617 618 /** Generates metrics dictionary for stress test */ 619 public void populateStressTestMetrics( 620 Map<String, String> metrics, final String resultKeyPrefix) { 621 metrics.put(resultKeyPrefix + "total_nr_of_tests", Integer.toString(mAllResults.size())); 622 metrics.put(resultKeyPrefix + "nr_of_good_tests", Integer.toString(mGoodResults.size())); 623 metrics.put(resultKeyPrefix + "latency_max", Double.toString(mLatencyStats.mMax)); 624 metrics.put(resultKeyPrefix + "latency_min", Double.toString(mLatencyStats.mMin)); 625 metrics.put(resultKeyPrefix + "latency_mean", Double.toString(mLatencyStats.mMean)); 626 metrics.put(resultKeyPrefix + "latency_median", Double.toString(mLatencyStats.mMedian)); 627 metrics.put(resultKeyPrefix + "confidence_max", Double.toString(mConfidenceStats.mMax)); 628 metrics.put(resultKeyPrefix + "confidence_min", Double.toString(mConfidenceStats.mMin)); 629 metrics.put(resultKeyPrefix + "confidence_mean", Double.toString(mConfidenceStats.mMean)); 630 metrics.put( 631 resultKeyPrefix + "confidence_median", Double.toString(mConfidenceStats.mMedian)); 632 } 633 } 634