1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 * in compliance with the License. You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software distributed under the License 10 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 * or implied. See the License for the specific language governing permissions and limitations under 12 * the License. 13 */ 14 15 package com.android.performance.tests; 16 17 import com.android.tradefed.device.DeviceNotAvailableException; 18 import com.android.tradefed.device.ITestDevice; 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 21 import com.android.tradefed.result.ITestInvocationListener; 22 import com.android.tradefed.result.InputStreamSource; 23 import com.android.tradefed.result.TestDescription; 24 import com.android.tradefed.testtype.IDeviceTest; 25 import com.android.tradefed.testtype.IRemoteTest; 26 import com.android.tradefed.util.RunUtil; 27 import com.android.tradefed.util.proto.TfMetricProtoUtil; 28 29 import java.io.BufferedReader; 30 import java.io.IOException; 31 import java.io.InputStreamReader; 32 import java.util.HashMap; 33 import java.util.Map; 34 35 /** 36 * A harness that launches VellamoBenchmark and reports result. Requires 37 * VellamoBenchmark apk. 38 */ 39 public class VellamoBenchmark implements IDeviceTest, IRemoteTest { 40 41 private static final String LOGTAG = "VAUTOMATIC"; 42 private static final String RUN_KEY = "vellamobenchmark-3202"; 43 private static final String PACKAGE_NAME = "com.quicinc.vellamo"; 44 private static final long TIMEOUT_MS = 30 * 60 * 1000; 45 private static final long POLLING_INTERVAL_MS = 10 * 1000; 46 private static final int INDEX_NAME = 0; 47 private static final int INDEX_CODE = 4; 48 private static final int INDEX_SCORE = 5; 49 50 private ITestDevice mDevice; 51 52 /** 53 * {@inheritDoc} 54 */ 55 @Override 56 public void setDevice(ITestDevice device) { 57 mDevice = device; 58 } 59 60 /** 61 * {@inheritDoc} 62 */ 63 @Override 64 public ITestDevice getDevice() { 65 return mDevice; 66 } 67 68 /** 69 * {@inheritDoc} 70 */ 71 @Override 72 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 73 TestDescription testId = new TestDescription(getClass().getCanonicalName(), RUN_KEY); 74 ITestDevice device = getDevice(); 75 listener.testRunStarted(RUN_KEY, 0); 76 listener.testStarted(testId); 77 78 long testStartTime = System.currentTimeMillis(); 79 Map<String, String> metrics = new HashMap<String, String>(); 80 String errMsg = null; 81 82 boolean isTimedOut = false; 83 boolean isResultGenerated = false; 84 boolean hasScore = false; 85 double sumScore = 0; 86 int errorCode = 0; 87 device.clearErrorDialogs(); 88 isTimedOut = false; 89 90 long benchmarkStartTime = System.currentTimeMillis(); 91 // start the vellamo benchmark app and run all the tests 92 // the documentation and binary for the Vellamo 3.2.2 for Automation APK 93 // can be found here: 94 // https://b.corp.google.com/issue?id=23107318 95 CLog.i("Starting Vellamo Benchmark"); 96 device.executeShellCommand("am start -a com.quicinc.vellamo.AUTOMATIC" 97 + " -e w com.quicinc.skunkworks.wvb" // use System WebView 98 + " -n com.quicinc.vellamo/.main.MainActivity"); 99 String line; 100 while (!isResultGenerated && !isTimedOut) { 101 RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); 102 isTimedOut = (System.currentTimeMillis() - benchmarkStartTime >= TIMEOUT_MS); 103 104 // get the logcat and parse 105 try (InputStreamSource logcatSource = device.getLogcat(); 106 BufferedReader logcat = 107 new BufferedReader( 108 new InputStreamReader(logcatSource.createInputStream()))) { 109 while ((line = logcat.readLine()) != null) { 110 // filter only output from the Vellamo process 111 if (!line.contains(LOGTAG)) { 112 continue; 113 } 114 line = line.substring(line.indexOf(LOGTAG) + LOGTAG.length()); 115 // we need to see if the score is generated since there are some 116 // cases the result with </automatic> tag is generated but no score is included 117 if (line.contains("</automatic>")) { 118 if (hasScore) { 119 isResultGenerated = true; 120 break; 121 } 122 } 123 // get the score out 124 if (line.contains(" b: ")) { 125 hasScore = true; 126 String[] results = line.split(" b: ")[1].split(","); 127 errorCode = Integer.parseInt(results[INDEX_CODE]); 128 if (errorCode != 0) { 129 CLog.w("Non-zero error code: %d from becnhmark '%s'", 130 errorCode, results[INDEX_NAME]); 131 } else { 132 sumScore += Double.parseDouble(results[INDEX_SCORE]); 133 } 134 metrics.put(results[INDEX_NAME], results[INDEX_SCORE]); 135 CLog.i("%s :: %s", results[INDEX_NAME], results[INDEX_SCORE]); 136 } 137 } 138 } catch (IOException e) { 139 CLog.e(e); 140 } 141 142 if (null == device.getProcessByName(PACKAGE_NAME)) { 143 break; 144 } 145 } 146 147 if (isTimedOut) { 148 errMsg = "Vellamo Benchmark timed out."; 149 } else { 150 CLog.i("Done running Vellamo Benchmark"); 151 } 152 if (!hasScore) { 153 errMsg = "Test ended but no scores can be found."; 154 } 155 if (errMsg != null) { 156 CLog.e(errMsg); 157 listener.testFailed(testId, errMsg); 158 } 159 long durationMs = System.currentTimeMillis() - testStartTime; 160 metrics.put("total", Double.toString(sumScore)); 161 CLog.i("total :: %f", sumScore); 162 listener.testEnded(testId, new HashMap<String, Metric>()); 163 listener.testRunEnded(durationMs, TfMetricProtoUtil.upgradeConvert(metrics)); 164 } 165 } 166
