1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.ddmlib.FileListingService; 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.OptionClass; 21 import com.android.tradefed.device.CollectingOutputReceiver; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ITestInvocationListener; 26 27 import java.util.ArrayList; 28 import java.util.HashMap; 29 import java.util.List; 30 import java.util.Map; 31 import java.util.concurrent.TimeUnit; 32 33 /** 34 * A Test that runs a Google benchmark test package on given device. 35 */ 36 @OptionClass(alias = "gbenchmark") 37 public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest { 38 39 static final String DEFAULT_TEST_PATH = "/data/benchmarktest"; 40 41 private static final String GBENCHMARK_JSON_OUTPUT_FORMAT = "--benchmark_format=json"; 42 43 @Option(name = "file-exclusion-filter-regex", 44 description = "Regex to exclude certain files from executing. Can be repeated") 45 private List<String> mFileExclusionFilterRegex = new ArrayList<>(); 46 47 @Option(name = "native-benchmark-device-path", 48 description="The path on the device where native stress tests are located.") 49 private String mDeviceTestPath = DEFAULT_TEST_PATH; 50 51 @Option(name = "benchmark-module-name", 52 description="The name of the native benchmark test module to run. " + 53 "If not specified all tests in --native-benchmark-device-path will be run.") 54 private String mTestModule = null; 55 56 @Option(name = "benchmark-run-name", 57 description="Optional name to pass to test reporters. If unspecified, will use " + 58 "test binary as run name.") 59 private String mReportRunName = null; 60 61 @Option(name = "max-run-time", description = 62 "The maximum time to allow for each benchmark run in ms.", isTimeVal=true) 63 private long mMaxRunTime = 15 * 60 * 1000; 64 65 private ITestDevice mDevice = null; 66 67 /** 68 * {@inheritDoc} 69 */ 70 @Override 71 public void setDevice(ITestDevice device) { 72 mDevice = device; 73 } 74 75 /** 76 * {@inheritDoc} 77 */ 78 @Override 79 public ITestDevice getDevice() { 80 return mDevice; 81 } 82 83 /** 84 * Set the Android native benchmark test module to run. 85 * 86 * @param moduleName The name of the native test module to run 87 */ 88 public void setModuleName(String moduleName) { 89 mTestModule = moduleName; 90 } 91 92 /** 93 * Get the Android native benchmark test module to run. 94 * 95 * @return the name of the native test module to run, or null if not set 96 */ 97 public String getModuleName() { 98 return mTestModule; 99 } 100 101 public void setReportRunName(String reportRunName) { 102 mReportRunName = reportRunName; 103 } 104 105 /** 106 * Adds an exclusion file filter regex. 107 * <p/> 108 * Exposed for unit testing 109 * 110 * @param regex to exclude file. 111 */ 112 void addFileExclusionFilterRegex(String regex) { 113 mFileExclusionFilterRegex.add(regex); 114 } 115 116 /** 117 * Gets the path where native benchmark tests live on the device. 118 * 119 * @return The path on the device where the native tests live. 120 */ 121 private String getTestPath() { 122 StringBuilder testPath = new StringBuilder(mDeviceTestPath); 123 if (mTestModule != null) { 124 testPath.append(FileListingService.FILE_SEPARATOR); 125 testPath.append(mTestModule); 126 } 127 return testPath.toString(); 128 } 129 130 /** 131 * Executes all native benchmark tests in a folder as well as in all subfolders recursively. 132 * 133 * @param root The root folder to begin searching for native tests 134 * @param testDevice The device to run tests on 135 * @param listener the run listener 136 * @throws DeviceNotAvailableException 137 */ 138 private void doRunAllTestsInSubdirectory(String root, ITestDevice testDevice, 139 ITestInvocationListener listener) throws DeviceNotAvailableException { 140 if (testDevice.isDirectory(root)) { 141 // recursively run tests in all subdirectories 142 for (String child : testDevice.getChildren(root)) { 143 doRunAllTestsInSubdirectory(root + "/" + child, testDevice, listener); 144 } 145 } else { 146 // assume every file is a valid benchmark test binary. 147 // use name of file as run name 148 String rootEntry = root.substring(root.lastIndexOf("/") + 1); 149 String runName = (mReportRunName == null ? rootEntry : mReportRunName); 150 151 // force file to be executable 152 testDevice.executeShellCommand(String.format("chmod 755 %s", root)); 153 if (shouldSkipFile(root)) { 154 return; 155 } 156 long startTime = System.currentTimeMillis(); 157 158 Map<String, String> metricMap = new HashMap<String, String>(); 159 CollectingOutputReceiver outputCollector = createOutputCollector(); 160 GoogleBenchmarkResultParser resultParser = createResultParser(runName, listener); 161 listener.testRunStarted(runName, 0); 162 try { 163 String cmd = String.format("%s %s", root, GBENCHMARK_JSON_OUTPUT_FORMAT); 164 CLog.i(String.format("Running google benchmark test on %s: %s", 165 mDevice.getSerialNumber(), cmd)); 166 testDevice.executeShellCommand(cmd, outputCollector, 167 mMaxRunTime, TimeUnit.MILLISECONDS, 0); 168 metricMap = resultParser.parse(outputCollector); 169 } finally { 170 final long elapsedTime = System.currentTimeMillis() - startTime; 171 listener.testRunEnded(elapsedTime, metricMap); 172 } 173 } 174 } 175 176 /** 177 * Helper method to determine if we should skip the execution of a given file. 178 * @param fullPath the full path of the file in question 179 * @return true if we should skip the said file. 180 */ 181 protected boolean shouldSkipFile(String fullPath) { 182 if (fullPath == null || fullPath.isEmpty()) { 183 return true; 184 } 185 if (mFileExclusionFilterRegex == null || mFileExclusionFilterRegex.isEmpty()) { 186 return false; 187 } 188 for (String regex : mFileExclusionFilterRegex) { 189 if (fullPath.matches(regex)) { 190 CLog.i(String.format("File %s matches exclusion file regex %s, skipping", 191 fullPath, regex)); 192 return true; 193 } 194 } 195 return false; 196 } 197 198 /** 199 * Exposed for testing 200 */ 201 CollectingOutputReceiver createOutputCollector() { 202 return new CollectingOutputReceiver(); 203 } 204 205 /** 206 * Exposed for testing 207 */ 208 GoogleBenchmarkResultParser createResultParser(String runName, 209 ITestInvocationListener listener) { 210 return new GoogleBenchmarkResultParser(runName, listener); 211 } 212 213 /** 214 * {@inheritDoc} 215 */ 216 @Override 217 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 218 if (mDevice == null) { 219 throw new IllegalArgumentException("Device has not been set"); 220 } 221 String testPath = getTestPath(); 222 if (!mDevice.doesFileExist(testPath)) { 223 CLog.w(String.format("Could not find native benchmark test directory %s in %s!", 224 testPath, mDevice.getSerialNumber())); 225 throw new RuntimeException( 226 String.format("Could not find native benchmark test directory %s", testPath)); 227 } 228 doRunAllTestsInSubdirectory(testPath, mDevice, listener); 229 } 230 } 231