1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.device.collectors; 17 18 import android.device.collectors.annotations.MetricOption; 19 import android.device.collectors.annotations.OptionClass; 20 import android.device.collectors.util.SendToInstrumentation; 21 import android.os.Bundle; 22 import android.os.Environment; 23 import android.os.ParcelFileDescriptor; 24 import android.support.annotation.VisibleForTesting; 25 import android.support.test.InstrumentationRegistry; 26 import android.support.test.internal.runner.listener.InstrumentationRunListener; 27 import android.util.Log; 28 29 import org.junit.runner.Description; 30 import org.junit.runner.Result; 31 import org.junit.runner.notification.Failure; 32 33 import java.io.ByteArrayOutputStream; 34 import java.io.File; 35 import java.io.IOException; 36 import java.io.InputStream; 37 import java.io.PrintStream; 38 import java.util.ArrayList; 39 import java.util.Arrays; 40 import java.util.HashSet; 41 import java.util.List; 42 import java.util.Set; 43 44 /** 45 * Base implementation of a device metric listener that will capture and output metrics for each 46 * test run or test cases. Collectors will have access to {@link DataRecord} objects where they 47 * can put results and the base class ensure these results will be send to the instrumentation. 48 * 49 * Any subclass that calls {@link #createAndEmptyDirectory(String)} needs external storage 50 * permission. So to use this class at runtime, your test need to 51 * <a href="{@docRoot}training/basics/data-storage/files.html#GetWritePermission">have storage 52 * permission enabled</a>, and preferably granted at install time (to avoid interrupting the test). 53 * For testing at desk, run adb install -r -g testpackage.apk 54 * "-g" grants all required permission at install time. 55 * 56 * Filtering: 57 * You can annotate any test method (@Test) with {@link MetricOption} and specify an arbitrary 58 * group name that the test will be part of. It is possible to trigger the collection only against 59 * test part of a group using '--include-filter-group [group name]' or to exclude a particular 60 * group using '--exclude-filter-group [group name]'. 61 * Several group name can be passed using a comma separated argument. 62 * 63 */ 64 public class BaseMetricListener extends InstrumentationRunListener { 65 66 public static final int BUFFER_SIZE = 1024; 67 68 /** Options keys that the collector can receive. */ 69 // Filter groups, comma separated list of group name to be included or excluded 70 public static final String INCLUDE_FILTER_GROUP_KEY = "include-filter-group"; 71 public static final String EXCLUDE_FILTER_GROUP_KEY = "exclude-filter-group"; 72 73 private static final String NAMESPACE_SEPARATOR = ":"; 74 75 private DataRecord mRunData; 76 private DataRecord mTestData; 77 78 private Bundle mArgsBundle = null; 79 private final List<String> mIncludeFilters; 80 private final List<String> mExcludeFilters; 81 82 public BaseMetricListener() { 83 mIncludeFilters = new ArrayList<>(); 84 mExcludeFilters = new ArrayList<>(); 85 } 86 87 /** 88 * Constructor to simulate receiving the instrumentation arguments. Should not be used except 89 * for testing. 90 */ 91 @VisibleForTesting 92 BaseMetricListener(Bundle argsBundle) { 93 this(); 94 mArgsBundle = argsBundle; 95 } 96 97 @Override 98 public final void testRunStarted(Description description) throws Exception { 99 parseArguments(); 100 try { 101 mRunData = createDataRecord(); 102 onTestRunStart(mRunData, description); 103 } catch (RuntimeException e) { 104 // Prevent exception from reporting events. 105 Log.e(getTag(), "Exception during onTestRunStart.", e); 106 } 107 super.testRunStarted(description); 108 } 109 110 @Override 111 public final void testRunFinished(Result result) throws Exception { 112 try { 113 onTestRunEnd(mRunData, result); 114 } catch (RuntimeException e) { 115 // Prevent exception from reporting events. 116 Log.e(getTag(), "Exception during onTestRunEnd.", e); 117 } 118 super.testRunFinished(result); 119 } 120 121 @Override 122 public final void testStarted(Description description) throws Exception { 123 if (shouldRun(description)) { 124 try { 125 mTestData = createDataRecord(); 126 onTestStart(mTestData, description); 127 } catch (RuntimeException e) { 128 // Prevent exception from reporting events. 129 Log.e(getTag(), "Exception during onTestStart.", e); 130 } 131 } 132 super.testStarted(description); 133 } 134 135 @Override 136 public final void testFailure(Failure failure) throws Exception { 137 Description description = failure.getDescription(); 138 if (shouldRun(description)) { 139 try { 140 onTestFail(mTestData, description, failure); 141 } catch (RuntimeException e) { 142 // Prevent exception from reporting events. 143 Log.e(getTag(), "Exception during onTestFail.", e); 144 } 145 } 146 super.testFailure(failure); 147 } 148 149 @Override 150 public final void testFinished(Description description) throws Exception { 151 if (shouldRun(description)) { 152 try { 153 onTestEnd(mTestData, description); 154 } catch (RuntimeException e) { 155 // Prevent exception from reporting events. 156 Log.e(getTag(), "Exception during onTestEnd.", e); 157 } 158 if (mTestData.hasMetrics()) { 159 // Only send the status progress if there are metrics 160 SendToInstrumentation.sendBundle(getInstrumentation(), 161 mTestData.createBundleFromMetrics()); 162 } 163 } 164 super.testFinished(description); 165 } 166 167 @Override 168 public void instrumentationRunFinished( 169 PrintStream streamResult, Bundle resultBundle, Result junitResults) { 170 // Test Run data goes into the INSTRUMENTATION_RESULT 171 if (mRunData != null) { 172 resultBundle.putAll(mRunData.createBundleFromMetrics()); 173 } 174 } 175 176 /** 177 * Create a {@link DataRecord}. Exposed for testing. 178 */ 179 @VisibleForTesting 180 DataRecord createDataRecord() { 181 return new DataRecord(); 182 } 183 184 // ---------- Interfaces that can be implemented to take action on each test state. 185 186 /** 187 * Called when {@link #testRunStarted(Description)} is called. 188 * 189 * @param runData structure where metrics can be put. 190 * @param description the {@link Description} for the run about to start. 191 */ 192 public void onTestRunStart(DataRecord runData, Description description) { 193 // Does nothing 194 } 195 196 /** 197 * Called when {@link #testRunFinished(Result result)} is called. 198 * 199 * @param runData structure where metrics can be put. 200 * @param result the {@link Result} for the run coming from the runner. 201 */ 202 public void onTestRunEnd(DataRecord runData, Result result) { 203 // Does nothing 204 } 205 206 /** 207 * Called when {@link #testStarted(Description)} is called. 208 * 209 * @param testData structure where metrics can be put. 210 * @param description the {@link Description} for the test case about to start. 211 */ 212 public void onTestStart(DataRecord testData, Description description) { 213 // Does nothing 214 } 215 216 /** 217 * Called when {@link #testFailure(Failure)} is called. 218 * 219 * @param testData structure where metrics can be put. 220 * @param description the {@link Description} for the test case that just failed. 221 * @param failure the {@link Failure} describing the failure. 222 */ 223 public void onTestFail(DataRecord testData, Description description, Failure failure) { 224 // Does nothing 225 } 226 227 /** 228 * Called when {@link #testFinished(Description)} is called. 229 * 230 * @param testData structure where metrics can be put. 231 * @param description the {@link Description} of the test coming from the runner. 232 */ 233 public void onTestEnd(DataRecord testData, Description description) { 234 // Does nothing 235 } 236 237 /** 238 * Turn executeShellCommand into a blocking operation. 239 * 240 * @param command shell command to be executed. 241 * @return byte array of execution result 242 */ 243 public byte[] executeCommandBlocking(String command) { 244 try ( 245 InputStream is = new ParcelFileDescriptor.AutoCloseInputStream( 246 getInstrumentation().getUiAutomation().executeShellCommand(command)); 247 ByteArrayOutputStream out = new ByteArrayOutputStream() 248 ) { 249 byte[] buf = new byte[BUFFER_SIZE]; 250 int length; 251 while ((length = is.read(buf)) >= 0) { 252 out.write(buf, 0, length); 253 } 254 return out.toByteArray(); 255 } catch (IOException e) { 256 Log.e(getTag(), "Error executing: " + command, e); 257 return null; 258 } 259 } 260 261 /** 262 * Create a directory inside external storage, and empty it. 263 * 264 * @param dir full path to the dir to be created. 265 * @return directory file created 266 */ 267 public File createAndEmptyDirectory(String dir) { 268 File rootDir = Environment.getExternalStorageDirectory(); 269 File destDir = new File(rootDir, dir); 270 executeCommandBlocking("rm -rf " + destDir.getAbsolutePath()); 271 if (!destDir.exists() && !destDir.mkdirs()) { 272 Log.e(getTag(), "Unable to create dir: " + destDir.getAbsolutePath()); 273 return null; 274 } 275 return destDir; 276 } 277 278 /** 279 * Returns the name of the current class to be used as a logging tag. 280 */ 281 String getTag() { 282 return this.getClass().getName(); 283 } 284 285 /** 286 * Returns the bundle containing the instrumentation arguments. 287 */ 288 protected final Bundle getArgsBundle() { 289 if (mArgsBundle == null) { 290 mArgsBundle = InstrumentationRegistry.getArguments(); 291 } 292 return mArgsBundle; 293 } 294 295 private void parseArguments() { 296 Bundle args = getArgsBundle(); 297 // First filter the arguments with the alias 298 filterAlias(args); 299 // Handle filtering 300 String includeGroup = args.getString(INCLUDE_FILTER_GROUP_KEY); 301 String excludeGroup = args.getString(EXCLUDE_FILTER_GROUP_KEY); 302 if (includeGroup != null) { 303 mIncludeFilters.addAll(Arrays.asList(includeGroup.split(","))); 304 } 305 if (excludeGroup != null) { 306 mExcludeFilters.addAll(Arrays.asList(excludeGroup.split(","))); 307 } 308 } 309 310 /** 311 * Filter the alias-ed options from the bundle, each implementation of BaseMetricListener will 312 * have its own list of arguments. 313 * TODO: Split the filtering logic outside the collector class in a utility/helper. 314 */ 315 private void filterAlias(Bundle bundle) { 316 Set<String> keySet = new HashSet<>(bundle.keySet()); 317 OptionClass optionClass = this.getClass().getAnnotation(OptionClass.class); 318 if (optionClass == null) { 319 // No @OptionClass was specified, remove all alias-ed options. 320 for (String key : keySet) { 321 if (key.indexOf(NAMESPACE_SEPARATOR) != -1) { 322 bundle.remove(key); 323 } 324 } 325 return; 326 } 327 // Alias is a required field so if OptionClass is set, alias is set. 328 String alias = optionClass.alias(); 329 for (String key : keySet) { 330 if (key.indexOf(NAMESPACE_SEPARATOR) == -1) { 331 continue; 332 } 333 String optionAlias = key.split(NAMESPACE_SEPARATOR)[0]; 334 if (alias.equals(optionAlias)) { 335 // Place the option again, without alias. 336 String optionName = key.split(NAMESPACE_SEPARATOR)[1]; 337 bundle.putString(optionName, bundle.getString(key)); 338 bundle.remove(key); 339 } else { 340 // Remove other aliases. 341 bundle.remove(key); 342 } 343 } 344 } 345 346 /** 347 * Helper to decide whether the collector should run or not against the test case. 348 * 349 * @param desc The {@link Description} of the method. 350 * @return True if the collector should run. 351 */ 352 private boolean shouldRun(Description desc) { 353 MetricOption annotation = desc.getAnnotation(MetricOption.class); 354 List<String> groups = new ArrayList<>(); 355 if (annotation != null) { 356 String group = annotation.group(); 357 groups.addAll(Arrays.asList(group.split(","))); 358 } 359 if (!mExcludeFilters.isEmpty()) { 360 for (String group : groups) { 361 // Exclude filters has priority, if any of the group is excluded, exclude the method 362 if (mExcludeFilters.contains(group)) { 363 return false; 364 } 365 } 366 } 367 // If we have include filters, we can only run what's part of them. 368 if (!mIncludeFilters.isEmpty()) { 369 for (String group : groups) { 370 if (mIncludeFilters.contains(group)) { 371 return true; 372 } 373 } 374 // We have include filter and did not match them. 375 return false; 376 } 377 return true; 378 } 379 } 380
