Home | History | Annotate | Download | only in tests
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.graphics.tests;
     18 
     19 import com.android.ddmlib.IDevice;
     20 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
     21 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
     22 import com.android.tradefed.device.DeviceNotAvailableException;
     23 import com.android.tradefed.device.ITestDevice;
     24 import com.android.tradefed.log.LogUtil.CLog;
     25 import com.android.tradefed.result.BugreportCollector;
     26 import com.android.tradefed.result.FileInputStreamSource;
     27 import com.android.tradefed.result.ITestInvocationListener;
     28 import com.android.tradefed.result.InputStreamSource;
     29 import com.android.tradefed.result.LogDataType;
     30 import com.android.tradefed.testtype.IDeviceTest;
     31 import com.android.tradefed.testtype.IRemoteTest;
     32 import com.android.tradefed.util.FileUtil;
     33 import com.android.tradefed.util.RunUtil;
     34 import com.android.tradefed.util.StreamUtil;
     35 
     36 import org.junit.Assert;
     37 
     38 import java.io.BufferedReader;
     39 import java.io.File;
     40 import java.io.FileReader;
     41 import java.io.IOException;
     42 import java.util.HashMap;
     43 import java.util.Map;
     44 import java.util.regex.Matcher;
     45 import java.util.regex.Pattern;
     46 
     47 /**
     48  * Run UiPerformanceTest suite which measures the performance of
     49  * system operations and majors applications.
     50  */
     51 public class UiPerformanceTest implements IDeviceTest, IRemoteTest {
     52     private ITestDevice mTestDevice = null;
     53     // Define instrumentation test package and runner.
     54     private static final String TEST_PACKAGE_NAME =
     55             "com.android.testing.uiautomation.platform.uiperformance";
     56     // TODO: Add TEST_CLASS_NAME later when different tests requiring
     57     //       different configurations.
     58     private static final String TEST_RUNNER_NAME =
     59             "com.android.testing.uiautomation.UiAutomationTestRunner";
     60     private static final String OUTPUT_FILE_NAME = "UiPerfTestsOutput.txt"; // output file
     61     private static final String RAW_DATA_DIRECTORY = "UiPerformanceRawData"; // raw data directory
     62 
     63     private static final String TEST_CASE_PREFIX = "test";
     64     private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes
     65 
     66     private static final Pattern JANKINESS_PATTERN =
     67             Pattern.compile("^number of jankiness: (\\d+)");
     68     private static final Pattern MEDIAN_FRAME_LATENCY_PATTERN =
     69             Pattern.compile("^median of frame latency: (\\d+)");
     70     private static final Pattern FRAME_RATE_PATTERN =
     71             Pattern.compile("^average frame rate: (\\d+\\.\\d+)");
     72     private static final Pattern[] mPatterns = {
     73         JANKINESS_PATTERN, FRAME_RATE_PATTERN, MEDIAN_FRAME_LATENCY_PATTERN
     74     };
     75     private static final String[] ITEM_KEYS = {"number_jankiness", "frame_rate", "frame_latency"};
     76 
     77     @Override
     78     public void setDevice(ITestDevice testDevice) {
     79         mTestDevice = testDevice;
     80     }
     81 
     82     @Override
     83     public ITestDevice getDevice() {
     84         return mTestDevice;
     85     }
     86 
     87     private void setupDevice() throws DeviceNotAvailableException {
     88         cleanOutputFiles();
     89         String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
     90         String rawFileDir = String.format("%s/%s", extStore, RAW_DATA_DIRECTORY);
     91 
     92         if (!mTestDevice.doesFileExist(rawFileDir)) {
     93             CLog.v(String.format("The raw directory %s doesn't exist.", RAW_DATA_DIRECTORY));
     94             mTestDevice.executeShellCommand(String.format("mkdir \"%s\"", rawFileDir));
     95         } else {
     96             // remove files
     97             mTestDevice.executeShellCommand(String.format("rm %s/*", rawFileDir));
     98             CLog.v("remove files under the raw data directory");
     99         }
    100     }
    101 
    102     /**
    103      * Run UiPerformanceTests and parsing results from test output.
    104      */
    105     @Override
    106     public void run(ITestInvocationListener standardListener)
    107             throws DeviceNotAvailableException {
    108         Assert.assertNotNull(mTestDevice);
    109         setupDevice();
    110         // start the test until device is fully booted and stable
    111         RunUtil.getDefault().sleep(START_TIMER);
    112         IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(
    113                 TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice());
    114 
    115         // Add bugreport listener for failed test
    116         BugreportCollector bugListener = new
    117             BugreportCollector(standardListener, mTestDevice);
    118         bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES);
    119         bugListener.setDescriptiveName(this.getClass().getName());
    120         mTestDevice.runInstrumentationTests(runner, bugListener);
    121         logOutputFile(bugListener);
    122         pullRawDataFile(bugListener);
    123         cleanOutputFiles();
    124     }
    125 
    126     private void pullRawDataFile(ITestInvocationListener listener)
    127             throws DeviceNotAvailableException {
    128         String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
    129         String rawFileDir = String.format("%s/%s", extStore, RAW_DATA_DIRECTORY);
    130 
    131         String rawFileList =
    132                 mTestDevice.executeShellCommand(String.format("ls \"%s\"", rawFileDir));
    133         String[] rawFileString = rawFileList.split("\r?\n");
    134         File resFile = null;
    135         InputStreamSource outputSource = null;
    136         for (int i = 0; i < rawFileString.length; i++) {
    137             CLog.v("file %d is:  \"%s\"", i, rawFileString[i]);
    138             try {
    139                 resFile = mTestDevice.pullFileFromExternal(
    140                         String.format("%s/%s", RAW_DATA_DIRECTORY, rawFileString[i]));
    141                 outputSource = new FileInputStreamSource(resFile, true /* delete */);
    142                 listener.testLog(rawFileString[i], LogDataType.TEXT, outputSource);
    143             } finally {
    144                 StreamUtil.cancel(outputSource);
    145             }
    146         }
    147     }
    148 
    149 
    150     // Parse the output file
    151     private void logOutputFile(ITestInvocationListener listener)
    152             throws DeviceNotAvailableException {
    153         // catch a bugreport after the test
    154         try (InputStreamSource bugreport = mTestDevice.getBugreport()) {
    155             listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport);
    156         }
    157 
    158         File resFile = null;
    159         InputStreamSource outputSource = null;
    160         Map<String, String> runMetrics = new HashMap<>();
    161         BufferedReader br = null;
    162         try {
    163             resFile = mTestDevice.pullFileFromExternal(OUTPUT_FILE_NAME);
    164             if (resFile == null) {
    165                 CLog.v("File %s doesn't exist or pulling the file failed");
    166                 return;
    167             }
    168             CLog.d("output file: %s", resFile.getPath());
    169             // Save a copy of the output file
    170             CLog.d("Sending %d byte file %s into the logosphere!",
    171                     resFile.length(), resFile);
    172             outputSource = new FileInputStreamSource(resFile);
    173             listener.testLog(OUTPUT_FILE_NAME, LogDataType.TEXT, outputSource);
    174 
    175             // Parse the results file
    176             br = new BufferedReader(new FileReader(resFile));
    177             String line = null;
    178             String unitKey = null;
    179             int size = mPatterns.length;
    180             while ((line = br.readLine()) != null) {
    181                 if (line.startsWith(TEST_CASE_PREFIX)) {
    182                     // report the previous test case results
    183                     if (unitKey != null) {
    184                         reportMetrics(unitKey, listener, runMetrics);
    185                     }
    186                     runMetrics.clear();
    187                     // processing the next test case
    188                     unitKey = line.trim();
    189                     continue;
    190                 } else {
    191                     for (int i = 0; i < size; i++) {
    192                         Matcher match = mPatterns[i].matcher(line);
    193                         if (match.matches()) {
    194                             String value = match.group(1);
    195                             runMetrics.put(ITEM_KEYS[i], value);
    196                             break;
    197                         }
    198                     }
    199                 }
    200             }
    201             reportMetrics(unitKey, listener, runMetrics);
    202         } catch (IOException e) {
    203             CLog.e("IOException while reading outputfile %s", OUTPUT_FILE_NAME);
    204         } finally {
    205             FileUtil.deleteFile(resFile);
    206             StreamUtil.cancel(outputSource);
    207             StreamUtil.close(br);
    208         }
    209     }
    210 
    211     // Report run metrics by creating an empty test run to stick them in
    212     private void reportMetrics(String metricsName, ITestInvocationListener listener,
    213             Map<String, String> metrics) {
    214         // Create an empty testRun to report the parsed runMetrics
    215         CLog.d("About to report metrics to %s: %s", metricsName, metrics);
    216         listener.testRunStarted(metricsName, 0);
    217         listener.testRunEnded(0, metrics);
    218     }
    219 
    220     // clean up output file
    221     private void cleanOutputFiles() throws DeviceNotAvailableException {
    222         String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
    223         mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, OUTPUT_FILE_NAME));
    224     }
    225 }
    226