Home | History | Annotate | Download | only in tests 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.browser.tests;
     18 
     19 import com.android.ddmlib.IDevice;
     20 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
     21 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
     22 import com.android.tradefed.config.Option;
     23 import com.android.tradefed.config.Option.Importance;
     24 import com.android.tradefed.device.DeviceNotAvailableException;
     25 import com.android.tradefed.device.ITestDevice;
     26 import com.android.tradefed.invoker.ITestInvocation;
     27 import com.android.tradefed.log.LogUtil.CLog;
     28 import com.android.tradefed.result.CollectingTestListener;
     29 import com.android.tradefed.result.ITestInvocationListener;
     30 import com.android.tradefed.testtype.IDeviceTest;
     31 import com.android.tradefed.testtype.IRemoteTest;
     32 
     33 import java.util.HashMap;
     34 import java.util.Map;
     35 import java.util.concurrent.TimeUnit;
     36 
     37 /**
     38  * Base test harness class for browser site load test.
     39  *
     40  * The class can be configured to run Android Browser stability test with a preconfigured list
     41  * of sites pushed into ${EXTERNAL_STORAGE}/popular_urls.txt
     42  *
     43  */
     44 public class BrowserSiteLoadTest implements IRemoteTest, IDeviceTest {
     45 
     46     private static final String URLS_FILE_NAME = "popular_urls.txt";
     47     private static final String STATUS_FILE_NAME = "test_status.txt";
     48     private static final String TEST_CLASS_NAME = "com.android.browser.PopularUrlsTest";
     49     private static final String TEST_METHOD_NAME = "testStability";
     50     private static final String TEST_PACKAGE_NAME = "com.android.browser.tests";
     51     private static final String TEST_RUNNER_NAME = "android.test.InstrumentationTestRunner";
     52 
     53     private static final int MAX_TIMEOUT_MS = 8 * 60 * 60 * 1000; // max test timeout is 8 hrs
     54 
     55     private ITestDevice mTestDevice = null;
     56 
     57     @Option(name = "metrics-name", description = "name used to identify the metrics for reporting",
     58             importance = Importance.ALWAYS)
     59     private String mMetricsName;
     60 
     61     @Option(name = "schema-key",
     62             description = "the schema key that number of successful loads should be reported under",
     63             importance = Importance.ALWAYS)
     64     private String mSchemaKey;
     65 
     66     @Option(name = "test-package",
     67             description = "package name of the stability test. defaults to that of AOSP Browser")
     68     private String mTestPackage = TEST_PACKAGE_NAME;
     69 
     70     @Option(name = "test-class",
     71             description = "class name of the stability test. defaults to that of AOSP Browser")
     72     private String mTestClass = TEST_CLASS_NAME;
     73 
     74     @Option(name = "test-method",
     75             description = "method name of the stability test. defaults to that of AOSP Browser")
     76     private String mTestMethod = TEST_METHOD_NAME;
     77 
     78     @Option(name = "total-sites", description = "expected total number of site loads")
     79     private int mTotalSites = 0;
     80 
     81     private String mUrlsFilePath;
     82     private String mStatusFilePath;
     83 
     84     /**
     85      * {@inheritDoc}
     86      */
     87     @Override
     88     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
     89         int failCounter = 0;
     90         Map<String, String> finalMetrics = new HashMap<String, String>();
     91 
     92         preTestSetup();
     93 
     94         // Create and config runner for instrumentation test
     95         IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(mTestPackage,
     96                 TEST_RUNNER_NAME, mTestDevice.getIDevice());
     97         runner.setClassName(mTestClass);
     98         runner.setMethodName(mTestClass, mTestMethod);
     99         // max timeout is the smaller of: 8 hrs or 1 minute per site
    100         runner.setMaxTimeToOutputResponse(Math.min(MAX_TIMEOUT_MS, 60 * 1000 * mTotalSites),
    101                 TimeUnit.MILLISECONDS);
    102 
    103         CollectingTestListener collectingTestListener = new CollectingTestListener();
    104         failCounter = runStabilityTest(runner, collectingTestListener);
    105         finalMetrics.put(mSchemaKey, Integer.toString(mTotalSites - failCounter));
    106 
    107         reportMetrics(listener, mMetricsName, finalMetrics);
    108     }
    109 
    110     /**
    111      * {@inheritDoc}
    112      */
    113     @Override
    114     public ITestDevice getDevice() {
    115         return mTestDevice;
    116     }
    117 
    118     /**
    119      * {@inheritDoc}
    120      */
    121     @Override
    122     public void setDevice(ITestDevice device) {
    123         mTestDevice = device;
    124     }
    125 
    126     /**
    127      * Wipes the device's external memory of test collateral from prior runs.
    128      *
    129      * @throws DeviceNotAvailableException If the device is unavailable or
    130      *             something happened while deleting files
    131      */
    132     private void preTestSetup() throws DeviceNotAvailableException {
    133         String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
    134         mUrlsFilePath = String.format("%s/%s", extStore, URLS_FILE_NAME);
    135         mStatusFilePath = String.format("%s/%s", extStore, STATUS_FILE_NAME);
    136         if (!mTestDevice.doesFileExist(mUrlsFilePath)) {
    137             throw new RuntimeException("missing URL list file at: " + mUrlsFilePath);
    138         }
    139         mTestDevice.executeShellCommand("rm " + mStatusFilePath);
    140     }
    141 
    142     /**
    143      * Report run metrics by creating an empty test run to stick them in.
    144      *
    145      * @param listener The {@link ITestInvocation} of test results
    146      * @param runName The test name
    147      * @param metrics The {@link Map} that contains metrics for the given test
    148      */
    149     private void reportMetrics(ITestInvocationListener listener, String runName,
    150             Map<String, String> metrics) {
    151         CLog.i(String.format("About to report metrics: loaded=%s", metrics.get(mSchemaKey)));
    152         listener.testRunStarted(runName, 0);
    153         listener.testRunEnded(0, metrics);
    154     }
    155 
    156     /**
    157      * Helper method for iterating through popular_urls.txt and reporting back
    158      * how many pages fail to load.
    159      *
    160      * @param runner The {@link IRemoteAndroidTestRunner} for instrumentation
    161      * @param listener The {@link CollectingTestListener} of test results
    162      * @return Number of pages that failed to load
    163      */
    164     private int runStabilityTest(IRemoteAndroidTestRunner runner, CollectingTestListener listener)
    165             throws DeviceNotAvailableException {
    166         boolean exitConditionMet = false;
    167         int failureCounter = 0;
    168         while (!exitConditionMet) {
    169             mTestDevice.runInstrumentationTests(runner, listener);
    170             // if the status file exists, then the previous instrumentation has crashed
    171             if (!mTestDevice.doesFileExist(mStatusFilePath)) {
    172                 exitConditionMet = true;
    173             } else {
    174                 ++failureCounter;
    175             }
    176         }
    177         return failureCounter;
    178     }
    179 }
    180