Home | History | Annotate | Download | only in dumprendertree 
      1 /*
      2  * Copyright (C) 2008 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.dumprendertree;
     18 
     19 import com.android.dumprendertree.TestShellActivity.DumpDataType;
     20 import com.android.dumprendertree.forwarder.AdbUtils;
     21 import com.android.dumprendertree.forwarder.ForwardService;
     22 
     23 import android.content.Context;
     24 import android.content.Intent;
     25 import android.os.Environment;
     26 import android.test.ActivityInstrumentationTestCase2;
     27 import android.util.Log;
     28 
     29 import java.io.BufferedOutputStream;
     30 import java.io.BufferedReader;
     31 import java.io.File;
     32 import java.io.FileNotFoundException;
     33 import java.io.FileOutputStream;
     34 import java.io.FileReader;
     35 import java.io.IOException;
     36 import java.io.InputStream;
     37 import java.io.OutputStream;
     38 import java.util.Vector;
     39 
     40 // TestRecorder creates four files ...
     41 // - passing tests
     42 // - failing tests
     43 // - tests for which results are ignored
     44 // - tests with no text results available
     45 // TestRecorder does not have the ability to clear the results.
     46 class MyTestRecorder {
     47     private BufferedOutputStream mBufferedOutputPassedStream;
     48     private BufferedOutputStream mBufferedOutputFailedStream;
     49     private BufferedOutputStream mBufferedOutputIgnoreResultStream;
     50     private BufferedOutputStream mBufferedOutputNoResultStream;
     51 
     52     public void passed(String layout_file) {
     53         try {
     54             mBufferedOutputPassedStream.write(layout_file.getBytes());
     55             mBufferedOutputPassedStream.write('\n');
     56             mBufferedOutputPassedStream.flush();
     57         } catch(Exception e) {
     58             e.printStackTrace();
     59         }
     60     }
     61 
     62     public void failed(String layout_file) {
     63         try {
     64             mBufferedOutputFailedStream.write(layout_file.getBytes());
     65             mBufferedOutputFailedStream.write('\n');
     66             mBufferedOutputFailedStream.flush();
     67         } catch(Exception e) {
     68             e.printStackTrace();
     69         }
     70     }
     71 
     72     public void ignoreResult(String layout_file) {
     73         try {
     74             mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
     75             mBufferedOutputIgnoreResultStream.write('\n');
     76             mBufferedOutputIgnoreResultStream.flush();
     77         } catch(Exception e) {
     78             e.printStackTrace();
     79         }
     80     }
     81 
     82     public void noResult(String layout_file) {
     83         try {
     84             mBufferedOutputNoResultStream.write(layout_file.getBytes());
     85             mBufferedOutputNoResultStream.write('\n');
     86             mBufferedOutputNoResultStream.flush();
     87         } catch(Exception e) {
     88             e.printStackTrace();
     89         }
     90     }
     91 
     92     public MyTestRecorder(boolean resume) {
     93         try {
     94             File externalDir = Environment.getExternalStorageDirectory();
     95             File resultsPassedFile = new File(externalDir, "layout_tests_passed.txt");
     96             File resultsFailedFile = new File(externalDir, "layout_tests_failed.txt");
     97             File resultsIgnoreResultFile = new File(externalDir, "layout_tests_ignored.txt");
     98             File noExpectedResultFile = new File(externalDir, "layout_tests_nontext.txt");
     99 
    100             mBufferedOutputPassedStream =
    101                 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
    102             mBufferedOutputFailedStream =
    103                 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
    104             mBufferedOutputIgnoreResultStream =
    105                 new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
    106             mBufferedOutputNoResultStream =
    107                 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
    108         } catch (Exception e) {
    109             e.printStackTrace();
    110         }
    111     }
    112 
    113     public void close() {
    114         try {
    115             mBufferedOutputPassedStream.close();
    116             mBufferedOutputFailedStream.close();
    117             mBufferedOutputIgnoreResultStream.close();
    118             mBufferedOutputNoResultStream.close();
    119         } catch (Exception e) {
    120             e.printStackTrace();
    121         }
    122     }
    123 }
    124 
    125 
    126 public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
    127 
    128     private static final String LOGTAG = "LayoutTests";
    129     static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000;
    130 
    131     static final String EXTERNAL_DIR = Environment.getExternalStorageDirectory().toString();
    132     static final String LAYOUT_TESTS_ROOT = EXTERNAL_DIR + "/webkit/layout_tests/";
    133     static final String LAYOUT_TESTS_RESULT_DIR = EXTERNAL_DIR + "/webkit/layout_tests_results/";
    134     static final String ANDROID_EXPECTED_RESULT_DIR = EXTERNAL_DIR + "/webkit/expected_results/";
    135     static final String LAYOUT_TESTS_LIST_FILE = EXTERNAL_DIR + "/webkit/layout_tests_list.txt";
    136     static final String TEST_STATUS_FILE = EXTERNAL_DIR + "/webkit/running_test.txt";
    137     static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = {
    138           "results/layout_tests_passed.txt",
    139           "results/layout_tests_failed.txt",
    140           "results/layout_tests_nontext.txt",
    141           "results/layout_tests_crashed.txt",
    142           "run_layout_tests.py"
    143     };
    144 
    145     static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt";
    146     static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt";
    147     static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt";
    148     static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py";
    149 
    150     private MyTestRecorder mResultRecorder;
    151     private Vector<String> mTestList;
    152     // Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
    153     private Vector<Boolean> mTestListIgnoreResult;
    154     private boolean mRebaselineResults;
    155     // The JavaScript engine currently in use. This determines which set of Android-specific
    156     // expected test results we use.
    157     private String mJsEngine;
    158     private String mTestPathPrefix;
    159     private boolean mFinished;
    160     private int mTestCount;
    161     private int mResumeIndex;
    162 
    163     public LayoutTestsAutoTest() {
    164       super(TestShellActivity.class);
    165     }
    166 
    167     private void getTestList() {
    168         // Read test list.
    169         try {
    170             BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
    171             String line = inReader.readLine();
    172             while (line != null) {
    173                 if (line.startsWith(mTestPathPrefix)) {
    174                     String[] components = line.split(" ");
    175                     mTestList.add(components[0]);
    176                     mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
    177                 }
    178                 line = inReader.readLine();
    179             }
    180             inReader.close();
    181             Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s).");
    182         } catch (Exception e) {
    183             Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
    184         }
    185         mTestCount = mTestList.size();
    186     }
    187 
    188     private void resumeTestList() {
    189         // read out the test name it stoped last time.
    190         try {
    191             String line = FsUtils.readTestStatus(TEST_STATUS_FILE);
    192             for (int i = 0; i < mTestList.size(); i++) {
    193                 if (mTestList.elementAt(i).equals(line)) {
    194                     mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
    195                     mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
    196                     mResumeIndex = i + 1;
    197                     break;
    198                 }
    199             }
    200         } catch (Exception e) {
    201             Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
    202         }
    203     }
    204 
    205     private void clearTestStatus() {
    206         // Delete TEST_STATUS_FILE
    207         try {
    208             File f = new File(TEST_STATUS_FILE);
    209             if (f.delete())
    210                 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
    211             else
    212                 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
    213         } catch (Exception e) {
    214             Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
    215         }
    216     }
    217 
    218     private String getResultFile(String test) {
    219         String shortName = test.substring(0, test.lastIndexOf('.'));
    220         // Write actual results to result directory.
    221         return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt";
    222     }
    223 
    224     // Gets the file which contains WebKit's expected results for this test.
    225     private String getExpectedResultFile(String test) {
    226         // The generic result is at <path>/<name>-expected.txt
    227         // First try the Android-specific result at
    228         // platform/android-<js-engine>/<path>/<name>-expected.txt
    229         // then
    230         // platform/android/<path>/<name>-expected.txt
    231         int pos = test.lastIndexOf('.');
    232         if (pos == -1)
    233             return null;
    234         String genericExpectedResult = test.substring(0, pos) + "-expected.txt";
    235         String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/";
    236         String androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT,
    237                 LAYOUT_TESTS_ROOT + androidExpectedResultsDir);
    238         File f = new File(androidExpectedResult);
    239         if (f.exists())
    240             return androidExpectedResult;
    241         androidExpectedResultsDir = "platform/android/";
    242         androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT,
    243                 LAYOUT_TESTS_ROOT + androidExpectedResultsDir);
    244         f = new File(androidExpectedResult);
    245         return f.exists() ? androidExpectedResult : genericExpectedResult;
    246     }
    247 
    248     // Gets the file which contains the actual results of running the test on
    249     // Android, generated by a previous run which set a new baseline.
    250     private String getAndroidExpectedResultFile(String expectedResultFile) {
    251         return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR);
    252     }
    253 
    254     // Wrap up
    255     private void failedCase(String file) {
    256         Log.w("Layout test: ", file + " failed");
    257         mResultRecorder.failed(file);
    258     }
    259 
    260     private void passedCase(String file) {
    261         Log.v("Layout test:", file + " passed");
    262         mResultRecorder.passed(file);
    263     }
    264 
    265     private void ignoreResultCase(String file) {
    266         Log.v("Layout test:", file + " ignore result");
    267         mResultRecorder.ignoreResult(file);
    268     }
    269 
    270     private void noResultCase(String file) {
    271         Log.v("Layout test:", file + " no expected result");
    272         mResultRecorder.noResult(file);
    273     }
    274 
    275     private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
    276         Log.v(LOGTAG, "  Processing result: " + testFile);
    277 
    278         if (ignoreResult) {
    279             ignoreResultCase(testFile);
    280             return;
    281         }
    282 
    283         File actual = new File(actualResultFile);
    284         File expected = new File(expectedResultFile);
    285         if (actual.exists() && expected.exists()) {
    286             try {
    287                 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) {
    288                     passedCase(testFile);
    289                 } else {
    290                     failedCase(testFile);
    291                 }
    292             } catch (FileNotFoundException ex) {
    293                 Log.e(LOGTAG, "File not found : " + ex.getMessage());
    294             } catch (IOException ex) {
    295                 Log.e(LOGTAG, "IO Error : " + ex.getMessage());
    296             }
    297             return;
    298         }
    299 
    300         if (!expected.exists()) {
    301             noResultCase(testFile);
    302         }
    303     }
    304 
    305     private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult, int testNumber) {
    306         activity.setCallback(new TestShellCallback() {
    307             public void finished() {
    308                 synchronized (LayoutTestsAutoTest.this) {
    309                     mFinished = true;
    310                     LayoutTestsAutoTest.this.notifyAll();
    311                 }
    312             }
    313 
    314             public void timedOut(String url) {
    315                 Log.v(LOGTAG, "layout timeout: " + url);
    316             }
    317 
    318             @Override
    319             public void dumpResult(String webViewDump) {
    320             }
    321         });
    322 
    323         String resultFile = getResultFile(test);
    324         if (resultFile == null) {
    325             // Simply ignore this test.
    326             return;
    327         }
    328         if (mRebaselineResults) {
    329             String expectedResultFile = getExpectedResultFile(test);
    330             File f = new File(expectedResultFile);
    331             if (f.exists()) {
    332                 return;  // don't run test and don't overwrite default tests.
    333             }
    334 
    335             resultFile = getAndroidExpectedResultFile(expectedResultFile);
    336         }
    337 
    338         mFinished = false;
    339         Intent intent = new Intent(Intent.ACTION_VIEW);
    340         intent.setClass(activity, TestShellActivity.class);
    341         intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
    342         intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test));
    343         intent.putExtra(TestShellActivity.RESULT_FILE, resultFile);
    344         intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
    345         intent.putExtra(TestShellActivity.TOTAL_TEST_COUNT, mTestCount);
    346         intent.putExtra(TestShellActivity.CURRENT_TEST_NUMBER, testNumber);
    347         intent.putExtra(TestShellActivity.STOP_ON_REF_ERROR, true);
    348         activity.startActivity(intent);
    349 
    350         // Wait until done.
    351         synchronized (this) {
    352             while(!mFinished){
    353                 try {
    354                     this.wait();
    355                 } catch (InterruptedException e) { }
    356             }
    357         }
    358 
    359         if (!mRebaselineResults) {
    360             String expectedResultFile = getExpectedResultFile(test);
    361             File f = new File(expectedResultFile);
    362             if (!f.exists()) {
    363                 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
    364             }
    365 
    366             processResult(test, resultFile, expectedResultFile, ignoreResult);
    367         }
    368     }
    369 
    370     // Invokes running of layout tests
    371     // and waits till it has finished running.
    372     public void executeLayoutTests(boolean resume) {
    373         LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
    374         // A convenient method to be called by another activity.
    375 
    376         if (runner.mTestPath == null) {
    377             Log.e(LOGTAG, "No test specified");
    378             return;
    379         }
    380 
    381         this.mTestList = new Vector<String>();
    382         this.mTestListIgnoreResult = new Vector<Boolean>();
    383 
    384         // Read settings
    385         mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
    386         mRebaselineResults = runner.mRebaseline;
    387         // V8 is the default JavaScript engine.
    388         mJsEngine = runner.mJsEngine == null ? "v8" : runner.mJsEngine;
    389 
    390         int timeout = runner.mTimeoutInMillis;
    391         if (timeout <= 0) {
    392             timeout = DEFAULT_TIMEOUT_IN_MILLIS;
    393         }
    394 
    395         this.mResultRecorder = new MyTestRecorder(resume);
    396 
    397         if (!resume)
    398             clearTestStatus();
    399 
    400         getTestList();
    401         if (resume)
    402             resumeTestList();
    403 
    404         TestShellActivity activity = getActivity();
    405         activity.setDefaultDumpDataType(DumpDataType.EXT_REPR);
    406 
    407         // Run tests.
    408         for (int i = 0; i < mTestList.size(); i++) {
    409             String s = mTestList.elementAt(i);
    410             boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
    411             FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
    412             // Run tests
    413             // i is 0 based, but test count is 1 based so add 1 to i here.
    414             runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult,
    415                     i + 1 + mResumeIndex);
    416         }
    417 
    418         FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
    419         ForwardService.getForwardService().stopForwardService();
    420         activity.finish();
    421     }
    422 
    423     private String getTestPath() {
    424         LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
    425 
    426         String test_path = LAYOUT_TESTS_ROOT;
    427         if (runner.mTestPath != null) {
    428             test_path += runner.mTestPath;
    429         }
    430         test_path = new File(test_path).getAbsolutePath();
    431         Log.v("LayoutTestsAutoTest", " Test path : " + test_path);
    432 
    433         return test_path;
    434     }
    435 
    436     public void generateTestList() {
    437         try {
    438             File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
    439             BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
    440             FsUtils.writeLayoutTestListRecursively(bos, getTestPath(), false); // Don't ignore results
    441             bos.flush();
    442             bos.close();
    443        } catch (Exception e) {
    444            Log.e(LOGTAG, "Error when creating test list: " + e.getMessage());
    445        }
    446     }
    447 
    448     // Running all the layout tests at once sometimes
    449     // causes the dumprendertree to run out of memory.
    450     // So, additional tests are added to run the tests
    451     // in chunks.
    452     public void startLayoutTests() {
    453         try {
    454             File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
    455             if (!tests_list.exists())
    456               generateTestList();
    457         } catch (Exception e) {
    458             e.printStackTrace();
    459         }
    460 
    461         executeLayoutTests(false);
    462     }
    463 
    464     public void resumeLayoutTests() {
    465         executeLayoutTests(true);
    466     }
    467 
    468     public void copyResultsAndRunnerAssetsToCache() {
    469         try {
    470             Context targetContext = getInstrumentation().getTargetContext();
    471             File cacheDir = targetContext.getCacheDir();
    472 
    473             for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) {
    474                 InputStream in = targetContext.getAssets().open(
    475                         LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]);
    476                 OutputStream out = new FileOutputStream(new File(cacheDir,
    477                         LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]));
    478 
    479                 byte[] buf = new byte[2048];
    480                 int len;
    481 
    482                 while ((len = in.read(buf)) >= 0 ) {
    483                     out.write(buf, 0, len);
    484                 }
    485                 out.close();
    486                 in.close();
    487             }
    488         }catch (IOException e) {
    489           e.printStackTrace();
    490         }
    491 
    492     }
    493 
    494 }
    495