1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.dumprendertree; 18 19 import com.android.dumprendertree.TestShellActivity.DumpDataType; 20 import com.android.dumprendertree.forwarder.AdbUtils; 21 import com.android.dumprendertree.forwarder.ForwardServer; 22 import com.android.dumprendertree.forwarder.ForwardService; 23 24 import android.app.Instrumentation; 25 import android.content.Intent; 26 import android.os.Bundle; 27 import android.test.ActivityInstrumentationTestCase2; 28 import android.util.Log; 29 30 import java.io.BufferedOutputStream; 31 import java.io.BufferedReader; 32 import java.io.File; 33 import java.io.FileNotFoundException; 34 import java.io.FileOutputStream; 35 import java.io.FileReader; 36 import java.io.IOException; 37 import java.io.InputStream; 38 import java.io.OutputStream; 39 import java.util.Vector; 40 41 // TestRecorder creates four files ... 42 // - passing tests 43 // - failing tests 44 // - tests for which results are ignored 45 // - tests with no text results available 46 // TestRecorder does not have the ability to clear the results. 47 class MyTestRecorder { 48 private BufferedOutputStream mBufferedOutputPassedStream; 49 private BufferedOutputStream mBufferedOutputFailedStream; 50 private BufferedOutputStream mBufferedOutputIgnoreResultStream; 51 private BufferedOutputStream mBufferedOutputNoResultStream; 52 53 public void passed(String layout_file) { 54 try { 55 mBufferedOutputPassedStream.write(layout_file.getBytes()); 56 mBufferedOutputPassedStream.write('\n'); 57 mBufferedOutputPassedStream.flush(); 58 } catch(Exception e) { 59 e.printStackTrace(); 60 } 61 } 62 63 public void failed(String layout_file) { 64 try { 65 mBufferedOutputFailedStream.write(layout_file.getBytes()); 66 mBufferedOutputFailedStream.write('\n'); 67 mBufferedOutputFailedStream.flush(); 68 } catch(Exception e) { 69 e.printStackTrace(); 70 } 71 } 72 73 public void ignoreResult(String layout_file) { 74 try { 75 mBufferedOutputIgnoreResultStream.write(layout_file.getBytes()); 76 mBufferedOutputIgnoreResultStream.write('\n'); 77 mBufferedOutputIgnoreResultStream.flush(); 78 } catch(Exception e) { 79 e.printStackTrace(); 80 } 81 } 82 83 public void noResult(String layout_file) { 84 try { 85 mBufferedOutputNoResultStream.write(layout_file.getBytes()); 86 mBufferedOutputNoResultStream.write('\n'); 87 mBufferedOutputNoResultStream.flush(); 88 } catch(Exception e) { 89 e.printStackTrace(); 90 } 91 } 92 93 public MyTestRecorder(boolean resume) { 94 try { 95 File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt"); 96 File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt"); 97 File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt"); 98 File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt"); 99 100 mBufferedOutputPassedStream = 101 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume)); 102 mBufferedOutputFailedStream = 103 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume)); 104 mBufferedOutputIgnoreResultStream = 105 new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume)); 106 mBufferedOutputNoResultStream = 107 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume)); 108 } catch (Exception e) { 109 e.printStackTrace(); 110 } 111 } 112 113 public void close() { 114 try { 115 mBufferedOutputPassedStream.close(); 116 mBufferedOutputFailedStream.close(); 117 mBufferedOutputIgnoreResultStream.close(); 118 mBufferedOutputNoResultStream.close(); 119 } catch (Exception e) { 120 e.printStackTrace(); 121 } 122 } 123 } 124 125 126 public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> { 127 128 private static final String LOGTAG = "LayoutTests"; 129 static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000; 130 131 static final String LAYOUT_TESTS_ROOT = "/sdcard/android/layout_tests/"; 132 static final String LAYOUT_TESTS_RESULT_DIR = "/sdcard/android/layout_tests_results/"; 133 static final String ANDROID_EXPECTED_RESULT_DIR = "/sdcard/android/expected_results/"; 134 static final String LAYOUT_TESTS_LIST_FILE = "/sdcard/android/layout_tests_list.txt"; 135 static final String TEST_STATUS_FILE = "/sdcard/android/running_test.txt"; 136 static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = { 137 "results/layout_tests_passed.txt", 138 "results/layout_tests_failed.txt", 139 "results/layout_tests_nontext.txt", 140 "results/layout_tests_crashed.txt", 141 "run_layout_tests.py" 142 }; 143 144 static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt"; 145 static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt"; 146 static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt"; 147 static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py"; 148 149 private MyTestRecorder mResultRecorder; 150 private Vector<String> mTestList; 151 // Whether we should ignore the result for the corresponding test. Ordered same as mTestList. 152 private Vector<Boolean> mTestListIgnoreResult; 153 private boolean mRebaselineResults; 154 // The JavaScript engine currently in use. This determines which set of Android-specific 155 // expected test results we use. 156 private String mJsEngine; 157 private String mTestPathPrefix; 158 private boolean mFinished; 159 160 public LayoutTestsAutoTest() { 161 super("com.android.dumprendertree", TestShellActivity.class); 162 } 163 164 // This function writes the result of the layout test to 165 // Am status so that it can be picked up from a script. 166 private void passOrFailCallback(String file, boolean result) { 167 Instrumentation inst = getInstrumentation(); 168 Bundle bundle = new Bundle(); 169 bundle.putBoolean(file, result); 170 inst.sendStatus(0, bundle); 171 } 172 173 private void getTestList() { 174 // Read test list. 175 try { 176 BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE)); 177 String line = inReader.readLine(); 178 while (line != null) { 179 if (line.startsWith(mTestPathPrefix)) { 180 String[] components = line.split(" "); 181 mTestList.add(components[0]); 182 mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT")); 183 } 184 line = inReader.readLine(); 185 } 186 inReader.close(); 187 Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s)."); 188 } catch (Exception e) { 189 Log.e(LOGTAG, "Error while reading test list : " + e.getMessage()); 190 } 191 } 192 193 private void resumeTestList() { 194 // read out the test name it stoped last time. 195 try { 196 String line = FsUtils.readTestStatus(TEST_STATUS_FILE); 197 for (int i = 0; i < mTestList.size(); i++) { 198 if (mTestList.elementAt(i).equals(line)) { 199 mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size())); 200 mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size())); 201 break; 202 } 203 } 204 } catch (Exception e) { 205 Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE); 206 } 207 } 208 209 private void clearTestStatus() { 210 // Delete TEST_STATUS_FILE 211 try { 212 File f = new File(TEST_STATUS_FILE); 213 if (f.delete()) 214 Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE); 215 else 216 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE); 217 } catch (Exception e) { 218 Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage()); 219 } 220 } 221 222 private String getResultFile(String test) { 223 String shortName = test.substring(0, test.lastIndexOf('.')); 224 // Write actual results to result directory. 225 return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt"; 226 } 227 228 // Gets the file which contains WebKit's expected results for this test. 229 private String getExpectedResultFile(String test) { 230 // The generic result is at <path>/<name>-expected.txt 231 // First try the Android-specific result at 232 // platform/android-<js-engine>/<path>/<name>-expected.txt 233 int pos = test.lastIndexOf('.'); 234 if (pos == -1) 235 return null; 236 String genericExpectedResult = test.substring(0, pos) + "-expected.txt"; 237 String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/"; 238 String androidExpectedResult = 239 genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_ROOT + androidExpectedResultsDir); 240 File f = new File(androidExpectedResult); 241 return f.exists() ? androidExpectedResult : genericExpectedResult; 242 } 243 244 // Gets the file which contains the actual results of running the test on 245 // Android, generated by a previous run which set a new baseline. 246 private String getAndroidExpectedResultFile(String expectedResultFile) { 247 return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR); 248 } 249 250 // Wrap up 251 private void failedCase(String file) { 252 Log.w("Layout test: ", file + " failed"); 253 mResultRecorder.failed(file); 254 } 255 256 private void passedCase(String file) { 257 Log.v("Layout test:", file + " passed"); 258 mResultRecorder.passed(file); 259 } 260 261 private void ignoreResultCase(String file) { 262 Log.v("Layout test:", file + " ignore result"); 263 mResultRecorder.ignoreResult(file); 264 } 265 266 private void noResultCase(String file) { 267 Log.v("Layout test:", file + " no expected result"); 268 mResultRecorder.noResult(file); 269 } 270 271 private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) { 272 Log.v(LOGTAG, " Processing result: " + testFile); 273 274 if (ignoreResult) { 275 ignoreResultCase(testFile); 276 return; 277 } 278 279 File actual = new File(actualResultFile); 280 File expected = new File(expectedResultFile); 281 if (actual.exists() && expected.exists()) { 282 try { 283 if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) { 284 passedCase(testFile); 285 } else { 286 failedCase(testFile); 287 } 288 } catch (FileNotFoundException ex) { 289 Log.e(LOGTAG, "File not found : " + ex.getMessage()); 290 } catch (IOException ex) { 291 Log.e(LOGTAG, "IO Error : " + ex.getMessage()); 292 } 293 return; 294 } 295 296 if (!expected.exists()) { 297 noResultCase(testFile); 298 } 299 } 300 301 private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult) { 302 activity.setCallback(new TestShellCallback() { 303 public void finished() { 304 synchronized (LayoutTestsAutoTest.this) { 305 mFinished = true; 306 LayoutTestsAutoTest.this.notifyAll(); 307 } 308 } 309 310 public void timedOut(String url) { 311 Log.v(LOGTAG, "layout timeout: " + url); 312 } 313 }); 314 315 String resultFile = getResultFile(test); 316 if (resultFile == null) { 317 // Simply ignore this test. 318 return; 319 } 320 if (mRebaselineResults) { 321 String expectedResultFile = getExpectedResultFile(test); 322 File f = new File(expectedResultFile); 323 if (f.exists()) { 324 return; // don't run test and don't overwrite default tests. 325 } 326 327 resultFile = getAndroidExpectedResultFile(expectedResultFile); 328 } 329 330 mFinished = false; 331 Intent intent = new Intent(Intent.ACTION_VIEW); 332 intent.setClass(activity, TestShellActivity.class); 333 intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP); 334 intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test)); 335 intent.putExtra(TestShellActivity.RESULT_FILE, resultFile); 336 intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout); 337 activity.startActivity(intent); 338 339 // Wait until done. 340 synchronized (this) { 341 while(!mFinished){ 342 try { 343 this.wait(); 344 } catch (InterruptedException e) { } 345 } 346 } 347 348 if (!mRebaselineResults) { 349 String expectedResultFile = getExpectedResultFile(test); 350 File f = new File(expectedResultFile); 351 if (!f.exists()) { 352 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile); 353 } 354 355 processResult(test, resultFile, expectedResultFile, ignoreResult); 356 } 357 } 358 359 // Invokes running of layout tests 360 // and waits till it has finished running. 361 public void executeLayoutTests(boolean resume) { 362 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 363 // A convenient method to be called by another activity. 364 365 if (runner.mTestPath == null) { 366 Log.e(LOGTAG, "No test specified"); 367 return; 368 } 369 370 this.mTestList = new Vector<String>(); 371 this.mTestListIgnoreResult = new Vector<Boolean>(); 372 373 // Read settings 374 mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath(); 375 mRebaselineResults = runner.mRebaseline; 376 // JSC is the default JavaScript engine. 377 mJsEngine = runner.mJsEngine == null ? "jsc" : runner.mJsEngine; 378 379 int timeout = runner.mTimeoutInMillis; 380 if (timeout <= 0) { 381 timeout = DEFAULT_TIMEOUT_IN_MILLIS; 382 } 383 384 this.mResultRecorder = new MyTestRecorder(resume); 385 386 if (!resume) 387 clearTestStatus(); 388 389 getTestList(); 390 if (resume) 391 resumeTestList(); 392 393 TestShellActivity activity = getActivity(); 394 activity.setDefaultDumpDataType(DumpDataType.DUMP_AS_TEXT); 395 396 // Run tests. 397 int addr = -1; 398 try{ 399 addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com"); 400 } catch (IOException ioe) { 401 Log.w(LOGTAG, "error while resolving test host name", ioe); 402 } 403 if(addr == -1) { 404 Log.w(LOGTAG, "failed to resolve test host. http tests will fail."); 405 } 406 for (int i = 0; i < mTestList.size(); i++) { 407 String s = mTestList.elementAt(i); 408 boolean ignoreResult = mTestListIgnoreResult.elementAt(i); 409 FsUtils.updateTestStatus(TEST_STATUS_FILE, s); 410 // Run tests 411 runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult); 412 } 413 414 FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE"); 415 ForwardService.getForwardService().stopForwardService(); 416 activity.finish(); 417 } 418 419 private String getTestPath() { 420 LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation(); 421 422 String test_path = LAYOUT_TESTS_ROOT; 423 if (runner.mTestPath != null) { 424 test_path += runner.mTestPath; 425 } 426 test_path = new File(test_path).getAbsolutePath(); 427 Log.v("LayoutTestsAutoTest", " Test path : " + test_path); 428 429 return test_path; 430 } 431 432 public void generateTestList() { 433 try { 434 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 435 BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false)); 436 FsUtils.findLayoutTestsRecursively(bos, getTestPath(), false); // Don't ignore results 437 bos.flush(); 438 bos.close(); 439 } catch (Exception e) { 440 Log.e(LOGTAG, "Error when creating test list: " + e.getMessage()); 441 } 442 } 443 444 // Running all the layout tests at once sometimes 445 // causes the dumprendertree to run out of memory. 446 // So, additional tests are added to run the tests 447 // in chunks. 448 public void startLayoutTests() { 449 try { 450 File tests_list = new File(LAYOUT_TESTS_LIST_FILE); 451 if (!tests_list.exists()) 452 generateTestList(); 453 } catch (Exception e) { 454 e.printStackTrace(); 455 } 456 457 executeLayoutTests(false); 458 } 459 460 public void resumeLayoutTests() { 461 executeLayoutTests(true); 462 } 463 464 public void copyResultsAndRunnerAssetsToCache() { 465 try { 466 String out_dir = getActivity().getApplicationContext().getCacheDir().getPath() + "/"; 467 468 for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) { 469 InputStream in = getActivity().getAssets().open(LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 470 OutputStream out = new FileOutputStream(out_dir + LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]); 471 472 byte[] buf = new byte[2048]; 473 int len; 474 475 while ((len = in.read(buf)) >= 0 ) { 476 out.write(buf, 0, len); 477 } 478 out.close(); 479 in.close(); 480 } 481 }catch (IOException e) { 482 e.printStackTrace(); 483 } 484 485 } 486 487 } 488