1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.dumprendertree2; 18 19 import android.content.Context; 20 import android.content.res.AssetManager; 21 import android.content.res.Configuration; 22 import android.content.res.Resources; 23 import android.database.Cursor; 24 import android.os.Build; 25 import android.os.Message; 26 import android.util.DisplayMetrics; 27 import android.util.Log; 28 29 import com.android.dumprendertree2.forwarder.ForwarderManager; 30 31 import java.io.File; 32 import java.net.MalformedURLException; 33 import java.net.URI; 34 import java.net.URL; 35 import java.text.SimpleDateFormat; 36 import java.util.ArrayList; 37 import java.util.Date; 38 import java.util.List; 39 import java.util.regex.Matcher; 40 import java.util.regex.Pattern; 41 42 /** 43 * A class that collects information about tests that ran and can create HTML 44 * files with summaries and easy navigation. 45 */ 46 public class Summarizer { 47 48 private static final String LOG_TAG = "Summarizer"; 49 50 private static final String CSS = 51 "<style type=\"text/css\">" + 52 "* {" + 53 " font-family: Verdana;" + 54 " border: 0;" + 55 " margin: 0;" + 56 " padding: 0;}" + 57 "body {" + 58 " margin: 10px;}" + 59 "h1 {" + 60 " font-size: 24px;" + 61 " margin: 4px 0 4px 0;}" + 62 "h2 {" + 63 " font-size:18px;" + 64 " text-transform: uppercase;" + 65 " margin: 20px 0 3px 0;}" + 66 "h3, h3 a {" + 67 " font-size: 14px;" + 68 " color: black;" + 69 " text-decoration: none;" + 70 " margin-top: 4px;" + 71 " margin-bottom: 2px;}" + 72 "h3 a span.path {" + 73 " text-decoration: underline;}" + 74 "h3 span.tri {" + 75 " text-decoration: none;" + 76 " float: left;" + 77 " width: 20px;}" + 78 "h3 span.sqr {" + 79 " text-decoration: none;" + 80 " float: left;" + 81 " width: 20px;}" + 82 "h3 span.sqr_pass {" + 83 " color: #8ee100;}" + 84 "h3 span.sqr_fail {" + 85 " color: #c30000;}" + 86 "span.source {" + 87 " display: block;" + 88 " font-size: 10px;" + 89 " color: #888;" + 90 " margin-left: 20px;" + 91 " margin-bottom: 1px;}" + 92 "span.source a {" + 93 " font-size: 10px;" + 94 " color: #888;}" + 95 "h3 img {" + 96 " width: 8px;" + 97 " margin-right: 4px;}" + 98 "div.diff {" + 99 " margin-bottom: 25px;}" + 100 "div.diff a {" + 101 " font-size: 12px;" + 102 " color: #888;}" + 103 "table.visual_diff {" + 104 " border-bottom: 0px solid;" + 105 " border-collapse: collapse;" + 106 " width: 100%;" + 107 " margin-bottom: 2px;}" + 108 "table.visual_diff tr.headers td {" + 109 " border-bottom: 1px solid;" + 110 " border-top: 0;" + 111 " padding-bottom: 3px;}" + 112 "table.visual_diff tr.results td {" + 113 " border-top: 1px dashed;" + 114 " border-right: 1px solid;" + 115 " font-size: 15px;" + 116 " vertical-align: top;}" + 117 "table.visual_diff tr.results td.line_count {" + 118 " background-color:#aaa;" + 119 " min-width:20px;" + 120 " text-align: right;" + 121 " border-right: 1px solid;" + 122 " border-left: 1px solid;" + 123 " padding: 2px 1px 2px 0px;}" + 124 "table.visual_diff tr.results td.line {" + 125 " padding: 2px 0px 2px 4px;" + 126 " border-right: 1px solid;" + 127 " width: 49.8%;}" + 128 "table.visual_diff tr.footers td {" + 129 " border-top: 1px solid;" + 130 " border-bottom: 0;}" + 131 "table.visual_diff tr td.space {" + 132 " border: 0;" + 133 " width: 0.4%}" + 134 "div.space {" + 135 " margin-top:4px;}" + 136 "span.eql {" + 137 " background-color: #f3f3f3;}" + 138 "span.del {" + 139 " background-color: #ff8888; }" + 140 "span.ins {" + 141 " background-color: #88ff88; }" + 142 "table.summary {" + 143 " border: 1px solid black;" + 144 " margin-top: 20px;}" + 145 "table.summary td {" + 146 " padding: 3px;}" + 147 "span.listItem {" + 148 " font-size: 11px;" + 149 " font-weight: normal;" + 150 " text-transform: uppercase;" + 151 " padding: 3px;" + 152 " -webkit-border-radius: 4px;}" + 153 "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" + 154 " background-color: #ccc;" + 155 " color: black;}" + 156 "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" + 157 " background-color: #a700e4;" + 158 " color: #fff;}" + 159 "span.timed_out {" + 160 " background-color: #f3cb00;" + 161 " color: black;}" + 162 "span.crashed {" + 163 " background-color: #c30000;" + 164 " color: #fff;}" + 165 "span.noLtc {" + 166 " background-color: #944000;" + 167 " color: #fff;}" + 168 "span.noEventSender {" + 169 " background-color: #815600;" + 170 " color: #fff;}" + 171 "</style>"; 172 173 private static final String SCRIPT = 174 "<script type=\"text/javascript\">" + 175 " function toggleDisplay(id) {" + 176 " element = document.getElementById(id);" + 177 " triangle = document.getElementById('tri.' + id);" + 178 " if (element.style.display == 'none') {" + 179 " element.style.display = 'inline';" + 180 " triangle.innerHTML = '▼ ';" + 181 " } else {" + 182 " element.style.display = 'none';" + 183 " triangle.innerHTML = '▶ ';" + 184 " }" + 185 " }" + 186 "</script>"; 187 188 /** TODO: Make it a setting */ 189 private static final String HTML_DETAILS_RELATIVE_PATH = "details.html"; 190 private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt"; 191 192 private static final int RESULTS_PER_DUMP = 500; 193 private static final int RESULTS_PER_DB_ACCESS = 50; 194 195 private int mCrashedTestsCount = 0; 196 private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>(); 197 private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>(); 198 private List<AbstractResult> mExpectedPasses = new ArrayList<AbstractResult>(); 199 private List<AbstractResult> mUnexpectedPasses = new ArrayList<AbstractResult>(); 200 201 private Cursor mUnexpectedFailuresCursor; 202 private Cursor mExpectedFailuresCursor; 203 private Cursor mUnexpectedPassesCursor; 204 private Cursor mExpectedPassesCursor; 205 206 private FileFilter mFileFilter; 207 private String mResultsRootDirPath; 208 private String mTestsRelativePath; 209 private Date mDate; 210 211 private int mResultsSinceLastHtmlDump = 0; 212 private int mResultsSinceLastDbAccess = 0; 213 214 private SummarizerDBHelper mDbHelper; 215 216 public Summarizer(String resultsRootDirPath, Context context) { 217 mFileFilter = new FileFilter(); 218 mResultsRootDirPath = resultsRootDirPath; 219 220 /** 221 * We don't run the database I/O in a separate thread to avoid consumer/producer problem 222 * and to simplify code. 223 */ 224 mDbHelper = new SummarizerDBHelper(context); 225 mDbHelper.open(); 226 } 227 228 public static URI getDetailsUri() { 229 return new File(ManagerService.RESULTS_ROOT_DIR_PATH + File.separator + 230 HTML_DETAILS_RELATIVE_PATH).toURI(); 231 } 232 233 public void appendTest(AbstractResult result) { 234 String relativePath = result.getRelativePath(); 235 236 if (result.didCrash()) { 237 mCrashedTestsCount++; 238 } 239 240 if (result.didPass()) { 241 result.clearResults(); 242 if (mFileFilter.isFail(relativePath)) { 243 mUnexpectedPasses.add(result); 244 } else { 245 mExpectedPasses.add(result); 246 } 247 } else { 248 if (mFileFilter.isFail(relativePath)) { 249 mExpectedFailures.add(result); 250 } else { 251 mUnexpectedFailures.add(result); 252 } 253 } 254 255 if (++mResultsSinceLastDbAccess == RESULTS_PER_DB_ACCESS) { 256 persistLists(); 257 clearLists(); 258 } 259 } 260 261 private void clearLists() { 262 mUnexpectedFailures.clear(); 263 mExpectedFailures.clear(); 264 mUnexpectedPasses.clear(); 265 mExpectedPasses.clear(); 266 } 267 268 private void persistLists() { 269 persistListToTable(mUnexpectedFailures, SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE); 270 persistListToTable(mExpectedFailures, SummarizerDBHelper.EXPECTED_FAILURES_TABLE); 271 persistListToTable(mUnexpectedPasses, SummarizerDBHelper.UNEXPECTED_PASSES_TABLE); 272 persistListToTable(mExpectedPasses, SummarizerDBHelper.EXPECTED_PASSES_TABLE); 273 mResultsSinceLastDbAccess = 0; 274 } 275 276 private void persistListToTable(List<AbstractResult> results, String table) { 277 for (AbstractResult abstractResult : results) { 278 mDbHelper.insertAbstractResult(abstractResult, table); 279 } 280 } 281 282 public void setTestsRelativePath(String testsRelativePath) { 283 mTestsRelativePath = testsRelativePath; 284 } 285 286 public void summarize(Message onFinishMessage) { 287 persistLists(); 288 clearLists(); 289 290 mUnexpectedFailuresCursor = 291 mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE); 292 mUnexpectedPassesCursor = 293 mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_PASSES_TABLE); 294 mExpectedFailuresCursor = 295 mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_FAILURES_TABLE); 296 mExpectedPassesCursor = 297 mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_PASSES_TABLE); 298 299 String webKitRevision = getWebKitRevision(); 300 createHtmlDetails(webKitRevision); 301 createTxtSummary(webKitRevision); 302 303 clearLists(); 304 mUnexpectedFailuresCursor.close(); 305 mUnexpectedPassesCursor.close(); 306 mExpectedFailuresCursor.close(); 307 mExpectedPassesCursor.close(); 308 309 onFinishMessage.sendToTarget(); 310 } 311 312 public void reset() { 313 mCrashedTestsCount = 0; 314 clearLists(); 315 mDbHelper.reset(); 316 mDate = new Date(); 317 } 318 319 private void dumpHtmlToFile(StringBuilder html, boolean append) { 320 FsUtils.writeDataToStorage(new File(mResultsRootDirPath, HTML_DETAILS_RELATIVE_PATH), 321 html.toString().getBytes(), append); 322 html.setLength(0); 323 mResultsSinceLastHtmlDump = 0; 324 } 325 326 private void createTxtSummary(String webKitRevision) { 327 StringBuilder txt = new StringBuilder(); 328 329 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); 330 txt.append("Path: " + mTestsRelativePath + "\n"); 331 txt.append("Date: " + dateFormat.format(mDate) + "\n"); 332 txt.append("Build fingerprint: " + Build.FINGERPRINT + "\n"); 333 txt.append("WebKit version: " + getWebKitVersionFromUserAgentString() + "\n"); 334 txt.append("WebKit revision: " + webKitRevision + "\n"); 335 336 txt.append("TOTAL: " + getTotalTestCount() + "\n"); 337 txt.append("CRASHED (among all tests): " + mCrashedTestsCount + "\n"); 338 txt.append("UNEXPECTED FAILURES: " + mUnexpectedFailuresCursor.getCount() + "\n"); 339 txt.append("UNEXPECTED PASSES: " + mUnexpectedPassesCursor.getCount() + "\n"); 340 txt.append("EXPECTED FAILURES: " + mExpectedFailuresCursor.getCount() + "\n"); 341 txt.append("EXPECTED PASSES: " + mExpectedPassesCursor.getCount() + "\n"); 342 343 FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH), 344 txt.toString().getBytes(), false); 345 } 346 347 private void createHtmlDetails(String webKitRevision) { 348 StringBuilder html = new StringBuilder(); 349 350 html.append("<html><head>"); 351 html.append(CSS); 352 html.append(SCRIPT); 353 html.append("</head><body>"); 354 355 createTopSummaryTable(webKitRevision, html); 356 dumpHtmlToFile(html, false); 357 358 createResultsList(html, "Unexpected failures", mUnexpectedFailuresCursor); 359 createResultsList(html, "Unexpected passes", mUnexpectedPassesCursor); 360 createResultsList(html, "Expected failures", mExpectedFailuresCursor); 361 createResultsList(html, "Expected passes", mExpectedPassesCursor); 362 363 html.append("</body></html>"); 364 dumpHtmlToFile(html, true); 365 } 366 367 private int getTotalTestCount() { 368 return mUnexpectedFailuresCursor.getCount() + 369 mUnexpectedPassesCursor.getCount() + 370 mExpectedPassesCursor.getCount() + 371 mExpectedFailuresCursor.getCount(); 372 } 373 374 private String getWebKitVersionFromUserAgentString() { 375 Resources resources = new Resources(new AssetManager(), new DisplayMetrics(), 376 new Configuration()); 377 String userAgent = 378 resources.getString(com.android.internal.R.string.web_user_agent); 379 380 Matcher matcher = Pattern.compile("AppleWebKit/([0-9]+?\\.[0-9])").matcher(userAgent); 381 if (matcher.find()) { 382 return matcher.group(1); 383 } 384 return "unknown"; 385 } 386 387 private String getWebKitRevision() { 388 URL url = null; 389 try { 390 url = new URL(ForwarderManager.getHostSchemePort(false) + "ThirdPartyProject.prop"); 391 } catch (MalformedURLException e) { 392 assert false; 393 } 394 395 String thirdPartyProjectContents = new String(FsUtils.readDataFromUrl(url)); 396 Matcher matcher = Pattern.compile("^version=([0-9]+)", Pattern.MULTILINE).matcher( 397 thirdPartyProjectContents); 398 if (matcher.find()) { 399 return matcher.group(1); 400 } 401 return "unknown"; 402 } 403 404 private void createTopSummaryTable(String webKitRevision, StringBuilder html) { 405 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); 406 html.append("<h1>" + "Layout tests' results for: " + 407 (mTestsRelativePath.equals("") ? "all tests" : mTestsRelativePath) + "</h1>"); 408 html.append("<h3>" + "Date: " + dateFormat.format(new Date()) + "</h3>"); 409 html.append("<h3>" + "Build fingerprint: " + Build.FINGERPRINT + "</h3>"); 410 html.append("<h3>" + "WebKit version: " + getWebKitVersionFromUserAgentString() + "</h3>"); 411 412 html.append("<h3>" + "WebKit revision: "); 413 html.append("<a href=\"http://trac.webkit.org/browser/trunk?rev=" + webKitRevision + 414 "\" target=\"_blank\"><span class=\"path\">" + webKitRevision + "</span></a>"); 415 html.append("</h3>"); 416 417 html.append("<table class=\"summary\">"); 418 createSummaryTableRow(html, "TOTAL", getTotalTestCount()); 419 createSummaryTableRow(html, "CRASHED (among all tests)", mCrashedTestsCount); 420 createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailuresCursor.getCount()); 421 createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPassesCursor.getCount()); 422 createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailuresCursor.getCount()); 423 createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPassesCursor.getCount()); 424 html.append("</table>"); 425 } 426 427 private void createSummaryTableRow(StringBuilder html, String caption, int size) { 428 html.append("<tr>"); 429 html.append(" <td>" + caption + "</td>"); 430 html.append(" <td>" + size + "</td>"); 431 html.append("</tr>"); 432 } 433 434 private void createResultsList( 435 StringBuilder html, String title, Cursor cursor) { 436 String relativePath; 437 String id = ""; 438 AbstractResult.ResultCode resultCode; 439 440 html.append("<h2>" + title + " [" + cursor.getCount() + "]</h2>"); 441 442 if (!cursor.moveToFirst()) { 443 return; 444 } 445 446 AbstractResult result; 447 do { 448 result = SummarizerDBHelper.getAbstractResult(cursor); 449 450 relativePath = result.getRelativePath(); 451 resultCode = result.getResultCode(); 452 453 html.append("<h3>"); 454 455 /** 456 * Technically, two different paths could end up being the same, because 457 * ':' is a valid character in a path. However, it is probably not going 458 * to cause any problems in this case 459 */ 460 id = relativePath.replace(File.separator, ":"); 461 462 /** Write the test name */ 463 if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { 464 html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); 465 html.append("return false;\">"); 466 html.append("<span class=\"tri\" id=\"tri." + id + "\">▶ </span>"); 467 html.append("<span class=\"path\">" + relativePath + "</span>"); 468 html.append("</a>"); 469 } else { 470 html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\""); 471 html.append(" target=\"_blank\">"); 472 html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail")); 473 html.append("\">■ </span>"); 474 html.append("<span class=\"path\">" + result.getRelativePath() + "</span>"); 475 html.append("</a>"); 476 } 477 478 if (!result.didPass()) { 479 appendTags(html, result); 480 } 481 482 html.append("</h3>"); 483 appendExpectedResultsSources(result, html); 484 485 if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { 486 html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">"); 487 html.append(result.getDiffAsHtml()); 488 html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); 489 html.append("return false;\">Hide</a>"); 490 html.append(" | "); 491 html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\""); 492 html.append(" target=\"_blank\">Show source</a>"); 493 html.append("</div>"); 494 } 495 496 html.append("<div class=\"space\"></div>"); 497 498 if (++mResultsSinceLastHtmlDump == RESULTS_PER_DUMP) { 499 dumpHtmlToFile(html, true); 500 } 501 502 cursor.moveToNext(); 503 } while (!cursor.isAfterLast()); 504 } 505 506 private void appendTags(StringBuilder html, AbstractResult result) { 507 /** Tag tests which crash, time out or where results don't match */ 508 if (result.didCrash()) { 509 html.append(" <span class=\"listItem crashed\">Crashed</span>"); 510 } else { 511 if (result.didTimeOut()) { 512 html.append(" <span class=\"listItem timed_out\">Timed out</span>"); 513 } 514 AbstractResult.ResultCode resultCode = result.getResultCode(); 515 if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) { 516 html.append(" <span class=\"listItem " + resultCode.name() + "\">"); 517 html.append(resultCode.toString()); 518 html.append("</span>"); 519 } 520 } 521 522 /** Detect missing LTC function */ 523 String additionalTextOutputString = result.getAdditionalTextOutputString(); 524 if (additionalTextOutputString != null && 525 additionalTextOutputString.contains("com.android.dumprendertree") && 526 additionalTextOutputString.contains("has no method")) { 527 if (additionalTextOutputString.contains("LayoutTestController")) { 528 html.append(" <span class=\"listItem noLtc\">LTC function missing</span>"); 529 } 530 if (additionalTextOutputString.contains("EventSender")) { 531 html.append(" <span class=\"listItem noEventSender\">"); 532 html.append("ES function missing</span>"); 533 } 534 } 535 } 536 537 private static final void appendExpectedResultsSources(AbstractResult result, 538 StringBuilder html) { 539 String textSource = result.getExpectedTextResultPath(); 540 String imageSource = result.getExpectedImageResultPath(); 541 542 if (result.didCrash()) { 543 html.append("<span class=\"source\">Did not look for expected results</span>"); 544 return; 545 } 546 547 if (textSource == null) { 548 // Show if a text result is missing. We may want to revisit this decision when we add 549 // support for image results. 550 html.append("<span class=\"source\">Expected textual result missing</span>"); 551 } else { 552 html.append("<span class=\"source\">Expected textual result from: "); 553 html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" + 554 textSource + "\""); 555 html.append(" target=\"_blank\">"); 556 html.append(textSource + "</a></span>"); 557 } 558 if (imageSource != null) { 559 html.append("<span class=\"source\">Expected image result from: "); 560 html.append("<a href=\"" + ForwarderManager.getHostSchemePort(false) + "LayoutTests/" + 561 imageSource + "\""); 562 html.append(" target=\"_blank\">"); 563 html.append(imageSource + "</a></span>"); 564 } 565 } 566 567 private static final URL getViewSourceUrl(String relativePath) { 568 URL url = null; 569 try { 570 url = new URL("http", "localhost", ForwarderManager.HTTP_PORT, 571 "/Tools/DumpRenderTree/android/view_source.php?src=" + 572 relativePath); 573 } catch (MalformedURLException e) { 574 assert false : "relativePath=" + relativePath; 575 } 576 return url; 577 } 578 } 579
