1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Rebaselining tool that automatically produces baselines for all platforms. 31 32 The script does the following for each platform specified: 33 1. Compile a list of tests that need rebaselining. 34 2. Download test result archive from buildbot for the platform. 35 3. Extract baselines from the archive file for all identified files. 36 4. Add new baselines to SVN repository. 37 5. For each test that has been rebaselined, remove this platform option from 38 the test in test_expectation.txt. If no other platforms remain after 39 removal, delete the rebaselined test from the file. 40 41 At the end, the script generates a html that compares old and new baselines. 42 """ 43 44 from __future__ import with_statement 45 46 import copy 47 import logging 48 import optparse 49 import re 50 import sys 51 import time 52 53 from webkitpy.common.checkout import scm 54 from webkitpy.common.system import zipfileset 55 from webkitpy.common.system import path 56 from webkitpy.common.system import urlfetcher 57 from webkitpy.common.system.executive import ScriptError 58 59 from webkitpy.layout_tests import port 60 from webkitpy.layout_tests import read_checksum_from_png 61 from webkitpy.layout_tests.layout_package import test_expectations 62 63 _log = logging.getLogger(__name__) 64 65 BASELINE_SUFFIXES = ('.txt', '.png', '.checksum') 66 67 ARCHIVE_DIR_NAME_DICT = { 68 'chromium-win-win7': 'Webkit_Win7', 69 'chromium-win-vista': 'Webkit_Vista', 70 'chromium-win-xp': 'Webkit_Win', 71 'chromium-mac-leopard': 'Webkit_Mac10_5', 72 'chromium-mac-snowleopard': 'Webkit_Mac10_6', 73 'chromium-linux-x86': 'Webkit_Linux', 74 'chromium-linux-x86_64': 'Webkit_Linux_64', 75 'chromium-gpu-mac-snowleopard': 'Webkit_Mac10_6_-_GPU', 76 'chromium-gpu-win-xp': 'Webkit_Win_-_GPU', 77 'chromium-gpu-win-win7': 'Webkit_Win7_-_GPU', 78 'chromium-gpu-linux': 'Webkit_Linux_-_GPU', 79 'chromium-gpu-linux-x86_64': 'Webkit_Linux_64_-_GPU', 80 } 81 82 83 def log_dashed_string(text, platform, logging_level=logging.INFO): 84 """Log text message with dashes on both sides.""" 85 86 msg = text 87 if platform: 88 msg += ': ' + platform 89 if len(msg) < 78: 90 dashes = '-' * ((78 - len(msg)) / 2) 91 msg = '%s %s %s' % (dashes, msg, dashes) 92 93 if logging_level == logging.ERROR: 94 _log.error(msg) 95 elif logging_level == logging.WARNING: 96 _log.warn(msg) 97 else: 98 _log.info(msg) 99 100 101 def setup_html_directory(filesystem, parent_directory): 102 """Setup the directory to store html results. 103 104 All html related files are stored in the "rebaseline_html" subdirectory of 105 the parent directory. The path to the created directory is returned. 106 """ 107 108 if not parent_directory: 109 parent_directory = str(filesystem.mkdtemp()) 110 else: 111 filesystem.maybe_make_directory(parent_directory) 112 113 html_directory = filesystem.join(parent_directory, 'rebaseline_html') 114 _log.info('Html directory: "%s"', html_directory) 115 116 if filesystem.exists(html_directory): 117 filesystem.rmtree(html_directory) 118 _log.info('Deleted html directory: "%s"', html_directory) 119 120 filesystem.maybe_make_directory(html_directory) 121 return html_directory 122 123 124 def get_result_file_fullpath(filesystem, html_directory, baseline_filename, platform, 125 result_type): 126 """Get full path of the baseline result file. 127 128 Args: 129 filesystem: wrapper object 130 html_directory: directory that stores the html related files. 131 baseline_filename: name of the baseline file. 132 platform: win, linux or mac 133 result_type: type of the baseline result: '.txt', '.png'. 134 135 Returns: 136 Full path of the baseline file for rebaselining result comparison. 137 """ 138 139 base, ext = filesystem.splitext(baseline_filename) 140 result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext) 141 fullpath = filesystem.join(html_directory, result_filename) 142 _log.debug(' Result file full path: "%s".', fullpath) 143 return fullpath 144 145 146 class Rebaseliner(object): 147 """Class to produce new baselines for a given platform.""" 148 149 REVISION_REGEX = r'<a href=\"(\d+)/\">' 150 151 def __init__(self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm): 152 """ 153 Args: 154 running_port: the Port the script is running on. 155 target_port: the Port the script uses to find port-specific 156 configuration information like the test_expectations.txt 157 file location and the list of test platforms. 158 platform: the test platform to rebaseline 159 options: the command-line options object. 160 url_fetcher: object that can fetch objects from URLs 161 zip_factory: optional object that can fetch zip files from URLs 162 scm: scm object for adding new baselines 163 """ 164 self._platform = platform 165 self._options = options 166 self._port = running_port 167 self._filesystem = running_port._filesystem 168 self._target_port = target_port 169 170 self._rebaseline_port = port.get(platform, options, filesystem=self._filesystem) 171 self._rebaselining_tests = set() 172 self._rebaselined_tests = [] 173 174 # Create tests and expectations helper which is used to: 175 # -. compile list of tests that need rebaselining. 176 # -. update the tests in test_expectations file after rebaseline 177 # is done. 178 expectations_str = self._rebaseline_port.test_expectations() 179 self._test_expectations = test_expectations.TestExpectations( 180 self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False) 181 self._url_fetcher = url_fetcher 182 self._zip_factory = zip_factory 183 self._scm = scm 184 185 def run(self): 186 """Run rebaseline process.""" 187 188 log_dashed_string('Compiling rebaselining tests', self._platform) 189 if not self._compile_rebaselining_tests(): 190 return False 191 if not self._rebaselining_tests: 192 return True 193 194 log_dashed_string('Downloading archive', self._platform) 195 archive_file = self._download_buildbot_archive() 196 _log.info('') 197 if not archive_file: 198 _log.error('No archive found.') 199 return False 200 201 log_dashed_string('Extracting and adding new baselines', self._platform) 202 if not self._extract_and_add_new_baselines(archive_file): 203 archive_file.close() 204 return False 205 206 archive_file.close() 207 208 log_dashed_string('Updating rebaselined tests in file', self._platform) 209 210 if len(self._rebaselining_tests) != len(self._rebaselined_tests): 211 _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN REBASELINED.') 212 _log.warning(' Total tests needing rebaselining: %d', len(self._rebaselining_tests)) 213 _log.warning(' Total tests rebaselined: %d', len(self._rebaselined_tests)) 214 return False 215 216 _log.warning('All tests needing rebaselining were successfully rebaselined.') 217 218 return True 219 220 def remove_rebaselining_expectations(self, tests, backup): 221 """if backup is True, we backup the original test expectations file.""" 222 new_expectations = self._test_expectations.remove_rebaselined_tests(tests) 223 path = self._target_port.path_to_test_expectations_file() 224 if backup: 225 date_suffix = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) 226 backup_file = '%s.orig.%s' % (path, date_suffix) 227 if self._filesystem.exists(backup_file): 228 self._filesystem.remove(backup_file) 229 _log.info('Saving original file to "%s"', backup_file) 230 self._filesystem.move(path, backup_file) 231 232 self._filesystem.write_text_file(path, new_expectations) 233 # self._scm.add(path) 234 235 def get_rebaselined_tests(self): 236 return self._rebaselined_tests 237 238 def _compile_rebaselining_tests(self): 239 """Compile list of tests that need rebaselining for the platform. 240 241 Returns: 242 False if reftests are wrongly marked as 'needs rebaselining' or True 243 """ 244 245 self._rebaselining_tests = self._test_expectations.get_rebaselining_failures() 246 if not self._rebaselining_tests: 247 _log.warn('No tests found that need rebaselining.') 248 return True 249 250 fs = self._target_port._filesystem 251 for test in self._rebaselining_tests: 252 test_abspath = self._target_port.abspath_for_test(test) 253 if (fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or 254 fs.exists(self._target_port.reftest_expected_mismatch_filename(test_abspath))): 255 _log.error('%s seems to be a reftest. We can not rebase for reftests.', test) 256 self._rebaselining_tests = set() 257 return False 258 259 _log.info('Total number of tests needing rebaselining for "%s": "%d"', 260 self._platform, len(self._rebaselining_tests)) 261 262 test_no = 1 263 for test in self._rebaselining_tests: 264 _log.info(' %d: %s', test_no, test) 265 test_no += 1 266 267 return True 268 269 def _get_latest_revision(self, url): 270 """Get the latest layout test revision number from buildbot. 271 272 Args: 273 url: Url to retrieve layout test revision numbers. 274 275 Returns: 276 latest revision or 277 None on failure. 278 """ 279 280 _log.debug('Url to retrieve revision: "%s"', url) 281 282 content = self._url_fetcher.fetch(url) 283 284 revisions = re.findall(self.REVISION_REGEX, content) 285 if not revisions: 286 _log.error('Failed to find revision, content: "%s"', content) 287 return None 288 289 revisions.sort(key=int) 290 _log.info('Latest revision: "%s"', revisions[len(revisions) - 1]) 291 return revisions[len(revisions) - 1] 292 293 def _get_archive_dir_name(self, platform): 294 """Get name of the layout test archive directory. 295 296 Returns: 297 Directory name or 298 None on failure 299 """ 300 301 if platform in ARCHIVE_DIR_NAME_DICT: 302 return ARCHIVE_DIR_NAME_DICT[platform] 303 else: 304 _log.error('Cannot find platform key %s in archive ' 305 'directory name dictionary', platform) 306 return None 307 308 def _get_archive_url(self): 309 """Generate the url to download latest layout test archive. 310 311 Returns: 312 Url to download archive or 313 None on failure 314 """ 315 316 if self._options.force_archive_url: 317 return self._options.force_archive_url 318 319 dir_name = self._get_archive_dir_name(self._platform) 320 if not dir_name: 321 return None 322 323 _log.debug('Buildbot platform dir name: "%s"', dir_name) 324 325 url_base = '%s/%s/' % (self._options.archive_url, dir_name) 326 latest_revision = self._get_latest_revision(url_base) 327 if latest_revision is None or latest_revision <= 0: 328 return None 329 archive_url = '%s%s/layout-test-results.zip' % (url_base, latest_revision) 330 _log.info('Archive url: "%s"', archive_url) 331 return archive_url 332 333 def _download_buildbot_archive(self): 334 """Download layout test archive file from buildbot and return a handle to it.""" 335 url = self._get_archive_url() 336 if url is None: 337 return None 338 339 archive_file = zipfileset.ZipFileSet(url, filesystem=self._filesystem, 340 zip_factory=self._zip_factory) 341 _log.info('Archive downloaded') 342 return archive_file 343 344 def _extract_and_add_new_baselines(self, zip_file): 345 """Extract new baselines from the zip file and add them to SVN repository. 346 347 Returns: 348 List of tests that have been rebaselined or None on failure.""" 349 zip_namelist = zip_file.namelist() 350 351 _log.debug('zip file namelist:') 352 for name in zip_namelist: 353 _log.debug(' ' + name) 354 355 _log.debug('Platform dir: "%s"', self._platform) 356 357 self._rebaselined_tests = [] 358 for test_no, test in enumerate(self._rebaselining_tests): 359 _log.info('Test %d: %s', test_no + 1, test) 360 self._extract_and_add_new_baseline(test, zip_file) 361 362 zip_file.close() 363 364 return self._rebaselined_tests 365 366 def _extract_and_add_new_baseline(self, test, zip_file): 367 found = False 368 scm_error = False 369 test_basename = self._filesystem.splitext(test)[0] 370 for suffix in BASELINE_SUFFIXES: 371 archive_test_name = 'layout-test-results/%s-actual%s' % (test_basename, suffix) 372 _log.debug(' Archive test file name: "%s"', archive_test_name) 373 if not archive_test_name in zip_file.namelist(): 374 _log.info(' %s file not in archive.', suffix) 375 continue 376 377 found = True 378 _log.info(' %s file found in archive.', suffix) 379 380 temp_name = self._extract_from_zip_to_tempfile(zip_file, archive_test_name) 381 382 expected_filename = '%s-expected%s' % (test_basename, suffix) 383 expected_fullpath = self._filesystem.join( 384 self._rebaseline_port.baseline_path(), expected_filename) 385 expected_fullpath = self._filesystem.normpath(expected_fullpath) 386 _log.debug(' Expected file full path: "%s"', expected_fullpath) 387 388 # TODO(victorw): for now, the rebaselining tool checks whether 389 # or not THIS baseline is duplicate and should be skipped. 390 # We could improve the tool to check all baselines in upper 391 # and lower levels and remove all duplicated baselines. 392 if self._is_dup_baseline(temp_name, expected_fullpath, test, suffix, self._platform): 393 self._filesystem.remove(temp_name) 394 self._delete_baseline(expected_fullpath) 395 continue 396 397 if suffix == '.checksum' and self._png_has_same_checksum(temp_name, test, expected_fullpath): 398 self._filesystem.remove(temp_name) 399 # If an old checksum exists, delete it. 400 self._delete_baseline(expected_fullpath) 401 continue 402 403 self._filesystem.maybe_make_directory(self._filesystem.dirname(expected_fullpath)) 404 self._filesystem.move(temp_name, expected_fullpath) 405 406 if self._scm.add(expected_fullpath, return_exit_code=True): 407 # FIXME: print detailed diagnose messages 408 scm_error = True 409 elif suffix != '.checksum': 410 self._create_html_baseline_files(expected_fullpath) 411 412 if not found: 413 _log.warn(' No new baselines found in archive.') 414 elif scm_error: 415 _log.warn(' Failed to add baselines to your repository.') 416 else: 417 _log.info(' Rebaseline succeeded.') 418 self._rebaselined_tests.append(test) 419 420 def _extract_from_zip_to_tempfile(self, zip_file, filename): 421 """Extracts |filename| from |zip_file|, a ZipFileSet. Returns the full 422 path name to the extracted file.""" 423 data = zip_file.read(filename) 424 suffix = self._filesystem.splitext(filename)[1] 425 tempfile, temp_name = self._filesystem.open_binary_tempfile(suffix) 426 tempfile.write(data) 427 tempfile.close() 428 return temp_name 429 430 def _png_has_same_checksum(self, checksum_path, test, checksum_expected_fullpath): 431 """Returns True if the fallback png for |checksum_expected_fullpath| 432 contains the same checksum.""" 433 fs = self._filesystem 434 png_fullpath = self._first_fallback_png_for_test(test) 435 436 if not fs.exists(png_fullpath): 437 _log.error(' Checksum without png file found! Expected %s to exist.' % png_fullpath) 438 return False 439 440 with fs.open_binary_file_for_reading(png_fullpath) as filehandle: 441 checksum_in_png = read_checksum_from_png.read_checksum(filehandle) 442 checksum_in_text_file = fs.read_text_file(checksum_path) 443 if checksum_in_png and checksum_in_png != checksum_in_text_file: 444 _log.error(" checksum in %s and %s don't match! Continuing" 445 " to copy but please investigate." % ( 446 checksum_expected_fullpath, png_fullpath)) 447 return checksum_in_text_file == checksum_in_png 448 449 def _first_fallback_png_for_test(self, test): 450 test_filepath = self._filesystem.join(self._target_port.layout_tests_dir(), test) 451 all_baselines = self._rebaseline_port.expected_baselines( 452 test_filepath, '.png', True) 453 return self._filesystem.join(all_baselines[0][0], all_baselines[0][1]) 454 455 def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix, platform): 456 """Check whether a baseline is duplicate and can fallback to same 457 baseline for another platform. For example, if a test has same 458 baseline on linux and windows, then we only store windows 459 baseline and linux baseline will fallback to the windows version. 460 461 Args: 462 new_baseline: temp filename containing the new baseline results 463 baseline_path: baseline expectation file name. 464 test: test name. 465 suffix: file suffix of the expected results, including dot; 466 e.g. '.txt' or '.png'. 467 platform: baseline platform 'mac', 'win' or 'linux'. 468 469 Returns: 470 True if the baseline is unnecessary. 471 False otherwise. 472 """ 473 test_filepath = self._filesystem.join(self._target_port.layout_tests_dir(), test) 474 all_baselines = self._rebaseline_port.expected_baselines( 475 test_filepath, suffix, True) 476 477 for fallback_dir, fallback_file in all_baselines: 478 if not fallback_dir or not fallback_file: 479 continue 480 481 fallback_fullpath = self._filesystem.normpath( 482 self._filesystem.join(fallback_dir, fallback_file)) 483 if fallback_fullpath.lower() == baseline_path.lower(): 484 continue 485 486 new_output = self._filesystem.read_binary_file(new_baseline) 487 fallback_output = self._filesystem.read_binary_file(fallback_fullpath) 488 is_image = baseline_path.lower().endswith('.png') 489 if not self._diff_baselines(new_output, fallback_output, is_image): 490 _log.info(' Found same baseline at %s', fallback_fullpath) 491 return True 492 return False 493 494 return False 495 496 def _diff_baselines(self, output1, output2, is_image): 497 """Check whether two baselines are different. 498 499 Args: 500 output1, output2: contents of the baselines to compare. 501 502 Returns: 503 True if two files are different or have different extensions. 504 False otherwise. 505 """ 506 507 if is_image: 508 return self._port.diff_image(output1, output2, None) 509 510 return self._port.compare_text(output1, output2) 511 512 def _delete_baseline(self, filename): 513 """Remove the file from repository and delete it from disk. 514 515 Args: 516 filename: full path of the file to delete. 517 """ 518 519 if not filename or not self._filesystem.isfile(filename): 520 return 521 self._scm.delete(filename) 522 523 def _create_html_baseline_files(self, baseline_fullpath): 524 """Create baseline files (old, new and diff) in html directory. 525 526 The files are used to compare the rebaselining results. 527 528 Args: 529 baseline_fullpath: full path of the expected baseline file. 530 """ 531 532 if not baseline_fullpath or not self._filesystem.exists(baseline_fullpath): 533 return 534 535 # Copy the new baseline to html directory for result comparison. 536 baseline_filename = self._filesystem.basename(baseline_fullpath) 537 new_file = get_result_file_fullpath(self._filesystem, self._options.html_directory, 538 baseline_filename, self._platform, 'new') 539 self._filesystem.copyfile(baseline_fullpath, new_file) 540 _log.info(' Html: copied new baseline file from "%s" to "%s".', 541 baseline_fullpath, new_file) 542 543 # Get the old baseline from the repository and save to the html directory. 544 try: 545 output = self._scm.show_head(baseline_fullpath) 546 except ScriptError, e: 547 _log.info(e) 548 output = "" 549 550 if (not output) or (output.upper().rstrip().endswith('NO SUCH FILE OR DIRECTORY')): 551 _log.info(' No base file: "%s"', baseline_fullpath) 552 return 553 base_file = get_result_file_fullpath(self._filesystem, self._options.html_directory, 554 baseline_filename, self._platform, 'old') 555 if base_file.upper().endswith('.PNG'): 556 self._filesystem.write_binary_file(base_file, output) 557 else: 558 self._filesystem.write_text_file(base_file, output) 559 _log.info(' Html: created old baseline file: "%s".', base_file) 560 561 # Get the diff between old and new baselines and save to the html dir. 562 if baseline_filename.upper().endswith('.TXT'): 563 output = self._scm.diff_for_file(baseline_fullpath, log=_log) 564 if output: 565 diff_file = get_result_file_fullpath(self._filesystem, 566 self._options.html_directory, baseline_filename, 567 self._platform, 'diff') 568 self._filesystem.write_text_file(diff_file, output) 569 _log.info(' Html: created baseline diff file: "%s".', diff_file) 570 571 572 class HtmlGenerator(object): 573 """Class to generate rebaselining result comparison html.""" 574 575 HTML_REBASELINE = ('<html>' 576 '<head>' 577 '<style>' 578 'body {font-family: sans-serif;}' 579 '.mainTable {background: #666666;}' 580 '.mainTable td , .mainTable th {background: white;}' 581 '.detail {margin-left: 10px; margin-top: 3px;}' 582 '</style>' 583 '<title>Rebaselining Result Comparison (%(time)s)' 584 '</title>' 585 '</head>' 586 '<body>' 587 '<h2>Rebaselining Result Comparison (%(time)s)</h2>' 588 '%(body)s' 589 '</body>' 590 '</html>') 591 HTML_NO_REBASELINING_TESTS = ( 592 '<p>No tests found that need rebaselining.</p>') 593 HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>' 594 '%s</table><br>') 595 HTML_TR_TEST = ('<tr>' 596 '<th style="background-color: #CDECDE; border-bottom: ' 597 '1px solid black; font-size: 18pt; font-weight: bold" ' 598 'colspan="5">' 599 '<a href="%s">%s</a>' 600 '</th>' 601 '</tr>') 602 HTML_TEST_DETAIL = ('<div class="detail">' 603 '<tr>' 604 '<th width="100">Baseline</th>' 605 '<th width="100">Platform</th>' 606 '<th width="200">Old</th>' 607 '<th width="200">New</th>' 608 '<th width="150">Difference</th>' 609 '</tr>' 610 '%s' 611 '</div>') 612 HTML_TD_NOLINK = '<td align=center><a>%s</a></td>' 613 HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>' 614 HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">' 615 '<img style="width: 200" src="%(uri)s" /></a></td>') 616 HTML_TR = '<tr>%s</tr>' 617 618 def __init__(self, port, target_port, options, platforms, rebaselining_tests): 619 self._html_directory = options.html_directory 620 self._port = port 621 self._target_port = target_port 622 self._options = options 623 self._platforms = platforms 624 self._rebaselining_tests = rebaselining_tests 625 self._filesystem = port._filesystem 626 self._html_file = self._filesystem.join(options.html_directory, 627 'rebaseline.html') 628 629 def abspath_to_uri(self, filename): 630 """Converts an absolute path to a file: URI.""" 631 return path.abspath_to_uri(filename, self._port._executive) 632 633 def generate_html(self): 634 """Generate html file for rebaselining result comparison.""" 635 636 _log.info('Generating html file') 637 638 html_body = '' 639 if not self._rebaselining_tests: 640 html_body += self.HTML_NO_REBASELINING_TESTS 641 else: 642 tests = list(self._rebaselining_tests) 643 tests.sort() 644 645 test_no = 1 646 for test in tests: 647 _log.info('Test %d: %s', test_no, test) 648 html_body += self._generate_html_for_one_test(test) 649 650 html = self.HTML_REBASELINE % ({'time': time.asctime(), 651 'body': html_body}) 652 _log.debug(html) 653 654 self._filesystem.write_text_file(self._html_file, html) 655 _log.info('Baseline comparison html generated at "%s"', self._html_file) 656 657 def show_html(self): 658 """Launch the rebaselining html in brwoser.""" 659 660 _log.info('Launching html: "%s"', self._html_file) 661 self._port._user.open_url(self._html_file) 662 _log.info('Html launched.') 663 664 def _generate_baseline_links(self, test_basename, suffix, platform): 665 """Generate links for baseline results (old, new and diff). 666 667 Args: 668 test_basename: base filename of the test 669 suffix: baseline file suffixes: '.txt', '.png' 670 platform: win, linux or mac 671 672 Returns: 673 html links for showing baseline results (old, new and diff) 674 """ 675 676 baseline_filename = '%s-expected%s' % (test_basename, suffix) 677 _log.debug(' baseline filename: "%s"', baseline_filename) 678 679 new_file = get_result_file_fullpath(self._filesystem, self._html_directory, 680 baseline_filename, platform, 'new') 681 _log.info(' New baseline file: "%s"', new_file) 682 if not self._filesystem.exists(new_file): 683 _log.info(' No new baseline file: "%s"', new_file) 684 return '' 685 686 old_file = get_result_file_fullpath(self._filesystem, self._html_directory, 687 baseline_filename, platform, 'old') 688 _log.info(' Old baseline file: "%s"', old_file) 689 if suffix == '.png': 690 html_td_link = self.HTML_TD_LINK_IMG 691 else: 692 html_td_link = self.HTML_TD_LINK 693 694 links = '' 695 if self._filesystem.exists(old_file): 696 links += html_td_link % { 697 'uri': self.abspath_to_uri(old_file), 698 'name': baseline_filename} 699 else: 700 _log.info(' No old baseline file: "%s"', old_file) 701 links += self.HTML_TD_NOLINK % '' 702 703 links += html_td_link % {'uri': self.abspath_to_uri(new_file), 704 'name': baseline_filename} 705 706 diff_file = get_result_file_fullpath(self._filesystem, self._html_directory, 707 baseline_filename, platform, 'diff') 708 _log.info(' Baseline diff file: "%s"', diff_file) 709 if self._filesystem.exists(diff_file): 710 links += html_td_link % {'uri': self.abspath_to_uri(diff_file), 711 'name': 'Diff'} 712 else: 713 _log.info(' No baseline diff file: "%s"', diff_file) 714 links += self.HTML_TD_NOLINK % '' 715 716 return links 717 718 def _generate_html_for_one_test(self, test): 719 """Generate html for one rebaselining test. 720 721 Args: 722 test: layout test name 723 724 Returns: 725 html that compares baseline results for the test. 726 """ 727 728 test_basename = self._filesystem.basename(self._filesystem.splitext(test)[0]) 729 _log.info(' basename: "%s"', test_basename) 730 rows = [] 731 for suffix in BASELINE_SUFFIXES: 732 if suffix == '.checksum': 733 continue 734 735 _log.info(' Checking %s files', suffix) 736 for platform in self._platforms: 737 links = self._generate_baseline_links(test_basename, suffix, platform) 738 if links: 739 row = self.HTML_TD_NOLINK % self._get_baseline_result_type(suffix) 740 row += self.HTML_TD_NOLINK % platform 741 row += links 742 _log.debug(' html row: %s', row) 743 744 rows.append(self.HTML_TR % row) 745 746 if rows: 747 test_path = self._filesystem.join(self._target_port.layout_tests_dir(), test) 748 html = self.HTML_TR_TEST % (self.abspath_to_uri(test_path), test) 749 html += self.HTML_TEST_DETAIL % ' '.join(rows) 750 751 _log.debug(' html for test: %s', html) 752 return self.HTML_TABLE_TEST % html 753 754 return '' 755 756 def _get_baseline_result_type(self, suffix): 757 """Name of the baseline result type.""" 758 759 if suffix == '.png': 760 return 'Pixel' 761 elif suffix == '.txt': 762 return 'Render Tree' 763 else: 764 return 'Other' 765 766 767 def get_host_port_object(options): 768 """Return a port object for the platform we're running on.""" 769 # The only thing we really need on the host is a way to diff 770 # text files and image files, which means we need to check that some 771 # version of ImageDiff has been built. We will look for either Debug 772 # or Release versions of the default port on the platform. 773 options.configuration = "Release" 774 port_obj = port.get(None, options) 775 if not port_obj.check_image_diff(override_step=None, logging=False): 776 _log.debug('No release version of the image diff binary was found.') 777 options.configuration = "Debug" 778 port_obj = port.get(None, options) 779 if not port_obj.check_image_diff(override_step=None, logging=False): 780 _log.error('No version of image diff was found. Check your build.') 781 return None 782 else: 783 _log.debug('Found the debug version of the image diff binary.') 784 else: 785 _log.debug('Found the release version of the image diff binary.') 786 return port_obj 787 788 789 def parse_options(args): 790 """Parse options and return a pair of host options and target options.""" 791 option_parser = optparse.OptionParser() 792 option_parser.add_option('-v', '--verbose', 793 action='store_true', 794 default=False, 795 help='include debug-level logging.') 796 797 option_parser.add_option('-q', '--quiet', 798 action='store_true', 799 help='Suppress result HTML viewing') 800 801 option_parser.add_option('-p', '--platforms', 802 default=None, 803 help=('Comma delimited list of platforms ' 804 'that need rebaselining.')) 805 806 option_parser.add_option('-u', '--archive_url', 807 default=('http://build.chromium.org/f/chromium/' 808 'layout_test_results'), 809 help=('Url to find the layout test result archive' 810 ' file.')) 811 option_parser.add_option('-U', '--force_archive_url', 812 help=('Url of result zip file. This option is for debugging ' 813 'purposes')) 814 815 option_parser.add_option('-b', '--backup', 816 action='store_true', 817 default=False, 818 help=('Whether or not to backup the original test' 819 ' expectations file after rebaseline.')) 820 821 option_parser.add_option('-d', '--html_directory', 822 default='', 823 help=('The directory that stores the results for ' 824 'rebaselining comparison.')) 825 826 option_parser.add_option('', '--use_drt', 827 action='store_true', 828 default=False, 829 help=('Use ImageDiff from DumpRenderTree instead ' 830 'of image_diff for pixel tests.')) 831 832 option_parser.add_option('-w', '--webkit_canary', 833 action='store_true', 834 default=False, 835 help=('DEPRECATED. This flag no longer has any effect.' 836 ' The canaries are always used.')) 837 838 option_parser.add_option('', '--target-platform', 839 default='chromium', 840 help=('The target platform to rebaseline ' 841 '("mac", "chromium", "qt", etc.). Defaults ' 842 'to "chromium".')) 843 844 options = option_parser.parse_args(args)[0] 845 if options.webkit_canary: 846 print "-w/--webkit-canary is no longer necessary, ignoring." 847 848 target_options = copy.copy(options) 849 if options.target_platform == 'chromium': 850 target_options.chromium = True 851 options.tolerance = 0 852 853 return (options, target_options) 854 855 856 def main(args): 857 """Bootstrap function that sets up the object references we need and calls real_main().""" 858 options, target_options = parse_options(args) 859 860 # Set up our logging format. 861 log_level = logging.INFO 862 if options.verbose: 863 log_level = logging.DEBUG 864 logging.basicConfig(level=log_level, 865 format=('%(asctime)s %(filename)s:%(lineno)-3d ' 866 '%(levelname)s %(message)s'), 867 datefmt='%y%m%d %H:%M:%S') 868 869 target_port_obj = port.get(None, target_options) 870 host_port_obj = get_host_port_object(options) 871 if not host_port_obj or not target_port_obj: 872 return 1 873 874 url_fetcher = urlfetcher.UrlFetcher(host_port_obj._filesystem) 875 scm_obj = scm.default_scm() 876 877 # We use the default zip factory method. 878 zip_factory = None 879 880 return real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, 881 zip_factory, scm_obj) 882 883 884 def real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, 885 zip_factory, scm_obj): 886 """Main function to produce new baselines. The Rebaseliner object uses two 887 different Port objects - one to represent the machine the object is running 888 on, and one to represent the port whose expectations are being updated. 889 E.g., you can run the script on a mac and rebaseline the 'win' port. 890 891 Args: 892 options: command-line argument used for the host_port_obj (see below) 893 target_options: command_line argument used for the target_port_obj. 894 This object may have slightly different values than |options|. 895 host_port_obj: a Port object for the platform the script is running 896 on. This is used to produce image and text diffs, mostly, and 897 is usually acquired from get_host_port_obj(). 898 target_port_obj: a Port obj representing the port getting rebaselined. 899 This is used to find the expectations file, the baseline paths, 900 etc. 901 url_fetcher: object used to download the build archives from the bots 902 zip_factory: factory function used to create zip file objects for 903 the archives. 904 scm_obj: object used to add new baselines to the source control system. 905 """ 906 options.html_directory = setup_html_directory(host_port_obj._filesystem, options.html_directory) 907 all_platforms = target_port_obj.all_baseline_variants() 908 if options.platforms: 909 bail = False 910 for platform in options.platforms: 911 if not platform in all_platforms: 912 _log.error('Invalid platform: "%s"' % (platform)) 913 bail = True 914 if bail: 915 return 1 916 rebaseline_platforms = options.platforms 917 else: 918 rebaseline_platforms = all_platforms 919 920 rebaselined_tests = set() 921 for platform in rebaseline_platforms: 922 rebaseliner = Rebaseliner(host_port_obj, target_port_obj, 923 platform, options, url_fetcher, zip_factory, 924 scm_obj) 925 926 _log.info('') 927 log_dashed_string('Rebaseline started', platform) 928 if rebaseliner.run(): 929 log_dashed_string('Rebaseline done', platform) 930 else: 931 log_dashed_string('Rebaseline failed', platform, logging.ERROR) 932 933 rebaselined_tests |= set(rebaseliner.get_rebaselined_tests()) 934 935 if rebaselined_tests: 936 rebaseliner.remove_rebaselining_expectations(rebaselined_tests, 937 options.backup) 938 939 _log.info('') 940 log_dashed_string('Rebaselining result comparison started', None) 941 html_generator = HtmlGenerator(host_port_obj, 942 target_port_obj, 943 options, 944 rebaseline_platforms, 945 rebaselined_tests) 946 html_generator.generate_html() 947 if not options.quiet: 948 html_generator.show_html() 949 log_dashed_string('Rebaselining result comparison done', None) 950 951 return 0 952 953 954 if '__main__' == __name__: 955 sys.exit(main(sys.argv[1:])) 956
