1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import json 30 import logging 31 import optparse 32 import re 33 import sys 34 import time 35 import urllib2 36 37 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer 38 from webkitpy.common.memoized import memoized 39 from webkitpy.common.system.executive import ScriptError 40 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter 41 from webkitpy.layout_tests.models import test_failures 42 from webkitpy.layout_tests.models.test_expectations import TestExpectations, BASELINE_SUFFIX_LIST 43 from webkitpy.layout_tests.port import builders 44 from webkitpy.layout_tests.port import factory 45 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 46 47 48 _log = logging.getLogger(__name__) 49 50 51 # FIXME: Should TestResultWriter know how to compute this string? 52 def _baseline_name(fs, test_name, suffix): 53 return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix 54 55 56 class AbstractRebaseliningCommand(AbstractDeclarativeCommand): 57 # not overriding execute() - pylint: disable=W0223 58 59 no_optimize_option = optparse.make_option('--no-optimize', dest='optimize', action='store_false', default=True, 60 help=('Do not optimize/de-dup the expectations after rebaselining (default is to de-dup automatically). ' 61 'You can use "webkit-patch optimize-baselines" to optimize separately.')) 62 63 platform_options = factory.platform_options(use_globs=True) 64 65 results_directory_option = optparse.make_option("--results-directory", help="Local results directory to use") 66 67 suffixes_option = optparse.make_option("--suffixes", default=','.join(BASELINE_SUFFIX_LIST), action="store", 68 help="Comma-separated-list of file types to rebaseline") 69 70 def __init__(self, options=None): 71 super(AbstractRebaseliningCommand, self).__init__(options=options) 72 self._baseline_suffix_list = BASELINE_SUFFIX_LIST 73 74 75 class BaseInternalRebaselineCommand(AbstractRebaseliningCommand): 76 def __init__(self): 77 super(BaseInternalRebaselineCommand, self).__init__(options=[ 78 self.results_directory_option, 79 self.suffixes_option, 80 optparse.make_option("--builder", help="Builder to pull new baselines from"), 81 optparse.make_option("--test", help="Test to rebaseline"), 82 ]) 83 self._scm_changes = {'add': [], 'remove-lines': []} 84 85 def _add_to_scm(self, path): 86 self._scm_changes['add'].append(path) 87 88 def _baseline_directory(self, builder_name): 89 port = self._tool.port_factory.get_from_builder_name(builder_name) 90 override_dir = builders.rebaseline_override_dir(builder_name) 91 if override_dir: 92 return self._tool.filesystem.join(port.layout_tests_dir(), 'platform', override_dir) 93 return port.baseline_version_dir() 94 95 def _test_root(self, test_name): 96 return self._tool.filesystem.splitext(test_name)[0] 97 98 def _file_name_for_actual_result(self, test_name, suffix): 99 return "%s-actual.%s" % (self._test_root(test_name), suffix) 100 101 def _file_name_for_expected_result(self, test_name, suffix): 102 return "%s-expected.%s" % (self._test_root(test_name), suffix) 103 104 105 class CopyExistingBaselinesInternal(BaseInternalRebaselineCommand): 106 name = "copy-existing-baselines-internal" 107 help_text = "Copy existing baselines down one level in the baseline order to ensure new baselines don't break existing passing platforms." 108 109 @memoized 110 def _immediate_predecessors_in_fallback(self, path_to_rebaseline): 111 port_names = self._tool.port_factory.all_port_names() 112 immediate_predecessors_in_fallback = [] 113 for port_name in port_names: 114 port = self._tool.port_factory.get(port_name) 115 if not port.buildbot_archives_baselines(): 116 continue 117 baseline_search_path = port.baseline_search_path() 118 try: 119 index = baseline_search_path.index(path_to_rebaseline) 120 if index: 121 immediate_predecessors_in_fallback.append(self._tool.filesystem.basename(baseline_search_path[index - 1])) 122 except ValueError: 123 # index throw's a ValueError if the item isn't in the list. 124 pass 125 return immediate_predecessors_in_fallback 126 127 def _port_for_primary_baseline(self, baseline): 128 for port in [self._tool.port_factory.get(port_name) for port_name in self._tool.port_factory.all_port_names()]: 129 if self._tool.filesystem.basename(port.baseline_version_dir()) == baseline: 130 return port 131 raise Exception("Failed to find port for primary baseline %s." % baseline) 132 133 def _copy_existing_baseline(self, builder_name, test_name, suffix): 134 baseline_directory = self._baseline_directory(builder_name) 135 ports = [self._port_for_primary_baseline(baseline) for baseline in self._immediate_predecessors_in_fallback(baseline_directory)] 136 137 old_baselines = [] 138 new_baselines = [] 139 140 # Need to gather all the baseline paths before modifying the filesystem since 141 # the modifications can affect the results of port.expected_filename. 142 for port in ports: 143 old_baseline = port.expected_filename(test_name, "." + suffix) 144 if not self._tool.filesystem.exists(old_baseline): 145 _log.debug("No existing baseline for %s." % test_name) 146 continue 147 148 new_baseline = self._tool.filesystem.join(port.baseline_path(), self._file_name_for_expected_result(test_name, suffix)) 149 if self._tool.filesystem.exists(new_baseline): 150 _log.debug("Existing baseline at %s, not copying over it." % new_baseline) 151 continue 152 153 old_baselines.append(old_baseline) 154 new_baselines.append(new_baseline) 155 156 for i in range(len(old_baselines)): 157 old_baseline = old_baselines[i] 158 new_baseline = new_baselines[i] 159 160 _log.debug("Copying baseline from %s to %s." % (old_baseline, new_baseline)) 161 self._tool.filesystem.maybe_make_directory(self._tool.filesystem.dirname(new_baseline)) 162 self._tool.filesystem.copyfile(old_baseline, new_baseline) 163 if not self._tool.scm().exists(new_baseline): 164 self._add_to_scm(new_baseline) 165 166 def execute(self, options, args, tool): 167 for suffix in options.suffixes.split(','): 168 self._copy_existing_baseline(options.builder, options.test, suffix) 169 print json.dumps(self._scm_changes) 170 171 172 class RebaselineTest(BaseInternalRebaselineCommand): 173 name = "rebaseline-test-internal" 174 help_text = "Rebaseline a single test from a buildbot. Only intended for use by other webkit-patch commands." 175 176 def _results_url(self, builder_name): 177 return self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name).latest_layout_test_results_url() 178 179 def _save_baseline(self, data, target_baseline, baseline_directory, test_name, suffix): 180 if not data: 181 _log.debug("No baseline data to save.") 182 return 183 184 filesystem = self._tool.filesystem 185 filesystem.maybe_make_directory(filesystem.dirname(target_baseline)) 186 filesystem.write_binary_file(target_baseline, data) 187 if not self._tool.scm().exists(target_baseline): 188 self._add_to_scm(target_baseline) 189 190 def _rebaseline_test(self, builder_name, test_name, suffix, results_url): 191 baseline_directory = self._baseline_directory(builder_name) 192 193 source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix)) 194 target_baseline = self._tool.filesystem.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix)) 195 196 _log.debug("Retrieving %s." % source_baseline) 197 self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline, baseline_directory, test_name, suffix) 198 199 def _rebaseline_test_and_update_expectations(self, options): 200 port = self._tool.port_factory.get_from_builder_name(options.builder) 201 if (port.reference_files(options.test)): 202 _log.warning("Cannot rebaseline reftest: %s", options.test) 203 return 204 205 if options.results_directory: 206 results_url = 'file://' + options.results_directory 207 else: 208 results_url = self._results_url(options.builder) 209 self._baseline_suffix_list = options.suffixes.split(',') 210 211 for suffix in self._baseline_suffix_list: 212 self._rebaseline_test(options.builder, options.test, suffix, results_url) 213 self._scm_changes['remove-lines'].append({'builder': options.builder, 'test': options.test}) 214 215 def execute(self, options, args, tool): 216 self._rebaseline_test_and_update_expectations(options) 217 print json.dumps(self._scm_changes) 218 219 220 class OptimizeBaselines(AbstractRebaseliningCommand): 221 name = "optimize-baselines" 222 help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible." 223 argument_names = "TEST_NAMES" 224 225 def __init__(self): 226 super(OptimizeBaselines, self).__init__(options=[self.suffixes_option] + self.platform_options) 227 228 def _optimize_baseline(self, optimizer, test_name): 229 for suffix in self._baseline_suffix_list: 230 baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix) 231 if not optimizer.optimize(baseline_name): 232 print "Heuristics failed to optimize %s" % baseline_name 233 234 def execute(self, options, args, tool): 235 self._baseline_suffix_list = options.suffixes.split(',') 236 port_names = tool.port_factory.all_port_names(options.platform) 237 if not port_names: 238 print "No port names match '%s'" % options.platform 239 return 240 241 optimizer = BaselineOptimizer(tool, port_names) 242 port = tool.port_factory.get(port_names[0]) 243 for test_name in port.tests(args): 244 _log.info("Optimizing %s" % test_name) 245 self._optimize_baseline(optimizer, test_name) 246 247 248 class AnalyzeBaselines(AbstractRebaseliningCommand): 249 name = "analyze-baselines" 250 help_text = "Analyzes the baselines for the given tests and prints results that are identical." 251 argument_names = "TEST_NAMES" 252 253 def __init__(self): 254 super(AnalyzeBaselines, self).__init__(options=[ 255 self.suffixes_option, 256 optparse.make_option('--missing', action='store_true', default=False, help='show missing baselines as well'), 257 ] + self.platform_options) 258 self._optimizer_class = BaselineOptimizer # overridable for testing 259 self._baseline_optimizer = None 260 self._port = None 261 262 def _write(self, msg): 263 print msg 264 265 def _analyze_baseline(self, options, test_name): 266 for suffix in self._baseline_suffix_list: 267 baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix) 268 results_by_directory = self._baseline_optimizer.read_results_by_directory(baseline_name) 269 if results_by_directory: 270 self._write("%s:" % baseline_name) 271 self._baseline_optimizer.write_by_directory(results_by_directory, self._write, " ") 272 elif options.missing: 273 self._write("%s: (no baselines found)" % baseline_name) 274 275 def execute(self, options, args, tool): 276 self._baseline_suffix_list = options.suffixes.split(',') 277 port_names = tool.port_factory.all_port_names(options.platform) 278 if not port_names: 279 print "No port names match '%s'" % options.platform 280 return 281 282 self._baseline_optimizer = self._optimizer_class(tool, port_names) 283 self._port = tool.port_factory.get(port_names[0]) 284 for test_name in self._port.tests(args): 285 self._analyze_baseline(options, test_name) 286 287 288 class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand): 289 # not overriding execute() - pylint: disable=W0223 290 291 def __init__(self, options=None): 292 super(AbstractParallelRebaselineCommand, self).__init__(options=options) 293 self._builder_data = {} 294 295 def builder_data(self): 296 if not self._builder_data: 297 for builder_name in self._release_builders(): 298 builder = self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name) 299 self._builder_data[builder_name] = builder.latest_layout_test_results() 300 return self._builder_data 301 302 # The release builders cycle much faster than the debug ones and cover all the platforms. 303 def _release_builders(self): 304 release_builders = [] 305 for builder_name in builders.all_builder_names(): 306 port = self._tool.port_factory.get_from_builder_name(builder_name) 307 if port.test_configuration().build_type == 'release': 308 release_builders.append(builder_name) 309 return release_builders 310 311 def _run_webkit_patch(self, args, verbose): 312 try: 313 verbose_args = ['--verbose'] if verbose else [] 314 stderr = self._tool.executive.run_command([self._tool.path()] + verbose_args + args, cwd=self._tool.scm().checkout_root, return_stderr=True) 315 for line in stderr.splitlines(): 316 _log.warning(line) 317 except ScriptError, e: 318 _log.error(e) 319 320 def _builders_to_fetch_from(self, builders_to_check): 321 # This routine returns the subset of builders that will cover all of the baseline search paths 322 # used in the input list. In particular, if the input list contains both Release and Debug 323 # versions of a configuration, we *only* return the Release version (since we don't save 324 # debug versions of baselines). 325 release_builders = set() 326 debug_builders = set() 327 builders_to_fallback_paths = {} 328 for builder in builders_to_check: 329 port = self._tool.port_factory.get_from_builder_name(builder) 330 if port.test_configuration().build_type == 'release': 331 release_builders.add(builder) 332 else: 333 debug_builders.add(builder) 334 for builder in list(release_builders) + list(debug_builders): 335 port = self._tool.port_factory.get_from_builder_name(builder) 336 fallback_path = port.baseline_search_path() 337 if fallback_path not in builders_to_fallback_paths.values(): 338 builders_to_fallback_paths[builder] = fallback_path 339 return builders_to_fallback_paths.keys() 340 341 def _rebaseline_commands(self, test_prefix_list, options): 342 path_to_webkit_patch = self._tool.path() 343 cwd = self._tool.scm().checkout_root 344 copy_baseline_commands = [] 345 rebaseline_commands = [] 346 port = self._tool.port_factory.get() 347 348 for test_prefix in test_prefix_list: 349 for test in port.tests([test_prefix]): 350 for builder in self._builders_to_fetch_from(test_prefix_list[test_prefix]): 351 actual_failures_suffixes = self._suffixes_for_actual_failures(test, builder, test_prefix_list[test_prefix][builder]) 352 if not actual_failures_suffixes: 353 continue 354 355 suffixes = ','.join(actual_failures_suffixes) 356 cmd_line = ['--suffixes', suffixes, '--builder', builder, '--test', test] 357 if options.results_directory: 358 cmd_line.extend(['--results-directory', options.results_directory]) 359 if options.verbose: 360 cmd_line.append('--verbose') 361 copy_baseline_commands.append(tuple([[path_to_webkit_patch, 'copy-existing-baselines-internal'] + cmd_line, cwd])) 362 rebaseline_commands.append(tuple([[path_to_webkit_patch, 'rebaseline-test-internal'] + cmd_line, cwd])) 363 return copy_baseline_commands, rebaseline_commands 364 365 def _files_to_add(self, command_results): 366 files_to_add = set() 367 lines_to_remove = {} 368 for output in [result[1].split('\n') for result in command_results]: 369 file_added = False 370 for line in output: 371 try: 372 if line: 373 parsed_line = json.loads(line) 374 if 'add' in parsed_line: 375 files_to_add.update(parsed_line['add']) 376 if 'remove-lines' in parsed_line: 377 for line_to_remove in parsed_line['remove-lines']: 378 test = line_to_remove['test'] 379 builder = line_to_remove['builder'] 380 if test not in lines_to_remove: 381 lines_to_remove[test] = [] 382 lines_to_remove[test].append(builder) 383 file_added = True 384 except ValueError: 385 _log.debug('"%s" is not a JSON object, ignoring' % line) 386 387 if not file_added: 388 _log.debug('Could not add file based off output "%s"' % output) 389 390 return list(files_to_add), lines_to_remove 391 392 def _optimize_baselines(self, test_prefix_list, verbose=False): 393 # We don't run this in parallel because modifying the SCM in parallel is unreliable. 394 for test in test_prefix_list: 395 all_suffixes = set() 396 for builder in self._builders_to_fetch_from(test_prefix_list[test]): 397 all_suffixes.update(self._suffixes_for_actual_failures(test, builder, test_prefix_list[test][builder])) 398 # FIXME: We should propagate the platform options as well. 399 self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test], verbose) 400 401 def _update_expectations_files(self, lines_to_remove): 402 for test in lines_to_remove: 403 for builder in lines_to_remove[test]: 404 port = self._tool.port_factory.get_from_builder_name(builder) 405 path = port.path_to_generic_test_expectations_file() 406 expectations = TestExpectations(port, include_overrides=False) 407 for test_configuration in port.all_test_configurations(): 408 if test_configuration.version == port.test_configuration().version: 409 expectationsString = expectations.remove_configuration_from_test(test, test_configuration) 410 self._tool.filesystem.write_text_file(path, expectationsString) 411 412 def _run_in_parallel_and_update_scm(self, commands): 413 command_results = self._tool.executive.run_in_parallel(commands) 414 log_output = '\n'.join(result[2] for result in command_results).replace('\n\n', '\n') 415 for line in log_output.split('\n'): 416 if line: 417 print >> sys.stderr, line # FIXME: Figure out how to log properly. 418 419 files_to_add, lines_to_remove = self._files_to_add(command_results) 420 if files_to_add: 421 self._tool.scm().add_list(list(files_to_add)) 422 if lines_to_remove: 423 self._update_expectations_files(lines_to_remove) 424 425 def _rebaseline(self, options, test_prefix_list): 426 for test, builders_to_check in sorted(test_prefix_list.items()): 427 _log.info("Rebaselining %s" % test) 428 for builder, suffixes in sorted(builders_to_check.items()): 429 _log.debug(" %s: %s" % (builder, ",".join(suffixes))) 430 431 copy_baseline_commands, rebaseline_commands = self._rebaseline_commands(test_prefix_list, options) 432 self._run_in_parallel_and_update_scm(copy_baseline_commands) 433 self._run_in_parallel_and_update_scm(rebaseline_commands) 434 435 if options.optimize: 436 self._optimize_baselines(test_prefix_list, options.verbose) 437 438 def _suffixes_for_actual_failures(self, test, builder_name, existing_suffixes): 439 actual_results = self.builder_data()[builder_name].actual_results(test) 440 if not actual_results: 441 return set() 442 return set(existing_suffixes) & TestExpectations.suffixes_for_actual_expectations_string(actual_results) 443 444 445 class RebaselineJson(AbstractParallelRebaselineCommand): 446 name = "rebaseline-json" 447 help_text = "Rebaseline based off JSON passed to stdin. Intended to only be called from other scripts." 448 449 def __init__(self,): 450 super(RebaselineJson, self).__init__(options=[ 451 self.no_optimize_option, 452 self.results_directory_option, 453 ]) 454 455 def execute(self, options, args, tool): 456 self._rebaseline(options, json.loads(sys.stdin.read())) 457 458 459 class RebaselineExpectations(AbstractParallelRebaselineCommand): 460 name = "rebaseline-expectations" 461 help_text = "Rebaselines the tests indicated in TestExpectations." 462 463 def __init__(self): 464 super(RebaselineExpectations, self).__init__(options=[ 465 self.no_optimize_option, 466 ] + self.platform_options) 467 self._test_prefix_list = None 468 469 def _tests_to_rebaseline(self, port): 470 tests_to_rebaseline = {} 471 for path, value in port.expectations_dict().items(): 472 expectations = TestExpectations(port, include_overrides=False, expectations_dict={path: value}) 473 for test in expectations.get_rebaselining_failures(): 474 suffixes = TestExpectations.suffixes_for_expectations(expectations.get_expectations(test)) 475 tests_to_rebaseline[test] = suffixes or BASELINE_SUFFIX_LIST 476 return tests_to_rebaseline 477 478 def _add_tests_to_rebaseline_for_port(self, port_name): 479 builder_name = builders.builder_name_for_port_name(port_name) 480 if not builder_name: 481 return 482 tests = self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).items() 483 484 if tests: 485 _log.info("Retrieving results for %s from %s." % (port_name, builder_name)) 486 487 for test_name, suffixes in tests: 488 _log.info(" %s (%s)" % (test_name, ','.join(suffixes))) 489 if test_name not in self._test_prefix_list: 490 self._test_prefix_list[test_name] = {} 491 self._test_prefix_list[test_name][builder_name] = suffixes 492 493 def execute(self, options, args, tool): 494 options.results_directory = None 495 self._test_prefix_list = {} 496 port_names = tool.port_factory.all_port_names(options.platform) 497 for port_name in port_names: 498 self._add_tests_to_rebaseline_for_port(port_name) 499 if not self._test_prefix_list: 500 _log.warning("Did not find any tests marked Rebaseline.") 501 return 502 503 self._rebaseline(options, self._test_prefix_list) 504 505 506 class Rebaseline(AbstractParallelRebaselineCommand): 507 name = "rebaseline" 508 help_text = "Rebaseline tests with results from the build bots. Shows the list of failing tests on the builders if no test names are provided." 509 argument_names = "[TEST_NAMES]" 510 511 def __init__(self): 512 super(Rebaseline, self).__init__(options=[ 513 self.no_optimize_option, 514 # FIXME: should we support the platform options in addition to (or instead of) --builders? 515 self.suffixes_option, 516 self.results_directory_option, 517 optparse.make_option("--builders", default=None, action="append", help="Comma-separated-list of builders to pull new baselines from (can also be provided multiple times)"), 518 ]) 519 520 def _builders_to_pull_from(self): 521 chosen_names = self._tool.user.prompt_with_list("Which builder to pull results from:", self._release_builders(), can_choose_multiple=True) 522 return [self._builder_with_name(name) for name in chosen_names] 523 524 def _builder_with_name(self, name): 525 return self._tool.buildbot_for_builder_name(name).builder_with_name(name) 526 527 def execute(self, options, args, tool): 528 if not args: 529 _log.error("Must list tests to rebaseline.") 530 return 531 532 if options.builders: 533 builders_to_check = [] 534 for builder_names in options.builders: 535 builders_to_check += [self._builder_with_name(name) for name in builder_names.split(",")] 536 else: 537 builders_to_check = self._builders_to_pull_from() 538 539 test_prefix_list = {} 540 suffixes_to_update = options.suffixes.split(",") 541 542 for builder in builders_to_check: 543 for test in args: 544 if test not in test_prefix_list: 545 test_prefix_list[test] = {} 546 test_prefix_list[test][builder.name()] = suffixes_to_update 547 548 if options.verbose: 549 _log.debug("rebaseline-json: " + str(test_prefix_list)) 550 551 self._rebaseline(options, test_prefix_list) 552 553 554 class AutoRebaseline(AbstractParallelRebaselineCommand): 555 name = "auto-rebaseline" 556 help_text = "Rebaselines any NeedsRebaseline lines in TestExpectations that have cycled through all the bots." 557 AUTO_REBASELINE_BRANCH_NAME = "auto-rebaseline-temporary-branch" 558 559 # Rietveld uploader stinks. Limit the number of rebaselines in a given patch to keep upload from failing. 560 # FIXME: http://crbug.com/263676 Obviously we should fix the uploader here. 561 MAX_LINES_TO_REBASELINE = 200 562 563 def __init__(self): 564 super(AutoRebaseline, self).__init__(options=[ 565 # FIXME: Remove this option. 566 self.no_optimize_option, 567 # FIXME: Remove this option. 568 self.results_directory_option, 569 ]) 570 571 def latest_revision_processed_on_all_bots(self): 572 revisions = [] 573 for result in self.builder_data().values(): 574 if result.run_was_interrupted(): 575 _log.error("Can't rebaseline. The latest run on %s did not complete." % builder_name) 576 return 0 577 revisions.append(result.blink_revision()) 578 return int(min(revisions)) 579 580 def tests_to_rebaseline(self, tool, min_revision, print_revisions): 581 port = tool.port_factory.get() 582 expectations_file_path = port.path_to_generic_test_expectations_file() 583 584 tests = set() 585 revision = None 586 author = None 587 bugs = set() 588 589 for line in tool.scm().blame(expectations_file_path).split("\n"): 590 if "NeedsRebaseline" not in line: 591 continue 592 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", line) 593 594 commit_hash = parsed_line.group(1) 595 svn_revision = tool.scm().svn_revision_from_git_commit(commit_hash) 596 597 test = parsed_line.group(3) 598 if print_revisions: 599 _log.info("%s is waiting for r%s" % (test, svn_revision)) 600 601 if not svn_revision or svn_revision > min_revision: 602 continue 603 604 if revision and svn_revision != revision: 605 continue 606 607 if not revision: 608 revision = svn_revision 609 author = parsed_line.group(2) 610 611 bugs.update(re.findall("crbug\.com\/(\d+)", line)) 612 tests.add(test) 613 614 if len(tests) >= self.MAX_LINES_TO_REBASELINE: 615 _log.info("Too many tests to rebaseline in one patch. Doing the first %d." % self.MAX_LINES_TO_REBASELINE) 616 break 617 618 return tests, revision, author, bugs 619 620 def link_to_patch(self, revision): 621 return "http://src.chromium.org/viewvc/blink?view=revision&revision=" + str(revision) 622 623 def commit_message(self, author, revision, bugs): 624 bug_string = "" 625 if bugs: 626 bug_string = "BUG=%s\n" % ",".join(bugs) 627 628 return """Auto-rebaseline for r%s 629 630 %s 631 632 %sTBR=%s 633 """ % (revision, self.link_to_patch(revision), bug_string, author) 634 635 def get_test_prefix_list(self, tests): 636 test_prefix_list = {} 637 lines_to_remove = {} 638 639 for builder_name in self._release_builders(): 640 port_name = builders.port_name_for_builder_name(builder_name) 641 port = self._tool.port_factory.get(port_name) 642 expectations = TestExpectations(port, include_overrides=True) 643 for test in expectations.get_needs_rebaseline_failures(): 644 if test not in tests: 645 continue 646 647 if test not in test_prefix_list: 648 lines_to_remove[test] = [] 649 test_prefix_list[test] = {} 650 lines_to_remove[test].append(builder_name) 651 test_prefix_list[test][builder_name] = BASELINE_SUFFIX_LIST 652 653 return test_prefix_list, lines_to_remove 654 655 def _run_git_cl_command(self, options, command): 656 subprocess_command = ['git', 'cl'] + command 657 if options.verbose: 658 subprocess_command.append('--verbose') 659 # Use call instead of run_command so that stdout doesn't get swallowed. 660 self._tool.executive.call(subprocess_command) 661 662 # FIXME: Move this somewhere more general. 663 def tree_status(self): 664 blink_tree_status_url = "http://blink-status.appspot.com/status" 665 status = urllib2.urlopen(blink_tree_status_url).read().lower() 666 if status.find('closed') != -1 or status == 0: 667 return 'closed' 668 elif status.find('open') != -1 or status == 1: 669 return 'open' 670 return 'unknown' 671 672 def execute(self, options, args, tool): 673 if tool.scm().executable_name == "svn": 674 _log.error("Auto rebaseline only works with a git checkout.") 675 return 676 677 if tool.scm().has_working_directory_changes(): 678 _log.error("Cannot proceed with working directory changes. Clean working directory first.") 679 return 680 681 min_revision = self.latest_revision_processed_on_all_bots() 682 if not min_revision: 683 return 684 685 if options.verbose: 686 _log.info("Bot min revision is %s." % min_revision) 687 688 tests, revision, author, bugs = self.tests_to_rebaseline(tool, min_revision, print_revisions=options.verbose) 689 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) 690 691 if not tests: 692 _log.debug('No tests to rebaseline.') 693 return 694 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, author)) 695 696 if self.tree_status() == 'closed': 697 _log.info('Cannot proceed. Tree is closed.') 698 return 699 700 try: 701 old_branch_name = tool.scm().current_branch() 702 tool.scm().delete_branch(self.AUTO_REBASELINE_BRANCH_NAME) 703 tool.scm().create_clean_branch(self.AUTO_REBASELINE_BRANCH_NAME) 704 705 # If the tests are passing everywhere, then this list will be empty. We don't need 706 # to rebaseline, but we'll still need to update TestExpectations. 707 if test_prefix_list: 708 self._rebaseline(options, test_prefix_list) 709 # If a test is not failing on the bot, we don't try to rebaseline it, but we still 710 # want to remove the NeedsRebaseline line. 711 self._update_expectations_files(lines_to_remove) 712 713 tool.scm().commit_locally_with_message(self.commit_message(author, revision, bugs)) 714 715 # FIXME: It would be nice if we could dcommit the patch without uploading, but still 716 # go through all the precommit hooks. For rebaselines with lots of files, uploading 717 # takes a long time and sometimes fails, but we don't want to commit if, e.g. the 718 # tree is closed. 719 self._run_git_cl_command(options, ['upload', '-f']) 720 721 # Uploading can take a very long time. Do another pull to make sure TestExpectations is up to date, 722 # so the dcommit can go through. 723 tool.executive.run_command(['git', 'pull']) 724 725 self._run_git_cl_command(options, ['dcommit', '-f']) 726 finally: 727 self._run_git_cl_command(options, ['set_close']) 728 tool.scm().ensure_cleanly_tracking_remote_master() 729 tool.scm().checkout_branch(old_branch_name) 730 tool.scm().delete_branch(self.AUTO_REBASELINE_BRANCH_NAME) 731 732 733 class RebaselineOMatic(AbstractDeclarativeCommand): 734 name = "rebaseline-o-matic" 735 help_text = "Calls webkit-patch auto-rebaseline in a loop." 736 737 SLEEP_TIME_IN_SECONDS = 30 738 739 def execute(self, options, args, tool): 740 while True: 741 tool.executive.run_command(['git', 'pull']) 742 rebaseline_command = [tool.filesystem.join(tool.scm().checkout_root, 'Tools', 'Scripts', 'webkit-patch'), 'auto-rebaseline'] 743 if options.verbose: 744 rebaseline_command.append('--verbose') 745 # Use call instead of run_command so that stdout doesn't get swallowed. 746 tool.executive.call(rebaseline_command) 747 time.sleep(self.SLEEP_TIME_IN_SECONDS) 748
