1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR/ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import Queue 30 import json 31 import logging 32 import optparse 33 import re 34 import sys 35 import threading 36 import time 37 import traceback 38 import urllib 39 import urllib2 40 41 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer 42 from webkitpy.common.memoized import memoized 43 from webkitpy.common.system.executive import ScriptError 44 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter 45 from webkitpy.layout_tests.models import test_failures 46 from webkitpy.layout_tests.models.test_expectations import TestExpectations, BASELINE_SUFFIX_LIST, SKIP 47 from webkitpy.layout_tests.port import builders 48 from webkitpy.layout_tests.port import factory 49 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 50 51 52 _log = logging.getLogger(__name__) 53 54 55 # FIXME: Should TestResultWriter know how to compute this string? 56 def _baseline_name(fs, test_name, suffix): 57 return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix 58 59 60 class AbstractRebaseliningCommand(AbstractDeclarativeCommand): 61 # not overriding execute() - pylint: disable=W0223 62 63 no_optimize_option = optparse.make_option('--no-optimize', dest='optimize', action='store_false', default=True, 64 help=('Do not optimize/de-dup the expectations after rebaselining (default is to de-dup automatically). ' 65 'You can use "webkit-patch optimize-baselines" to optimize separately.')) 66 67 platform_options = factory.platform_options(use_globs=True) 68 69 results_directory_option = optparse.make_option("--results-directory", help="Local results directory to use") 70 71 suffixes_option = optparse.make_option("--suffixes", default=','.join(BASELINE_SUFFIX_LIST), action="store", 72 help="Comma-separated-list of file types to rebaseline") 73 74 def __init__(self, options=None): 75 super(AbstractRebaseliningCommand, self).__init__(options=options) 76 self._baseline_suffix_list = BASELINE_SUFFIX_LIST 77 self._scm_changes = {'add': [], 'delete': [], 'remove-lines': []} 78 79 def _add_to_scm_later(self, path): 80 self._scm_changes['add'].append(path) 81 82 def _delete_from_scm_later(self, path): 83 self._scm_changes['delete'].append(path) 84 85 86 class BaseInternalRebaselineCommand(AbstractRebaseliningCommand): 87 def __init__(self): 88 super(BaseInternalRebaselineCommand, self).__init__(options=[ 89 self.results_directory_option, 90 self.suffixes_option, 91 optparse.make_option("--builder", help="Builder to pull new baselines from"), 92 optparse.make_option("--test", help="Test to rebaseline"), 93 ]) 94 95 def _baseline_directory(self, builder_name): 96 port = self._tool.port_factory.get_from_builder_name(builder_name) 97 override_dir = builders.rebaseline_override_dir(builder_name) 98 if override_dir: 99 return self._tool.filesystem.join(port.layout_tests_dir(), 'platform', override_dir) 100 return port.baseline_version_dir() 101 102 def _test_root(self, test_name): 103 return self._tool.filesystem.splitext(test_name)[0] 104 105 def _file_name_for_actual_result(self, test_name, suffix): 106 return "%s-actual.%s" % (self._test_root(test_name), suffix) 107 108 def _file_name_for_expected_result(self, test_name, suffix): 109 return "%s-expected.%s" % (self._test_root(test_name), suffix) 110 111 112 class CopyExistingBaselinesInternal(BaseInternalRebaselineCommand): 113 name = "copy-existing-baselines-internal" 114 help_text = "Copy existing baselines down one level in the baseline order to ensure new baselines don't break existing passing platforms." 115 116 @memoized 117 def _immediate_predecessors_in_fallback(self, path_to_rebaseline): 118 port_names = self._tool.port_factory.all_port_names() 119 immediate_predecessors_in_fallback = [] 120 for port_name in port_names: 121 port = self._tool.port_factory.get(port_name) 122 if not port.buildbot_archives_baselines(): 123 continue 124 baseline_search_path = port.baseline_search_path() 125 try: 126 index = baseline_search_path.index(path_to_rebaseline) 127 if index: 128 immediate_predecessors_in_fallback.append(self._tool.filesystem.basename(baseline_search_path[index - 1])) 129 except ValueError: 130 # index throw's a ValueError if the item isn't in the list. 131 pass 132 return immediate_predecessors_in_fallback 133 134 def _port_for_primary_baseline(self, baseline): 135 for port in [self._tool.port_factory.get(port_name) for port_name in self._tool.port_factory.all_port_names()]: 136 if self._tool.filesystem.basename(port.baseline_version_dir()) == baseline: 137 return port 138 raise Exception("Failed to find port for primary baseline %s." % baseline) 139 140 def _copy_existing_baseline(self, builder_name, test_name, suffix): 141 baseline_directory = self._baseline_directory(builder_name) 142 ports = [self._port_for_primary_baseline(baseline) for baseline in self._immediate_predecessors_in_fallback(baseline_directory)] 143 144 old_baselines = [] 145 new_baselines = [] 146 147 # Need to gather all the baseline paths before modifying the filesystem since 148 # the modifications can affect the results of port.expected_filename. 149 for port in ports: 150 old_baseline = port.expected_filename(test_name, "." + suffix) 151 if not self._tool.filesystem.exists(old_baseline): 152 _log.debug("No existing baseline for %s." % test_name) 153 continue 154 155 new_baseline = self._tool.filesystem.join(port.baseline_path(), self._file_name_for_expected_result(test_name, suffix)) 156 if self._tool.filesystem.exists(new_baseline): 157 _log.debug("Existing baseline at %s, not copying over it." % new_baseline) 158 continue 159 160 expectations = TestExpectations(port, [test_name]) 161 if SKIP in expectations.get_expectations(test_name): 162 _log.debug("%s is skipped on %s." % (test_name, port.name())) 163 continue 164 165 old_baselines.append(old_baseline) 166 new_baselines.append(new_baseline) 167 168 for i in range(len(old_baselines)): 169 old_baseline = old_baselines[i] 170 new_baseline = new_baselines[i] 171 172 _log.debug("Copying baseline from %s to %s." % (old_baseline, new_baseline)) 173 self._tool.filesystem.maybe_make_directory(self._tool.filesystem.dirname(new_baseline)) 174 self._tool.filesystem.copyfile(old_baseline, new_baseline) 175 if not self._tool.scm().exists(new_baseline): 176 self._add_to_scm_later(new_baseline) 177 178 def execute(self, options, args, tool): 179 for suffix in options.suffixes.split(','): 180 self._copy_existing_baseline(options.builder, options.test, suffix) 181 print json.dumps(self._scm_changes) 182 183 184 class RebaselineTest(BaseInternalRebaselineCommand): 185 name = "rebaseline-test-internal" 186 help_text = "Rebaseline a single test from a buildbot. Only intended for use by other webkit-patch commands." 187 188 def _results_url(self, builder_name): 189 return self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name).latest_layout_test_results_url() 190 191 def _save_baseline(self, data, target_baseline, baseline_directory, test_name, suffix): 192 if not data: 193 _log.debug("No baseline data to save.") 194 return 195 196 filesystem = self._tool.filesystem 197 filesystem.maybe_make_directory(filesystem.dirname(target_baseline)) 198 filesystem.write_binary_file(target_baseline, data) 199 if not self._tool.scm().exists(target_baseline): 200 self._add_to_scm_later(target_baseline) 201 202 def _rebaseline_test(self, builder_name, test_name, suffix, results_url): 203 baseline_directory = self._baseline_directory(builder_name) 204 205 source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix)) 206 target_baseline = self._tool.filesystem.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix)) 207 208 _log.debug("Retrieving %s." % source_baseline) 209 self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline, baseline_directory, test_name, suffix) 210 211 def _rebaseline_test_and_update_expectations(self, options): 212 port = self._tool.port_factory.get_from_builder_name(options.builder) 213 if (port.reference_files(options.test)): 214 _log.warning("Cannot rebaseline reftest: %s", options.test) 215 return 216 217 if options.results_directory: 218 results_url = 'file://' + options.results_directory 219 else: 220 results_url = self._results_url(options.builder) 221 self._baseline_suffix_list = options.suffixes.split(',') 222 223 for suffix in self._baseline_suffix_list: 224 self._rebaseline_test(options.builder, options.test, suffix, results_url) 225 self._scm_changes['remove-lines'].append({'builder': options.builder, 'test': options.test}) 226 227 def execute(self, options, args, tool): 228 self._rebaseline_test_and_update_expectations(options) 229 print json.dumps(self._scm_changes) 230 231 232 class OptimizeBaselines(AbstractRebaseliningCommand): 233 name = "optimize-baselines" 234 help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible." 235 show_in_main_help = True 236 argument_names = "TEST_NAMES" 237 238 def __init__(self): 239 super(OptimizeBaselines, self).__init__(options=[ 240 self.suffixes_option, 241 optparse.make_option('--no-modify-scm', action='store_true', default=False, help='Dump SCM commands as JSON instead of '), 242 ] + self.platform_options) 243 244 def _optimize_baseline(self, optimizer, test_name): 245 files_to_delete = [] 246 files_to_add = [] 247 for suffix in self._baseline_suffix_list: 248 baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix) 249 succeeded, more_files_to_delete, more_files_to_add = optimizer.optimize(baseline_name) 250 if not succeeded: 251 print "Heuristics failed to optimize %s" % baseline_name 252 files_to_delete.extend(more_files_to_delete) 253 files_to_add.extend(more_files_to_add) 254 return files_to_delete, files_to_add 255 256 def execute(self, options, args, tool): 257 self._baseline_suffix_list = options.suffixes.split(',') 258 port_names = tool.port_factory.all_port_names(options.platform) 259 if not port_names: 260 print "No port names match '%s'" % options.platform 261 return 262 port = tool.port_factory.get(port_names[0]) 263 optimizer = BaselineOptimizer(tool, port, port_names, skip_scm_commands=options.no_modify_scm) 264 tests = port.tests(args) 265 for test_name in tests: 266 files_to_delete, files_to_add = self._optimize_baseline(optimizer, test_name) 267 for path in files_to_delete: 268 self._delete_from_scm_later(path) 269 for path in files_to_add: 270 self._add_to_scm_later(path) 271 272 print json.dumps(self._scm_changes) 273 274 275 class AnalyzeBaselines(AbstractRebaseliningCommand): 276 name = "analyze-baselines" 277 help_text = "Analyzes the baselines for the given tests and prints results that are identical." 278 show_in_main_help = True 279 argument_names = "TEST_NAMES" 280 281 def __init__(self): 282 super(AnalyzeBaselines, self).__init__(options=[ 283 self.suffixes_option, 284 optparse.make_option('--missing', action='store_true', default=False, help='show missing baselines as well'), 285 ] + self.platform_options) 286 self._optimizer_class = BaselineOptimizer # overridable for testing 287 self._baseline_optimizer = None 288 self._port = None 289 290 def _write(self, msg): 291 print msg 292 293 def _analyze_baseline(self, options, test_name): 294 for suffix in self._baseline_suffix_list: 295 baseline_name = _baseline_name(self._tool.filesystem, test_name, suffix) 296 results_by_directory = self._baseline_optimizer.read_results_by_directory(baseline_name) 297 if results_by_directory: 298 self._write("%s:" % baseline_name) 299 self._baseline_optimizer.write_by_directory(results_by_directory, self._write, " ") 300 elif options.missing: 301 self._write("%s: (no baselines found)" % baseline_name) 302 303 def execute(self, options, args, tool): 304 self._baseline_suffix_list = options.suffixes.split(',') 305 port_names = tool.port_factory.all_port_names(options.platform) 306 if not port_names: 307 print "No port names match '%s'" % options.platform 308 return 309 self._port = tool.port_factory.get(port_names[0]) 310 self._baseline_optimizer = self._optimizer_class(tool, self._port, port_names, skip_scm_commands=False) 311 for test_name in self._port.tests(args): 312 self._analyze_baseline(options, test_name) 313 314 315 class AbstractParallelRebaselineCommand(AbstractRebaseliningCommand): 316 # not overriding execute() - pylint: disable=W0223 317 318 def __init__(self, options=None): 319 super(AbstractParallelRebaselineCommand, self).__init__(options=options) 320 self._builder_data = {} 321 322 def builder_data(self): 323 if not self._builder_data: 324 for builder_name in self._release_builders(): 325 builder = self._tool.buildbot_for_builder_name(builder_name).builder_with_name(builder_name) 326 self._builder_data[builder_name] = builder.latest_layout_test_results() 327 return self._builder_data 328 329 # The release builders cycle much faster than the debug ones and cover all the platforms. 330 def _release_builders(self): 331 release_builders = [] 332 for builder_name in builders.all_builder_names(): 333 if builder_name.find('ASAN') != -1: 334 continue 335 port = self._tool.port_factory.get_from_builder_name(builder_name) 336 if port.test_configuration().build_type == 'release': 337 release_builders.append(builder_name) 338 return release_builders 339 340 def _run_webkit_patch(self, args, verbose): 341 try: 342 verbose_args = ['--verbose'] if verbose else [] 343 stderr = self._tool.executive.run_command([self._tool.path()] + verbose_args + args, cwd=self._tool.scm().checkout_root, return_stderr=True) 344 for line in stderr.splitlines(): 345 _log.warning(line) 346 except ScriptError, e: 347 _log.error(e) 348 349 def _builders_to_fetch_from(self, builders_to_check): 350 # This routine returns the subset of builders that will cover all of the baseline search paths 351 # used in the input list. In particular, if the input list contains both Release and Debug 352 # versions of a configuration, we *only* return the Release version (since we don't save 353 # debug versions of baselines). 354 release_builders = set() 355 debug_builders = set() 356 builders_to_fallback_paths = {} 357 for builder in builders_to_check: 358 port = self._tool.port_factory.get_from_builder_name(builder) 359 if port.test_configuration().build_type == 'release': 360 release_builders.add(builder) 361 else: 362 debug_builders.add(builder) 363 for builder in list(release_builders) + list(debug_builders): 364 port = self._tool.port_factory.get_from_builder_name(builder) 365 fallback_path = port.baseline_search_path() 366 if fallback_path not in builders_to_fallback_paths.values(): 367 builders_to_fallback_paths[builder] = fallback_path 368 return builders_to_fallback_paths.keys() 369 370 def _rebaseline_commands(self, test_prefix_list, options): 371 path_to_webkit_patch = self._tool.path() 372 cwd = self._tool.scm().checkout_root 373 copy_baseline_commands = [] 374 rebaseline_commands = [] 375 lines_to_remove = {} 376 port = self._tool.port_factory.get() 377 378 for test_prefix in test_prefix_list: 379 for test in port.tests([test_prefix]): 380 for builder in self._builders_to_fetch_from(test_prefix_list[test_prefix]): 381 actual_failures_suffixes = self._suffixes_for_actual_failures(test, builder, test_prefix_list[test_prefix][builder]) 382 if not actual_failures_suffixes: 383 # If we're not going to rebaseline the test because it's passing on this 384 # builder, we still want to remove the line from TestExpectations. 385 if test not in lines_to_remove: 386 lines_to_remove[test] = [] 387 lines_to_remove[test].append(builder) 388 continue 389 390 suffixes = ','.join(actual_failures_suffixes) 391 cmd_line = ['--suffixes', suffixes, '--builder', builder, '--test', test] 392 if options.results_directory: 393 cmd_line.extend(['--results-directory', options.results_directory]) 394 if options.verbose: 395 cmd_line.append('--verbose') 396 copy_baseline_commands.append(tuple([[self._tool.executable, path_to_webkit_patch, 'copy-existing-baselines-internal'] + cmd_line, cwd])) 397 rebaseline_commands.append(tuple([[self._tool.executable, path_to_webkit_patch, 'rebaseline-test-internal'] + cmd_line, cwd])) 398 return copy_baseline_commands, rebaseline_commands, lines_to_remove 399 400 def _serial_commands(self, command_results): 401 files_to_add = set() 402 files_to_delete = set() 403 lines_to_remove = {} 404 for output in [result[1].split('\n') for result in command_results]: 405 file_added = False 406 for line in output: 407 try: 408 if line: 409 parsed_line = json.loads(line) 410 if 'add' in parsed_line: 411 files_to_add.update(parsed_line['add']) 412 if 'delete' in parsed_line: 413 files_to_delete.update(parsed_line['delete']) 414 if 'remove-lines' in parsed_line: 415 for line_to_remove in parsed_line['remove-lines']: 416 test = line_to_remove['test'] 417 builder = line_to_remove['builder'] 418 if test not in lines_to_remove: 419 lines_to_remove[test] = [] 420 lines_to_remove[test].append(builder) 421 file_added = True 422 except ValueError: 423 _log.debug('"%s" is not a JSON object, ignoring' % line) 424 425 if not file_added: 426 _log.debug('Could not add file based off output "%s"' % output) 427 428 return list(files_to_add), list(files_to_delete), lines_to_remove 429 430 def _optimize_baselines(self, test_prefix_list, verbose=False): 431 optimize_commands = [] 432 for test in test_prefix_list: 433 all_suffixes = set() 434 for builder in self._builders_to_fetch_from(test_prefix_list[test]): 435 all_suffixes.update(self._suffixes_for_actual_failures(test, builder, test_prefix_list[test][builder])) 436 437 # FIXME: We should propagate the platform options as well. 438 cmd_line = ['--no-modify-scm', '--suffixes', ','.join(all_suffixes), test] 439 if verbose: 440 cmd_line.append('--verbose') 441 442 path_to_webkit_patch = self._tool.path() 443 cwd = self._tool.scm().checkout_root 444 optimize_commands.append(tuple([[self._tool.executable, path_to_webkit_patch, 'optimize-baselines'] + cmd_line, cwd])) 445 return optimize_commands 446 447 def _update_expectations_files(self, lines_to_remove): 448 # FIXME: This routine is way too expensive. We're creating O(n ports) TestExpectations objects. 449 # This is slow and uses a lot of memory. 450 tests = lines_to_remove.keys() 451 to_remove = [] 452 453 # This is so we remove lines for builders that skip this test, e.g. Android skips most 454 # tests and we don't want to leave stray [ Android ] lines in TestExpectations.. 455 # This is only necessary for "webkit-patch rebaseline" and for rebaselining expected 456 # failures from garden-o-matic. rebaseline-expectations and auto-rebaseline will always 457 # pass the exact set of ports to rebaseline. 458 for port_name in self._tool.port_factory.all_port_names(): 459 port = self._tool.port_factory.get(port_name) 460 generic_expectations = TestExpectations(port, tests=tests, include_overrides=False) 461 full_expectations = TestExpectations(port, tests=tests, include_overrides=True) 462 for test in tests: 463 if self._port_skips_test(port, test, generic_expectations, full_expectations): 464 for test_configuration in port.all_test_configurations(): 465 if test_configuration.version == port.test_configuration().version: 466 to_remove.append((test, test_configuration)) 467 468 for test in lines_to_remove: 469 for builder in lines_to_remove[test]: 470 port = self._tool.port_factory.get_from_builder_name(builder) 471 for test_configuration in port.all_test_configurations(): 472 if test_configuration.version == port.test_configuration().version: 473 to_remove.append((test, test_configuration)) 474 475 port = self._tool.port_factory.get() 476 expectations = TestExpectations(port, include_overrides=False) 477 expectationsString = expectations.remove_configurations(to_remove) 478 path = port.path_to_generic_test_expectations_file() 479 self._tool.filesystem.write_text_file(path, expectationsString) 480 481 def _port_skips_test(self, port, test, generic_expectations, full_expectations): 482 fs = port.host.filesystem 483 if port.default_smoke_test_only(): 484 smoke_test_filename = fs.join(port.layout_tests_dir(), 'SmokeTests') 485 if fs.exists(smoke_test_filename) and test not in fs.read_text_file(smoke_test_filename): 486 return True 487 488 return (SKIP in full_expectations.get_expectations(test) and 489 SKIP not in generic_expectations.get_expectations(test)) 490 491 def _run_in_parallel_and_update_scm(self, commands): 492 command_results = self._tool.executive.run_in_parallel(commands) 493 log_output = '\n'.join(result[2] for result in command_results).replace('\n\n', '\n') 494 for line in log_output.split('\n'): 495 if line: 496 print >> sys.stderr, line # FIXME: Figure out how to log properly. 497 498 files_to_add, files_to_delete, lines_to_remove = self._serial_commands(command_results) 499 if files_to_delete: 500 self._tool.scm().delete_list(files_to_delete) 501 if files_to_add: 502 self._tool.scm().add_list(files_to_add) 503 return lines_to_remove 504 505 def _rebaseline(self, options, test_prefix_list): 506 for test, builders_to_check in sorted(test_prefix_list.items()): 507 _log.info("Rebaselining %s" % test) 508 for builder, suffixes in sorted(builders_to_check.items()): 509 _log.debug(" %s: %s" % (builder, ",".join(suffixes))) 510 511 copy_baseline_commands, rebaseline_commands, extra_lines_to_remove = self._rebaseline_commands(test_prefix_list, options) 512 lines_to_remove = {} 513 514 if copy_baseline_commands: 515 self._run_in_parallel_and_update_scm(copy_baseline_commands) 516 if rebaseline_commands: 517 lines_to_remove = self._run_in_parallel_and_update_scm(rebaseline_commands) 518 519 for test in extra_lines_to_remove: 520 if test in lines_to_remove: 521 lines_to_remove[test] = lines_to_remove[test] + extra_lines_to_remove[test] 522 else: 523 lines_to_remove[test] = extra_lines_to_remove[test] 524 525 if lines_to_remove: 526 self._update_expectations_files(lines_to_remove) 527 528 if options.optimize: 529 self._run_in_parallel_and_update_scm(self._optimize_baselines(test_prefix_list, options.verbose)) 530 531 def _suffixes_for_actual_failures(self, test, builder_name, existing_suffixes): 532 actual_results = self.builder_data()[builder_name].actual_results(test) 533 if not actual_results: 534 return set() 535 return set(existing_suffixes) & TestExpectations.suffixes_for_actual_expectations_string(actual_results) 536 537 538 class RebaselineJson(AbstractParallelRebaselineCommand): 539 name = "rebaseline-json" 540 help_text = "Rebaseline based off JSON passed to stdin. Intended to only be called from other scripts." 541 542 def __init__(self,): 543 super(RebaselineJson, self).__init__(options=[ 544 self.no_optimize_option, 545 self.results_directory_option, 546 ]) 547 548 def execute(self, options, args, tool): 549 self._rebaseline(options, json.loads(sys.stdin.read())) 550 551 552 class RebaselineExpectations(AbstractParallelRebaselineCommand): 553 name = "rebaseline-expectations" 554 help_text = "Rebaselines the tests indicated in TestExpectations." 555 show_in_main_help = True 556 557 def __init__(self): 558 super(RebaselineExpectations, self).__init__(options=[ 559 self.no_optimize_option, 560 ] + self.platform_options) 561 self._test_prefix_list = None 562 563 def _tests_to_rebaseline(self, port): 564 tests_to_rebaseline = {} 565 for path, value in port.expectations_dict().items(): 566 expectations = TestExpectations(port, include_overrides=False, expectations_dict={path: value}) 567 for test in expectations.get_rebaselining_failures(): 568 suffixes = TestExpectations.suffixes_for_expectations(expectations.get_expectations(test)) 569 tests_to_rebaseline[test] = suffixes or BASELINE_SUFFIX_LIST 570 return tests_to_rebaseline 571 572 def _add_tests_to_rebaseline_for_port(self, port_name): 573 builder_name = builders.builder_name_for_port_name(port_name) 574 if not builder_name: 575 return 576 tests = self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).items() 577 578 if tests: 579 _log.info("Retrieving results for %s from %s." % (port_name, builder_name)) 580 581 for test_name, suffixes in tests: 582 _log.info(" %s (%s)" % (test_name, ','.join(suffixes))) 583 if test_name not in self._test_prefix_list: 584 self._test_prefix_list[test_name] = {} 585 self._test_prefix_list[test_name][builder_name] = suffixes 586 587 def execute(self, options, args, tool): 588 options.results_directory = None 589 self._test_prefix_list = {} 590 port_names = tool.port_factory.all_port_names(options.platform) 591 for port_name in port_names: 592 self._add_tests_to_rebaseline_for_port(port_name) 593 if not self._test_prefix_list: 594 _log.warning("Did not find any tests marked Rebaseline.") 595 return 596 597 self._rebaseline(options, self._test_prefix_list) 598 599 600 class Rebaseline(AbstractParallelRebaselineCommand): 601 name = "rebaseline" 602 help_text = "Rebaseline tests with results from the build bots. Shows the list of failing tests on the builders if no test names are provided." 603 show_in_main_help = True 604 argument_names = "[TEST_NAMES]" 605 606 def __init__(self): 607 super(Rebaseline, self).__init__(options=[ 608 self.no_optimize_option, 609 # FIXME: should we support the platform options in addition to (or instead of) --builders? 610 self.suffixes_option, 611 self.results_directory_option, 612 optparse.make_option("--builders", default=None, action="append", help="Comma-separated-list of builders to pull new baselines from (can also be provided multiple times)"), 613 ]) 614 615 def _builders_to_pull_from(self): 616 chosen_names = self._tool.user.prompt_with_list("Which builder to pull results from:", self._release_builders(), can_choose_multiple=True) 617 return [self._builder_with_name(name) for name in chosen_names] 618 619 def _builder_with_name(self, name): 620 return self._tool.buildbot_for_builder_name(name).builder_with_name(name) 621 622 def execute(self, options, args, tool): 623 if not args: 624 _log.error("Must list tests to rebaseline.") 625 return 626 627 if options.builders: 628 builders_to_check = [] 629 for builder_names in options.builders: 630 builders_to_check += [self._builder_with_name(name) for name in builder_names.split(",")] 631 else: 632 builders_to_check = self._builders_to_pull_from() 633 634 test_prefix_list = {} 635 suffixes_to_update = options.suffixes.split(",") 636 637 for builder in builders_to_check: 638 for test in args: 639 if test not in test_prefix_list: 640 test_prefix_list[test] = {} 641 test_prefix_list[test][builder.name()] = suffixes_to_update 642 643 if options.verbose: 644 _log.debug("rebaseline-json: " + str(test_prefix_list)) 645 646 self._rebaseline(options, test_prefix_list) 647 648 649 class AutoRebaseline(AbstractParallelRebaselineCommand): 650 name = "auto-rebaseline" 651 help_text = "Rebaselines any NeedsRebaseline lines in TestExpectations that have cycled through all the bots." 652 AUTO_REBASELINE_BRANCH_NAME = "auto-rebaseline-temporary-branch" 653 654 # Rietveld uploader stinks. Limit the number of rebaselines in a given patch to keep upload from failing. 655 # FIXME: http://crbug.com/263676 Obviously we should fix the uploader here. 656 MAX_LINES_TO_REBASELINE = 200 657 658 SECONDS_BEFORE_GIVING_UP = 300 659 660 def __init__(self): 661 super(AutoRebaseline, self).__init__(options=[ 662 # FIXME: Remove this option. 663 self.no_optimize_option, 664 # FIXME: Remove this option. 665 self.results_directory_option, 666 ]) 667 668 def bot_revision_data(self): 669 revisions = [] 670 for result in self.builder_data().values(): 671 if result.run_was_interrupted(): 672 _log.error("Can't rebaseline because the latest run on %s exited early." % result.builder_name()) 673 return [] 674 revisions.append({ 675 "builder": result.builder_name(), 676 "revision": result.blink_revision(), 677 }) 678 return revisions 679 680 def tests_to_rebaseline(self, tool, min_revision, print_revisions): 681 port = tool.port_factory.get() 682 expectations_file_path = port.path_to_generic_test_expectations_file() 683 684 tests = set() 685 revision = None 686 author = None 687 bugs = set() 688 has_any_needs_rebaseline_lines = False 689 690 for line in tool.scm().blame(expectations_file_path).split("\n"): 691 comment_index = line.find("#") 692 if comment_index == -1: 693 comment_index = len(line) 694 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].strip()) 695 696 if "NeedsRebaseline" not in line_without_comments: 697 continue 698 699 has_any_needs_rebaseline_lines = True 700 701 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", line_without_comments) 702 703 commit_hash = parsed_line.group(1) 704 svn_revision = tool.scm().svn_revision_from_git_commit(commit_hash) 705 706 test = parsed_line.group(3) 707 if print_revisions: 708 _log.info("%s is waiting for r%s" % (test, svn_revision)) 709 710 if not svn_revision or svn_revision > min_revision: 711 continue 712 713 if revision and svn_revision != revision: 714 continue 715 716 if not revision: 717 revision = svn_revision 718 author = parsed_line.group(2) 719 720 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) 721 tests.add(test) 722 723 if len(tests) >= self.MAX_LINES_TO_REBASELINE: 724 _log.info("Too many tests to rebaseline in one patch. Doing the first %d." % self.MAX_LINES_TO_REBASELINE) 725 break 726 727 return tests, revision, author, bugs, has_any_needs_rebaseline_lines 728 729 def link_to_patch(self, revision): 730 return "http://src.chromium.org/viewvc/blink?view=revision&revision=" + str(revision) 731 732 def commit_message(self, author, revision, bugs): 733 bug_string = "" 734 if bugs: 735 bug_string = "BUG=%s\n" % ",".join(bugs) 736 737 return """Auto-rebaseline for r%s 738 739 %s 740 741 %sTBR=%s 742 """ % (revision, self.link_to_patch(revision), bug_string, author) 743 744 def get_test_prefix_list(self, tests): 745 test_prefix_list = {} 746 lines_to_remove = {} 747 748 for builder_name in self._release_builders(): 749 port_name = builders.port_name_for_builder_name(builder_name) 750 port = self._tool.port_factory.get(port_name) 751 expectations = TestExpectations(port, include_overrides=True) 752 for test in expectations.get_needs_rebaseline_failures(): 753 if test not in tests: 754 continue 755 756 if test not in test_prefix_list: 757 lines_to_remove[test] = [] 758 test_prefix_list[test] = {} 759 lines_to_remove[test].append(builder_name) 760 test_prefix_list[test][builder_name] = BASELINE_SUFFIX_LIST 761 762 return test_prefix_list, lines_to_remove 763 764 def _run_git_cl_command(self, options, command): 765 subprocess_command = ['git', 'cl'] + command 766 if options.verbose: 767 subprocess_command.append('--verbose') 768 769 process = self._tool.executive.popen(subprocess_command, stdout=self._tool.executive.PIPE, stderr=self._tool.executive.STDOUT) 770 last_output_time = time.time() 771 772 # git cl sometimes completely hangs. Bail if we haven't gotten any output to stdout/stderr in a while. 773 while process.poll() == None and time.time() < last_output_time + self.SECONDS_BEFORE_GIVING_UP: 774 # FIXME: This doesn't make any sense. readline blocks, so all this code to 775 # try and bail is useless. Instead, we should do the readline calls on a 776 # subthread. Then the rest of this code would make sense. 777 out = process.stdout.readline().rstrip('\n') 778 if out: 779 last_output_time = time.time() 780 _log.info(out) 781 782 if process.poll() == None: 783 _log.error('Command hung: %s' % subprocess_command) 784 return False 785 return True 786 787 # FIXME: Move this somewhere more general. 788 def tree_status(self): 789 blink_tree_status_url = "http://blink-status.appspot.com/status" 790 status = urllib2.urlopen(blink_tree_status_url).read().lower() 791 if status.find('closed') != -1 or status == "0": 792 return 'closed' 793 elif status.find('open') != -1 or status == "1": 794 return 'open' 795 return 'unknown' 796 797 def execute(self, options, args, tool): 798 if tool.scm().executable_name == "svn": 799 _log.error("Auto rebaseline only works with a git checkout.") 800 return 801 802 if tool.scm().has_working_directory_changes(): 803 _log.error("Cannot proceed with working directory changes. Clean working directory first.") 804 return 805 806 revision_data = self.bot_revision_data() 807 if not revision_data: 808 return 809 810 min_revision = int(min([item["revision"] for item in revision_data])) 811 tests, revision, author, bugs, has_any_needs_rebaseline_lines = self.tests_to_rebaseline(tool, min_revision, print_revisions=options.verbose) 812 813 if options.verbose: 814 _log.info("Min revision across all bots is %s." % min_revision) 815 for item in revision_data: 816 _log.info("%s: r%s" % (item["builder"], item["revision"])) 817 818 if not tests: 819 _log.debug('No tests to rebaseline.') 820 return 821 822 if self.tree_status() == 'closed': 823 _log.info('Cannot proceed. Tree is closed.') 824 return 825 826 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, author)) 827 828 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) 829 830 did_finish = False 831 try: 832 old_branch_name = tool.scm().current_branch() 833 tool.scm().delete_branch(self.AUTO_REBASELINE_BRANCH_NAME) 834 tool.scm().create_clean_branch(self.AUTO_REBASELINE_BRANCH_NAME) 835 836 # If the tests are passing everywhere, then this list will be empty. We don't need 837 # to rebaseline, but we'll still need to update TestExpectations. 838 if test_prefix_list: 839 self._rebaseline(options, test_prefix_list) 840 841 tool.scm().commit_locally_with_message(self.commit_message(author, revision, bugs)) 842 843 # FIXME: It would be nice if we could dcommit the patch without uploading, but still 844 # go through all the precommit hooks. For rebaselines with lots of files, uploading 845 # takes a long time and sometimes fails, but we don't want to commit if, e.g. the 846 # tree is closed. 847 did_finish = self._run_git_cl_command(options, ['upload', '-f']) 848 849 if did_finish: 850 # Uploading can take a very long time. Do another pull to make sure TestExpectations is up to date, 851 # so the dcommit can go through. 852 # FIXME: Log the pull and dcommit stdout/stderr to the log-server. 853 tool.executive.run_command(['git', 'pull']) 854 855 self._run_git_cl_command(options, ['dcommit', '-f']) 856 except Exception as e: 857 _log.error(e) 858 finally: 859 if did_finish: 860 self._run_git_cl_command(options, ['set_close']) 861 tool.scm().ensure_cleanly_tracking_remote_master() 862 tool.scm().checkout_branch(old_branch_name) 863 tool.scm().delete_branch(self.AUTO_REBASELINE_BRANCH_NAME) 864 865 866 class RebaselineOMatic(AbstractDeclarativeCommand): 867 name = "rebaseline-o-matic" 868 help_text = "Calls webkit-patch auto-rebaseline in a loop." 869 show_in_main_help = True 870 871 SLEEP_TIME_IN_SECONDS = 30 872 LOG_SERVER = 'blinkrebaseline.appspot.com' 873 QUIT_LOG = '##QUIT##' 874 875 # Uploaded log entries append to the existing entry unless the 876 # newentry flag is set. In that case it starts a new entry to 877 # start appending to. 878 def _log_to_server(self, log='', is_new_entry=False): 879 query = { 880 'log': log, 881 } 882 if is_new_entry: 883 query['newentry'] = 'on' 884 try: 885 urllib2.urlopen("http://" + self.LOG_SERVER + "/updatelog", data=urllib.urlencode(query)) 886 except: 887 traceback.print_exc(file=sys.stderr) 888 889 def _log_to_server_thread(self): 890 is_new_entry = True 891 while True: 892 messages = [self._log_queue.get()] 893 while not self._log_queue.empty(): 894 messages.append(self._log_queue.get()) 895 self._log_to_server('\n'.join(messages), is_new_entry=is_new_entry) 896 is_new_entry = False 897 if self.QUIT_LOG in messages: 898 return 899 900 def _post_log_to_server(self, log): 901 self._log_queue.put(log) 902 903 def _log_line(self, handle): 904 out = handle.readline().rstrip('\n') 905 if out: 906 if self._verbose: 907 print out 908 self._post_log_to_server(out) 909 return out 910 911 def _run_logged_command(self, command): 912 process = self._tool.executive.popen(command, stdout=self._tool.executive.PIPE, stderr=self._tool.executive.STDOUT) 913 914 out = self._log_line(process.stdout) 915 while out: 916 # FIXME: This should probably batch up lines if they're available and log to the server once. 917 out = self._log_line(process.stdout) 918 919 def _do_one_rebaseline(self): 920 self._log_queue = Queue.Queue(256) 921 log_thread = threading.Thread(name='LogToServer', target=self._log_to_server_thread) 922 log_thread.start() 923 try: 924 old_branch_name = self._tool.scm().current_branch() 925 self._run_logged_command(['git', 'pull']) 926 rebaseline_command = [self._tool.filesystem.join(self._tool.scm().checkout_root, 'Tools', 'Scripts', 'webkit-patch'), 'auto-rebaseline'] 927 if self._verbose: 928 rebaseline_command.append('--verbose') 929 self._run_logged_command(rebaseline_command) 930 except: 931 self._log_queue.put(self.QUIT_LOG) 932 traceback.print_exc(file=sys.stderr) 933 # Sometimes git crashes and leaves us on a detached head. 934 self._tool.scm().checkout_branch(old_branch_name) 935 else: 936 self._log_queue.put(self.QUIT_LOG) 937 log_thread.join() 938 939 def execute(self, options, args, tool): 940 self._verbose = options.verbose 941 while True: 942 self._do_one_rebaseline() 943 time.sleep(self.SLEEP_TIME_IN_SECONDS) 944
