1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 # 6 # Most of this file was ported over from Blink's 7 # Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 8 # Tools/Scripts/webkitpy/common/net/file_uploader.py 9 # 10 11 import json 12 import logging 13 import mimetypes 14 import os 15 import time 16 import urllib2 17 18 _log = logging.getLogger(__name__) 19 20 _JSON_PREFIX = 'ADD_RESULTS(' 21 _JSON_SUFFIX = ');' 22 23 24 def HasJSONWrapper(string): 25 return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX) 26 27 28 def StripJSONWrapper(json_content): 29 # FIXME: Kill this code once the server returns json instead of jsonp. 30 if HasJSONWrapper(json_content): 31 return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] 32 return json_content 33 34 35 def WriteJSON(json_object, file_path, callback=None): 36 # Specify separators in order to get compact encoding. 37 json_string = json.dumps(json_object, separators=(',', ':')) 38 if callback: 39 json_string = callback + '(' + json_string + ');' 40 with open(file_path, 'w') as fp: 41 fp.write(json_string) 42 43 44 def ConvertTrieToFlatPaths(trie, prefix=None): 45 """Flattens the trie of paths, prepending a prefix to each.""" 46 result = {} 47 for name, data in trie.iteritems(): 48 if prefix: 49 name = prefix + '/' + name 50 51 if len(data) and not 'results' in data: 52 result.update(ConvertTrieToFlatPaths(data, name)) 53 else: 54 result[name] = data 55 56 return result 57 58 59 def AddPathToTrie(path, value, trie): 60 """Inserts a single path and value into a directory trie structure.""" 61 if not '/' in path: 62 trie[path] = value 63 return 64 65 directory, _, rest = path.partition('/') 66 if not directory in trie: 67 trie[directory] = {} 68 AddPathToTrie(rest, value, trie[directory]) 69 70 71 def TestTimingsTrie(individual_test_timings): 72 """Breaks a test name into dicts by directory 73 74 foo/bar/baz.html: 1ms 75 foo/bar/baz1.html: 3ms 76 77 becomes 78 foo: { 79 bar: { 80 baz.html: 1, 81 baz1.html: 3 82 } 83 } 84 """ 85 trie = {} 86 for test_result in individual_test_timings: 87 test = test_result.test_name 88 89 AddPathToTrie(test, int(1000 * test_result.test_run_time), trie) 90 91 return trie 92 93 94 class TestResult(object): 95 """A simple class that represents a single test result.""" 96 97 # Test modifier constants. 98 (NONE, FAILS, FLAKY, DISABLED) = range(4) 99 100 def __init__(self, test, failed=False, elapsed_time=0): 101 self.test_name = test 102 self.failed = failed 103 self.test_run_time = elapsed_time 104 105 test_name = test 106 try: 107 test_name = test.split('.')[1] 108 except IndexError: 109 _log.warn('Invalid test name: %s.', test) 110 111 if test_name.startswith('FAILS_'): 112 self.modifier = self.FAILS 113 elif test_name.startswith('FLAKY_'): 114 self.modifier = self.FLAKY 115 elif test_name.startswith('DISABLED_'): 116 self.modifier = self.DISABLED 117 else: 118 self.modifier = self.NONE 119 120 def Fixable(self): 121 return self.failed or self.modifier == self.DISABLED 122 123 124 class JSONResultsGeneratorBase(object): 125 """A JSON results generator for generic tests.""" 126 127 MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 128 # Min time (seconds) that will be added to the JSON. 129 MIN_TIME = 1 130 131 # Note that in non-chromium tests those chars are used to indicate 132 # test modifiers (FAILS, FLAKY, etc) but not actual test results. 133 PASS_RESULT = 'P' 134 SKIP_RESULT = 'X' 135 FAIL_RESULT = 'F' 136 FLAKY_RESULT = 'L' 137 NO_DATA_RESULT = 'N' 138 139 MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT, 140 TestResult.DISABLED: SKIP_RESULT, 141 TestResult.FAILS: FAIL_RESULT, 142 TestResult.FLAKY: FLAKY_RESULT} 143 144 VERSION = 4 145 VERSION_KEY = 'version' 146 RESULTS = 'results' 147 TIMES = 'times' 148 BUILD_NUMBERS = 'buildNumbers' 149 TIME = 'secondsSinceEpoch' 150 TESTS = 'tests' 151 152 FIXABLE_COUNT = 'fixableCount' 153 FIXABLE = 'fixableCounts' 154 ALL_FIXABLE_COUNT = 'allFixableCount' 155 156 RESULTS_FILENAME = 'results.json' 157 TIMES_MS_FILENAME = 'times_ms.json' 158 INCREMENTAL_RESULTS_FILENAME = 'incremental_results.json' 159 160 # line too long pylint: disable=line-too-long 161 URL_FOR_TEST_LIST_JSON = ( 162 'http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&master=%s') 163 # pylint: enable=line-too-long 164 165 def __init__(self, builder_name, build_name, build_number, 166 results_file_base_path, builder_base_url, 167 test_results_map, svn_repositories=None, 168 test_results_server=None, 169 test_type='', 170 master_name=''): 171 """Modifies the results.json file. Grabs it off the archive directory 172 if it is not found locally. 173 174 Args 175 builder_name: the builder name (e.g. Webkit). 176 build_name: the build name (e.g. webkit-rel). 177 build_number: the build number. 178 results_file_base_path: Absolute path to the directory containing the 179 results json file. 180 builder_base_url: the URL where we have the archived test results. 181 If this is None no archived results will be retrieved. 182 test_results_map: A dictionary that maps test_name to TestResult. 183 svn_repositories: A (json_field_name, svn_path) pair for SVN 184 repositories that tests rely on. The SVN revision will be 185 included in the JSON with the given json_field_name. 186 test_results_server: server that hosts test results json. 187 test_type: test type string (e.g. 'layout-tests'). 188 master_name: the name of the buildbot master. 189 """ 190 self._builder_name = builder_name 191 self._build_name = build_name 192 self._build_number = build_number 193 self._builder_base_url = builder_base_url 194 self._results_directory = results_file_base_path 195 196 self._test_results_map = test_results_map 197 self._test_results = test_results_map.values() 198 199 self._svn_repositories = svn_repositories 200 if not self._svn_repositories: 201 self._svn_repositories = {} 202 203 self._test_results_server = test_results_server 204 self._test_type = test_type 205 self._master_name = master_name 206 207 self._archived_results = None 208 209 def GenerateJSONOutput(self): 210 json_object = self.GetJSON() 211 if json_object: 212 file_path = ( 213 os.path.join( 214 self._results_directory, 215 self.INCREMENTAL_RESULTS_FILENAME)) 216 WriteJSON(json_object, file_path) 217 218 def GenerateTimesMSFile(self): 219 times = TestTimingsTrie(self._test_results_map.values()) 220 file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME) 221 WriteJSON(times, file_path) 222 223 def GetJSON(self): 224 """Gets the results for the results.json file.""" 225 results_json = {} 226 227 if not results_json: 228 results_json, error = self._GetArchivedJSONResults() 229 if error: 230 # If there was an error don't write a results.json 231 # file at all as it would lose all the information on the 232 # bot. 233 _log.error('Archive directory is inaccessible. Not ' 234 'modifying or clobbering the results.json ' 235 'file: ' + str(error)) 236 return None 237 238 builder_name = self._builder_name 239 if results_json and builder_name not in results_json: 240 _log.debug('Builder name (%s) is not in the results.json file.', 241 builder_name) 242 243 self._ConvertJSONToCurrentVersion(results_json) 244 245 if builder_name not in results_json: 246 results_json[builder_name] = ( 247 self._CreateResultsForBuilderJSON()) 248 249 results_for_builder = results_json[builder_name] 250 251 if builder_name: 252 self._InsertGenericMetaData(results_for_builder) 253 254 self._InsertFailureSummaries(results_for_builder) 255 256 # Update the all failing tests with result type and time. 257 tests = results_for_builder[self.TESTS] 258 all_failing_tests = self._GetFailedTestNames() 259 all_failing_tests.update(ConvertTrieToFlatPaths(tests)) 260 261 for test in all_failing_tests: 262 self._InsertTestTimeAndResult(test, tests) 263 264 return results_json 265 266 def SetArchivedResults(self, archived_results): 267 self._archived_results = archived_results 268 269 def UploadJSONFiles(self, json_files): 270 """Uploads the given json_files to the test_results_server (if the 271 test_results_server is given).""" 272 if not self._test_results_server: 273 return 274 275 if not self._master_name: 276 _log.error( 277 '--test-results-server was set, but --master-name was not. Not ' 278 'uploading JSON files.') 279 return 280 281 _log.info('Uploading JSON files for builder: %s', self._builder_name) 282 attrs = [('builder', self._builder_name), 283 ('testtype', self._test_type), 284 ('master', self._master_name)] 285 286 files = [(json_file, os.path.join(self._results_directory, json_file)) 287 for json_file in json_files] 288 289 url = 'http://%s/testfile/upload' % self._test_results_server 290 # Set uploading timeout in case appengine server is having problems. 291 # 120 seconds are more than enough to upload test results. 292 uploader = _FileUploader(url, 120) 293 try: 294 response = uploader.UploadAsMultipartFormData(files, attrs) 295 if response: 296 if response.code == 200: 297 _log.info('JSON uploaded.') 298 else: 299 _log.debug( 300 "JSON upload failed, %d: '%s'", response.code, response.read()) 301 else: 302 _log.error('JSON upload failed; no response returned') 303 except Exception, err: # pylint: disable=broad-except 304 _log.error('Upload failed: %s', err) 305 return 306 307 def _GetTestTiming(self, test_name): 308 """Returns test timing data (elapsed time) in second 309 for the given test_name.""" 310 if test_name in self._test_results_map: 311 # Floor for now to get time in seconds. 312 return int(self._test_results_map[test_name].test_run_time) 313 return 0 314 315 def _GetFailedTestNames(self): 316 """Returns a set of failed test names.""" 317 return set([r.test_name for r in self._test_results if r.failed]) 318 319 def _GetModifierChar(self, test_name): 320 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 321 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier 322 for the given test_name. 323 """ 324 if test_name not in self._test_results_map: 325 return self.__class__.NO_DATA_RESULT 326 327 test_result = self._test_results_map[test_name] 328 if test_result.modifier in self.MODIFIER_TO_CHAR.keys(): 329 return self.MODIFIER_TO_CHAR[test_result.modifier] 330 331 return self.__class__.PASS_RESULT 332 333 def _get_result_char(self, test_name): 334 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 335 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result 336 for the given test_name. 337 """ 338 if test_name not in self._test_results_map: 339 return self.__class__.NO_DATA_RESULT 340 341 test_result = self._test_results_map[test_name] 342 if test_result.modifier == TestResult.DISABLED: 343 return self.__class__.SKIP_RESULT 344 345 if test_result.failed: 346 return self.__class__.FAIL_RESULT 347 348 return self.__class__.PASS_RESULT 349 350 def _GetSVNRevision(self, in_directory): 351 """Returns the svn revision for the given directory. 352 353 Args: 354 in_directory: The directory where svn is to be run. 355 """ 356 # This is overridden in flakiness_dashboard_results_uploader.py. 357 raise NotImplementedError() 358 359 def _GetArchivedJSONResults(self): 360 """Download JSON file that only contains test 361 name list from test-results server. This is for generating incremental 362 JSON so the file generated has info for tests that failed before but 363 pass or are skipped from current run. 364 365 Returns (archived_results, error) tuple where error is None if results 366 were successfully read. 367 """ 368 results_json = {} 369 old_results = None 370 error = None 371 372 if not self._test_results_server: 373 return {}, None 374 375 results_file_url = (self.URL_FOR_TEST_LIST_JSON % 376 (urllib2.quote(self._test_results_server), 377 urllib2.quote(self._builder_name), 378 self.RESULTS_FILENAME, 379 urllib2.quote(self._test_type), 380 urllib2.quote(self._master_name))) 381 382 try: 383 # FIXME: We should talk to the network via a Host object. 384 results_file = urllib2.urlopen(results_file_url) 385 old_results = results_file.read() 386 except urllib2.HTTPError, http_error: 387 # A non-4xx status code means the bot is hosed for some reason 388 # and we can't grab the results.json file off of it. 389 if http_error.code < 400 and http_error.code >= 500: 390 error = http_error 391 except urllib2.URLError, url_error: 392 error = url_error 393 394 if old_results: 395 # Strip the prefix and suffix so we can get the actual JSON object. 396 old_results = StripJSONWrapper(old_results) 397 398 try: 399 results_json = json.loads(old_results) 400 except Exception: # pylint: disable=broad-except 401 _log.debug('results.json was not valid JSON. Clobbering.') 402 # The JSON file is not valid JSON. Just clobber the results. 403 results_json = {} 404 else: 405 _log.debug('Old JSON results do not exist. Starting fresh.') 406 results_json = {} 407 408 return results_json, error 409 410 def _InsertFailureSummaries(self, results_for_builder): 411 """Inserts aggregate pass/failure statistics into the JSON. 412 This method reads self._test_results and generates 413 FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries. 414 415 Args: 416 results_for_builder: Dictionary containing the test results for a 417 single builder. 418 """ 419 # Insert the number of tests that failed or skipped. 420 fixable_count = len([r for r in self._test_results if r.Fixable()]) 421 self._InsertItemIntoRawList(results_for_builder, 422 fixable_count, self.FIXABLE_COUNT) 423 424 # Create a test modifiers (FAILS, FLAKY etc) summary dictionary. 425 entry = {} 426 for test_name in self._test_results_map.iterkeys(): 427 result_char = self._GetModifierChar(test_name) 428 entry[result_char] = entry.get(result_char, 0) + 1 429 430 # Insert the pass/skip/failure summary dictionary. 431 self._InsertItemIntoRawList(results_for_builder, entry, 432 self.FIXABLE) 433 434 # Insert the number of all the tests that are supposed to pass. 435 all_test_count = len(self._test_results) 436 self._InsertItemIntoRawList(results_for_builder, 437 all_test_count, self.ALL_FIXABLE_COUNT) 438 439 def _InsertItemIntoRawList(self, results_for_builder, item, key): 440 """Inserts the item into the list with the given key in the results for 441 this builder. Creates the list if no such list exists. 442 443 Args: 444 results_for_builder: Dictionary containing the test results for a 445 single builder. 446 item: Number or string to insert into the list. 447 key: Key in results_for_builder for the list to insert into. 448 """ 449 if key in results_for_builder: 450 raw_list = results_for_builder[key] 451 else: 452 raw_list = [] 453 454 raw_list.insert(0, item) 455 raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] 456 results_for_builder[key] = raw_list 457 458 def _InsertItemRunLengthEncoded(self, item, encoded_results): 459 """Inserts the item into the run-length encoded results. 460 461 Args: 462 item: String or number to insert. 463 encoded_results: run-length encoded results. An array of arrays, e.g. 464 [[3,'A'],[1,'Q']] encodes AAAQ. 465 """ 466 if len(encoded_results) and item == encoded_results[0][1]: 467 num_results = encoded_results[0][0] 468 if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 469 encoded_results[0][0] = num_results + 1 470 else: 471 # Use a list instead of a class for the run-length encoding since 472 # we want the serialized form to be concise. 473 encoded_results.insert(0, [1, item]) 474 475 def _InsertGenericMetaData(self, results_for_builder): 476 """ Inserts generic metadata (such as version number, current time etc) 477 into the JSON. 478 479 Args: 480 results_for_builder: Dictionary containing the test results for 481 a single builder. 482 """ 483 self._InsertItemIntoRawList(results_for_builder, 484 self._build_number, self.BUILD_NUMBERS) 485 486 # Include SVN revisions for the given repositories. 487 for (name, path) in self._svn_repositories: 488 # Note: for JSON file's backward-compatibility we use 'chrome' rather 489 # than 'chromium' here. 490 lowercase_name = name.lower() 491 if lowercase_name == 'chromium': 492 lowercase_name = 'chrome' 493 self._InsertItemIntoRawList(results_for_builder, 494 self._GetSVNRevision(path), 495 lowercase_name + 'Revision') 496 497 self._InsertItemIntoRawList(results_for_builder, 498 int(time.time()), 499 self.TIME) 500 501 def _InsertTestTimeAndResult(self, test_name, tests): 502 """ Insert a test item with its results to the given tests dictionary. 503 504 Args: 505 tests: Dictionary containing test result entries. 506 """ 507 508 result = self._get_result_char(test_name) 509 test_time = self._GetTestTiming(test_name) 510 511 this_test = tests 512 for segment in test_name.split('/'): 513 if segment not in this_test: 514 this_test[segment] = {} 515 this_test = this_test[segment] 516 517 if not len(this_test): 518 self._PopulateResultsAndTimesJSON(this_test) 519 520 if self.RESULTS in this_test: 521 self._InsertItemRunLengthEncoded(result, this_test[self.RESULTS]) 522 else: 523 this_test[self.RESULTS] = [[1, result]] 524 525 if self.TIMES in this_test: 526 self._InsertItemRunLengthEncoded(test_time, this_test[self.TIMES]) 527 else: 528 this_test[self.TIMES] = [[1, test_time]] 529 530 def _ConvertJSONToCurrentVersion(self, results_json): 531 """If the JSON does not match the current version, converts it to the 532 current version and adds in the new version number. 533 """ 534 if self.VERSION_KEY in results_json: 535 archive_version = results_json[self.VERSION_KEY] 536 if archive_version == self.VERSION: 537 return 538 else: 539 archive_version = 3 540 541 # version 3->4 542 if archive_version == 3: 543 for results in results_json.values(): 544 self._ConvertTestsToTrie(results) 545 546 results_json[self.VERSION_KEY] = self.VERSION 547 548 def _ConvertTestsToTrie(self, results): 549 if not self.TESTS in results: 550 return 551 552 test_results = results[self.TESTS] 553 test_results_trie = {} 554 for test in test_results.iterkeys(): 555 single_test_result = test_results[test] 556 AddPathToTrie(test, single_test_result, test_results_trie) 557 558 results[self.TESTS] = test_results_trie 559 560 def _PopulateResultsAndTimesJSON(self, results_and_times): 561 results_and_times[self.RESULTS] = [] 562 results_and_times[self.TIMES] = [] 563 return results_and_times 564 565 def _CreateResultsForBuilderJSON(self): 566 results_for_builder = {} 567 results_for_builder[self.TESTS] = {} 568 return results_for_builder 569 570 def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list): 571 """Removes items from the run-length encoded list after the final 572 item that exceeds the max number of builds to track. 573 574 Args: 575 encoded_results: run-length encoded results. An array of arrays, e.g. 576 [[3,'A'],[1,'Q']] encodes AAAQ. 577 """ 578 num_builds = 0 579 index = 0 580 for result in encoded_list: 581 num_builds = num_builds + result[0] 582 index = index + 1 583 if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 584 return encoded_list[:index] 585 return encoded_list 586 587 def _NormalizeResultsJSON(self, test, test_name, tests): 588 """ Prune tests where all runs pass or tests that no longer exist and 589 truncate all results to maxNumberOfBuilds. 590 591 Args: 592 test: ResultsAndTimes object for this test. 593 test_name: Name of the test. 594 tests: The JSON object with all the test results for this builder. 595 """ 596 test[self.RESULTS] = self._RemoveItemsOverMaxNumberOfBuilds( 597 test[self.RESULTS]) 598 test[self.TIMES] = self._RemoveItemsOverMaxNumberOfBuilds( 599 test[self.TIMES]) 600 601 is_all_pass = self._IsResultsAllOfType(test[self.RESULTS], 602 self.PASS_RESULT) 603 is_all_no_data = self._IsResultsAllOfType(test[self.RESULTS], 604 self.NO_DATA_RESULT) 605 max_time = max([test_time[1] for test_time in test[self.TIMES]]) 606 607 # Remove all passes/no-data from the results to reduce noise and 608 # filesize. If a test passes every run, but takes > MIN_TIME to run, 609 # don't throw away the data. 610 if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME): 611 del tests[test_name] 612 613 # method could be a function pylint: disable=R0201 614 def _IsResultsAllOfType(self, results, result_type): 615 """Returns whether all the results are of the given type 616 (e.g. all passes).""" 617 return len(results) == 1 and results[0][1] == result_type 618 619 620 class _FileUploader(object): 621 622 def __init__(self, url, timeout_seconds): 623 self._url = url 624 self._timeout_seconds = timeout_seconds 625 626 def UploadAsMultipartFormData(self, files, attrs): 627 file_objs = [] 628 for filename, path in files: 629 with file(path, 'rb') as fp: 630 file_objs.append(('file', filename, fp.read())) 631 632 # FIXME: We should use the same variable names for the formal and actual 633 # parameters. 634 content_type, data = _EncodeMultipartFormData(attrs, file_objs) 635 return self._UploadData(content_type, data) 636 637 def _UploadData(self, content_type, data): 638 start = time.time() 639 end = start + self._timeout_seconds 640 while time.time() < end: 641 try: 642 request = urllib2.Request(self._url, data, 643 {'Content-Type': content_type}) 644 return urllib2.urlopen(request) 645 except urllib2.HTTPError as e: 646 _log.warn("Received HTTP status %s loading \"%s\". " 647 'Retrying in 10 seconds...', e.code, e.filename) 648 time.sleep(10) 649 650 651 def _GetMIMEType(filename): 652 return mimetypes.guess_type(filename)[0] or 'application/octet-stream' 653 654 655 # FIXME: Rather than taking tuples, this function should take more 656 # structured data. 657 def _EncodeMultipartFormData(fields, files): 658 """Encode form fields for multipart/form-data. 659 660 Args: 661 fields: A sequence of (name, value) elements for regular form fields. 662 files: A sequence of (name, filename, value) elements for data to be 663 uploaded as files. 664 Returns: 665 (content_type, body) ready for httplib.HTTP instance. 666 667 Source: 668 http://code.google.com/p/rietveld/source/browse/trunk/upload.py 669 """ 670 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' 671 CRLF = '\r\n' 672 lines = [] 673 674 for key, value in fields: 675 lines.append('--' + BOUNDARY) 676 lines.append('Content-Disposition: form-data; name="%s"' % key) 677 lines.append('') 678 if isinstance(value, unicode): 679 value = value.encode('utf-8') 680 lines.append(value) 681 682 for key, filename, value in files: 683 lines.append('--' + BOUNDARY) 684 lines.append('Content-Disposition: form-data; name="%s"; ' 685 'filename="%s"' % (key, filename)) 686 lines.append('Content-Type: %s' % _GetMIMEType(filename)) 687 lines.append('') 688 if isinstance(value, unicode): 689 value = value.encode('utf-8') 690 lines.append(value) 691 692 lines.append('--' + BOUNDARY + '--') 693 lines.append('') 694 body = CRLF.join(lines) 695 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY 696 return content_type, body 697
