1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 # 6 # Most of this file was ported over from Blink's 7 # Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 8 # Tools/Scripts/webkitpy/common/net/file_uploader.py 9 # 10 11 import json 12 import logging 13 import mimetypes 14 import os 15 import time 16 import urllib2 17 18 _log = logging.getLogger(__name__) 19 20 _JSON_PREFIX = 'ADD_RESULTS(' 21 _JSON_SUFFIX = ');' 22 23 24 def HasJSONWrapper(string): 25 return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX) 26 27 28 def StripJSONWrapper(json_content): 29 # FIXME: Kill this code once the server returns json instead of jsonp. 30 if HasJSONWrapper(json_content): 31 return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] 32 return json_content 33 34 35 def WriteJSON(json_object, file_path, callback=None): 36 # Specify separators in order to get compact encoding. 37 json_string = json.dumps(json_object, separators=(',', ':')) 38 if callback: 39 json_string = callback + '(' + json_string + ');' 40 with open(file_path, 'w') as fp: 41 fp.write(json_string) 42 43 44 def ConvertTrieToFlatPaths(trie, prefix=None): 45 """Flattens the trie of paths, prepending a prefix to each.""" 46 result = {} 47 for name, data in trie.iteritems(): 48 if prefix: 49 name = prefix + '/' + name 50 51 if len(data) and not 'results' in data: 52 result.update(ConvertTrieToFlatPaths(data, name)) 53 else: 54 result[name] = data 55 56 return result 57 58 59 def AddPathToTrie(path, value, trie): 60 """Inserts a single path and value into a directory trie structure.""" 61 if not '/' in path: 62 trie[path] = value 63 return 64 65 directory, _slash, rest = path.partition('/') 66 if not directory in trie: 67 trie[directory] = {} 68 AddPathToTrie(rest, value, trie[directory]) 69 70 71 def TestTimingsTrie(individual_test_timings): 72 """Breaks a test name into dicts by directory 73 74 foo/bar/baz.html: 1ms 75 foo/bar/baz1.html: 3ms 76 77 becomes 78 foo: { 79 bar: { 80 baz.html: 1, 81 baz1.html: 3 82 } 83 } 84 """ 85 trie = {} 86 for test_result in individual_test_timings: 87 test = test_result.test_name 88 89 AddPathToTrie(test, int(1000 * test_result.test_run_time), trie) 90 91 return trie 92 93 94 class TestResult(object): 95 """A simple class that represents a single test result.""" 96 97 # Test modifier constants. 98 (NONE, FAILS, FLAKY, DISABLED) = range(4) 99 100 def __init__(self, test, failed=False, elapsed_time=0): 101 self.test_name = test 102 self.failed = failed 103 self.test_run_time = elapsed_time 104 105 test_name = test 106 try: 107 test_name = test.split('.')[1] 108 except IndexError: 109 _log.warn('Invalid test name: %s.', test) 110 111 if test_name.startswith('FAILS_'): 112 self.modifier = self.FAILS 113 elif test_name.startswith('FLAKY_'): 114 self.modifier = self.FLAKY 115 elif test_name.startswith('DISABLED_'): 116 self.modifier = self.DISABLED 117 else: 118 self.modifier = self.NONE 119 120 def Fixable(self): 121 return self.failed or self.modifier == self.DISABLED 122 123 124 class JSONResultsGeneratorBase(object): 125 """A JSON results generator for generic tests.""" 126 127 MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 128 # Min time (seconds) that will be added to the JSON. 129 MIN_TIME = 1 130 131 # Note that in non-chromium tests those chars are used to indicate 132 # test modifiers (FAILS, FLAKY, etc) but not actual test results. 133 PASS_RESULT = 'P' 134 SKIP_RESULT = 'X' 135 FAIL_RESULT = 'F' 136 FLAKY_RESULT = 'L' 137 NO_DATA_RESULT = 'N' 138 139 MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT, 140 TestResult.DISABLED: SKIP_RESULT, 141 TestResult.FAILS: FAIL_RESULT, 142 TestResult.FLAKY: FLAKY_RESULT} 143 144 VERSION = 4 145 VERSION_KEY = 'version' 146 RESULTS = 'results' 147 TIMES = 'times' 148 BUILD_NUMBERS = 'buildNumbers' 149 TIME = 'secondsSinceEpoch' 150 TESTS = 'tests' 151 152 FIXABLE_COUNT = 'fixableCount' 153 FIXABLE = 'fixableCounts' 154 ALL_FIXABLE_COUNT = 'allFixableCount' 155 156 RESULTS_FILENAME = 'results.json' 157 TIMES_MS_FILENAME = 'times_ms.json' 158 INCREMENTAL_RESULTS_FILENAME = 'incremental_results.json' 159 160 # line too long pylint: disable=C0301 161 URL_FOR_TEST_LIST_JSON = ( 162 'http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&master=%s') 163 # pylint: enable=C0301 164 165 def __init__(self, builder_name, build_name, build_number, 166 results_file_base_path, builder_base_url, 167 test_results_map, svn_repositories=None, 168 test_results_server=None, 169 test_type='', 170 master_name=''): 171 """Modifies the results.json file. Grabs it off the archive directory 172 if it is not found locally. 173 174 Args 175 builder_name: the builder name (e.g. Webkit). 176 build_name: the build name (e.g. webkit-rel). 177 build_number: the build number. 178 results_file_base_path: Absolute path to the directory containing the 179 results json file. 180 builder_base_url: the URL where we have the archived test results. 181 If this is None no archived results will be retrieved. 182 test_results_map: A dictionary that maps test_name to TestResult. 183 svn_repositories: A (json_field_name, svn_path) pair for SVN 184 repositories that tests rely on. The SVN revision will be 185 included in the JSON with the given json_field_name. 186 test_results_server: server that hosts test results json. 187 test_type: test type string (e.g. 'layout-tests'). 188 master_name: the name of the buildbot master. 189 """ 190 self._builder_name = builder_name 191 self._build_name = build_name 192 self._build_number = build_number 193 self._builder_base_url = builder_base_url 194 self._results_directory = results_file_base_path 195 196 self._test_results_map = test_results_map 197 self._test_results = test_results_map.values() 198 199 self._svn_repositories = svn_repositories 200 if not self._svn_repositories: 201 self._svn_repositories = {} 202 203 self._test_results_server = test_results_server 204 self._test_type = test_type 205 self._master_name = master_name 206 207 self._archived_results = None 208 209 def GenerateJSONOutput(self): 210 json_object = self.GetJSON() 211 if json_object: 212 file_path = ( 213 os.path.join( 214 self._results_directory, 215 self.INCREMENTAL_RESULTS_FILENAME)) 216 WriteJSON(json_object, file_path) 217 218 def GenerateTimesMSFile(self): 219 times = TestTimingsTrie(self._test_results_map.values()) 220 file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME) 221 WriteJSON(times, file_path) 222 223 def GetJSON(self): 224 """Gets the results for the results.json file.""" 225 results_json = {} 226 227 if not results_json: 228 results_json, error = self._GetArchivedJSONResults() 229 if error: 230 # If there was an error don't write a results.json 231 # file at all as it would lose all the information on the 232 # bot. 233 _log.error('Archive directory is inaccessible. Not ' 234 'modifying or clobbering the results.json ' 235 'file: ' + str(error)) 236 return None 237 238 builder_name = self._builder_name 239 if results_json and builder_name not in results_json: 240 _log.debug('Builder name (%s) is not in the results.json file.' 241 % builder_name) 242 243 self._ConvertJSONToCurrentVersion(results_json) 244 245 if builder_name not in results_json: 246 results_json[builder_name] = ( 247 self._CreateResultsForBuilderJSON()) 248 249 results_for_builder = results_json[builder_name] 250 251 if builder_name: 252 self._InsertGenericMetaData(results_for_builder) 253 254 self._InsertFailureSummaries(results_for_builder) 255 256 # Update the all failing tests with result type and time. 257 tests = results_for_builder[self.TESTS] 258 all_failing_tests = self._GetFailedTestNames() 259 all_failing_tests.update(ConvertTrieToFlatPaths(tests)) 260 261 for test in all_failing_tests: 262 self._InsertTestTimeAndResult(test, tests) 263 264 return results_json 265 266 def SetArchivedResults(self, archived_results): 267 self._archived_results = archived_results 268 269 def UploadJSONFiles(self, json_files): 270 """Uploads the given json_files to the test_results_server (if the 271 test_results_server is given).""" 272 if not self._test_results_server: 273 return 274 275 if not self._master_name: 276 _log.error( 277 '--test-results-server was set, but --master-name was not. Not ' 278 'uploading JSON files.') 279 return 280 281 _log.info('Uploading JSON files for builder: %s', self._builder_name) 282 attrs = [('builder', self._builder_name), 283 ('testtype', self._test_type), 284 ('master', self._master_name)] 285 286 files = [(json_file, os.path.join(self._results_directory, json_file)) 287 for json_file in json_files] 288 289 url = 'http://%s/testfile/upload' % self._test_results_server 290 # Set uploading timeout in case appengine server is having problems. 291 # 120 seconds are more than enough to upload test results. 292 uploader = _FileUploader(url, 120) 293 try: 294 response = uploader.UploadAsMultipartFormData(files, attrs) 295 if response: 296 if response.code == 200: 297 _log.info('JSON uploaded.') 298 else: 299 _log.debug( 300 "JSON upload failed, %d: '%s'" % 301 (response.code, response.read())) 302 else: 303 _log.error('JSON upload failed; no response returned') 304 except Exception, err: 305 _log.error('Upload failed: %s' % err) 306 return 307 308 def _GetTestTiming(self, test_name): 309 """Returns test timing data (elapsed time) in second 310 for the given test_name.""" 311 if test_name in self._test_results_map: 312 # Floor for now to get time in seconds. 313 return int(self._test_results_map[test_name].test_run_time) 314 return 0 315 316 def _GetFailedTestNames(self): 317 """Returns a set of failed test names.""" 318 return set([r.test_name for r in self._test_results if r.failed]) 319 320 def _GetModifierChar(self, test_name): 321 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 322 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier 323 for the given test_name. 324 """ 325 if test_name not in self._test_results_map: 326 return self.__class__.NO_DATA_RESULT 327 328 test_result = self._test_results_map[test_name] 329 if test_result.modifier in self.MODIFIER_TO_CHAR.keys(): 330 return self.MODIFIER_TO_CHAR[test_result.modifier] 331 332 return self.__class__.PASS_RESULT 333 334 def _get_result_char(self, test_name): 335 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 336 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result 337 for the given test_name. 338 """ 339 if test_name not in self._test_results_map: 340 return self.__class__.NO_DATA_RESULT 341 342 test_result = self._test_results_map[test_name] 343 if test_result.modifier == TestResult.DISABLED: 344 return self.__class__.SKIP_RESULT 345 346 if test_result.failed: 347 return self.__class__.FAIL_RESULT 348 349 return self.__class__.PASS_RESULT 350 351 def _GetSVNRevision(self, in_directory): 352 """Returns the svn revision for the given directory. 353 354 Args: 355 in_directory: The directory where svn is to be run. 356 """ 357 # This is overridden in flakiness_dashboard_results_uploader.py. 358 raise NotImplementedError() 359 360 def _GetArchivedJSONResults(self): 361 """Download JSON file that only contains test 362 name list from test-results server. This is for generating incremental 363 JSON so the file generated has info for tests that failed before but 364 pass or are skipped from current run. 365 366 Returns (archived_results, error) tuple where error is None if results 367 were successfully read. 368 """ 369 results_json = {} 370 old_results = None 371 error = None 372 373 if not self._test_results_server: 374 return {}, None 375 376 results_file_url = (self.URL_FOR_TEST_LIST_JSON % 377 (urllib2.quote(self._test_results_server), 378 urllib2.quote(self._builder_name), 379 self.RESULTS_FILENAME, 380 urllib2.quote(self._test_type), 381 urllib2.quote(self._master_name))) 382 383 try: 384 # FIXME: We should talk to the network via a Host object. 385 results_file = urllib2.urlopen(results_file_url) 386 old_results = results_file.read() 387 except urllib2.HTTPError, http_error: 388 # A non-4xx status code means the bot is hosed for some reason 389 # and we can't grab the results.json file off of it. 390 if (http_error.code < 400 and http_error.code >= 500): 391 error = http_error 392 except urllib2.URLError, url_error: 393 error = url_error 394 395 if old_results: 396 # Strip the prefix and suffix so we can get the actual JSON object. 397 old_results = StripJSONWrapper(old_results) 398 399 try: 400 results_json = json.loads(old_results) 401 except Exception: 402 _log.debug('results.json was not valid JSON. Clobbering.') 403 # The JSON file is not valid JSON. Just clobber the results. 404 results_json = {} 405 else: 406 _log.debug('Old JSON results do not exist. Starting fresh.') 407 results_json = {} 408 409 return results_json, error 410 411 def _InsertFailureSummaries(self, results_for_builder): 412 """Inserts aggregate pass/failure statistics into the JSON. 413 This method reads self._test_results and generates 414 FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries. 415 416 Args: 417 results_for_builder: Dictionary containing the test results for a 418 single builder. 419 """ 420 # Insert the number of tests that failed or skipped. 421 fixable_count = len([r for r in self._test_results if r.Fixable()]) 422 self._InsertItemIntoRawList(results_for_builder, 423 fixable_count, self.FIXABLE_COUNT) 424 425 # Create a test modifiers (FAILS, FLAKY etc) summary dictionary. 426 entry = {} 427 for test_name in self._test_results_map.iterkeys(): 428 result_char = self._GetModifierChar(test_name) 429 entry[result_char] = entry.get(result_char, 0) + 1 430 431 # Insert the pass/skip/failure summary dictionary. 432 self._InsertItemIntoRawList(results_for_builder, entry, 433 self.FIXABLE) 434 435 # Insert the number of all the tests that are supposed to pass. 436 all_test_count = len(self._test_results) 437 self._InsertItemIntoRawList(results_for_builder, 438 all_test_count, self.ALL_FIXABLE_COUNT) 439 440 def _InsertItemIntoRawList(self, results_for_builder, item, key): 441 """Inserts the item into the list with the given key in the results for 442 this builder. Creates the list if no such list exists. 443 444 Args: 445 results_for_builder: Dictionary containing the test results for a 446 single builder. 447 item: Number or string to insert into the list. 448 key: Key in results_for_builder for the list to insert into. 449 """ 450 if key in results_for_builder: 451 raw_list = results_for_builder[key] 452 else: 453 raw_list = [] 454 455 raw_list.insert(0, item) 456 raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] 457 results_for_builder[key] = raw_list 458 459 def _InsertItemRunLengthEncoded(self, item, encoded_results): 460 """Inserts the item into the run-length encoded results. 461 462 Args: 463 item: String or number to insert. 464 encoded_results: run-length encoded results. An array of arrays, e.g. 465 [[3,'A'],[1,'Q']] encodes AAAQ. 466 """ 467 if len(encoded_results) and item == encoded_results[0][1]: 468 num_results = encoded_results[0][0] 469 if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 470 encoded_results[0][0] = num_results + 1 471 else: 472 # Use a list instead of a class for the run-length encoding since 473 # we want the serialized form to be concise. 474 encoded_results.insert(0, [1, item]) 475 476 def _InsertGenericMetaData(self, results_for_builder): 477 """ Inserts generic metadata (such as version number, current time etc) 478 into the JSON. 479 480 Args: 481 results_for_builder: Dictionary containing the test results for 482 a single builder. 483 """ 484 self._InsertItemIntoRawList(results_for_builder, 485 self._build_number, self.BUILD_NUMBERS) 486 487 # Include SVN revisions for the given repositories. 488 for (name, path) in self._svn_repositories: 489 # Note: for JSON file's backward-compatibility we use 'chrome' rather 490 # than 'chromium' here. 491 lowercase_name = name.lower() 492 if lowercase_name == 'chromium': 493 lowercase_name = 'chrome' 494 self._InsertItemIntoRawList(results_for_builder, 495 self._GetSVNRevision(path), 496 lowercase_name + 'Revision') 497 498 self._InsertItemIntoRawList(results_for_builder, 499 int(time.time()), 500 self.TIME) 501 502 def _InsertTestTimeAndResult(self, test_name, tests): 503 """ Insert a test item with its results to the given tests dictionary. 504 505 Args: 506 tests: Dictionary containing test result entries. 507 """ 508 509 result = self._get_result_char(test_name) 510 test_time = self._GetTestTiming(test_name) 511 512 this_test = tests 513 for segment in test_name.split('/'): 514 if segment not in this_test: 515 this_test[segment] = {} 516 this_test = this_test[segment] 517 518 if not len(this_test): 519 self._PopulateResutlsAndTimesJSON(this_test) 520 521 if self.RESULTS in this_test: 522 self._InsertItemRunLengthEncoded(result, this_test[self.RESULTS]) 523 else: 524 this_test[self.RESULTS] = [[1, result]] 525 526 if self.TIMES in this_test: 527 self._InsertItemRunLengthEncoded(test_time, this_test[self.TIMES]) 528 else: 529 this_test[self.TIMES] = [[1, test_time]] 530 531 def _ConvertJSONToCurrentVersion(self, results_json): 532 """If the JSON does not match the current version, converts it to the 533 current version and adds in the new version number. 534 """ 535 if self.VERSION_KEY in results_json: 536 archive_version = results_json[self.VERSION_KEY] 537 if archive_version == self.VERSION: 538 return 539 else: 540 archive_version = 3 541 542 # version 3->4 543 if archive_version == 3: 544 for results in results_json.values(): 545 self._ConvertTestsToTrie(results) 546 547 results_json[self.VERSION_KEY] = self.VERSION 548 549 def _ConvertTestsToTrie(self, results): 550 if not self.TESTS in results: 551 return 552 553 test_results = results[self.TESTS] 554 test_results_trie = {} 555 for test in test_results.iterkeys(): 556 single_test_result = test_results[test] 557 AddPathToTrie(test, single_test_result, test_results_trie) 558 559 results[self.TESTS] = test_results_trie 560 561 def _PopulateResutlsAndTimesJSON(self, results_and_times): 562 results_and_times[self.RESULTS] = [] 563 results_and_times[self.TIMES] = [] 564 return results_and_times 565 566 def _CreateResultsForBuilderJSON(self): 567 results_for_builder = {} 568 results_for_builder[self.TESTS] = {} 569 return results_for_builder 570 571 def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list): 572 """Removes items from the run-length encoded list after the final 573 item that exceeds the max number of builds to track. 574 575 Args: 576 encoded_results: run-length encoded results. An array of arrays, e.g. 577 [[3,'A'],[1,'Q']] encodes AAAQ. 578 """ 579 num_builds = 0 580 index = 0 581 for result in encoded_list: 582 num_builds = num_builds + result[0] 583 index = index + 1 584 if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 585 return encoded_list[:index] 586 return encoded_list 587 588 def _NormalizeResultsJSON(self, test, test_name, tests): 589 """ Prune tests where all runs pass or tests that no longer exist and 590 truncate all results to maxNumberOfBuilds. 591 592 Args: 593 test: ResultsAndTimes object for this test. 594 test_name: Name of the test. 595 tests: The JSON object with all the test results for this builder. 596 """ 597 test[self.RESULTS] = self._RemoveItemsOverMaxNumberOfBuilds( 598 test[self.RESULTS]) 599 test[self.TIMES] = self._RemoveItemsOverMaxNumberOfBuilds( 600 test[self.TIMES]) 601 602 is_all_pass = self._IsResultsAllOfType(test[self.RESULTS], 603 self.PASS_RESULT) 604 is_all_no_data = self._IsResultsAllOfType(test[self.RESULTS], 605 self.NO_DATA_RESULT) 606 max_time = max([test_time[1] for test_time in test[self.TIMES]]) 607 608 # Remove all passes/no-data from the results to reduce noise and 609 # filesize. If a test passes every run, but takes > MIN_TIME to run, 610 # don't throw away the data. 611 if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME): 612 del tests[test_name] 613 614 # method could be a function pylint: disable=R0201 615 def _IsResultsAllOfType(self, results, result_type): 616 """Returns whether all the results are of the given type 617 (e.g. all passes).""" 618 return len(results) == 1 and results[0][1] == result_type 619 620 621 class _FileUploader(object): 622 623 def __init__(self, url, timeout_seconds): 624 self._url = url 625 self._timeout_seconds = timeout_seconds 626 627 def UploadAsMultipartFormData(self, files, attrs): 628 file_objs = [] 629 for filename, path in files: 630 with file(path, 'rb') as fp: 631 file_objs.append(('file', filename, fp.read())) 632 633 # FIXME: We should use the same variable names for the formal and actual 634 # parameters. 635 content_type, data = _EncodeMultipartFormData(attrs, file_objs) 636 return self._UploadData(content_type, data) 637 638 def _UploadData(self, content_type, data): 639 start = time.time() 640 end = start + self._timeout_seconds 641 while time.time() < end: 642 try: 643 request = urllib2.Request(self._url, data, 644 {'Content-Type': content_type}) 645 return urllib2.urlopen(request) 646 except urllib2.HTTPError as e: 647 _log.warn("Received HTTP status %s loading \"%s\". " 648 'Retrying in 10 seconds...' % (e.code, e.filename)) 649 time.sleep(10) 650 651 652 def _GetMIMEType(filename): 653 return mimetypes.guess_type(filename)[0] or 'application/octet-stream' 654 655 656 # FIXME: Rather than taking tuples, this function should take more 657 # structured data. 658 def _EncodeMultipartFormData(fields, files): 659 """Encode form fields for multipart/form-data. 660 661 Args: 662 fields: A sequence of (name, value) elements for regular form fields. 663 files: A sequence of (name, filename, value) elements for data to be 664 uploaded as files. 665 Returns: 666 (content_type, body) ready for httplib.HTTP instance. 667 668 Source: 669 http://code.google.com/p/rietveld/source/browse/trunk/upload.py 670 """ 671 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' 672 CRLF = '\r\n' 673 lines = [] 674 675 for key, value in fields: 676 lines.append('--' + BOUNDARY) 677 lines.append('Content-Disposition: form-data; name="%s"' % key) 678 lines.append('') 679 if isinstance(value, unicode): 680 value = value.encode('utf-8') 681 lines.append(value) 682 683 for key, filename, value in files: 684 lines.append('--' + BOUNDARY) 685 lines.append('Content-Disposition: form-data; name="%s"; ' 686 'filename="%s"' % (key, filename)) 687 lines.append('Content-Type: %s' % _GetMIMEType(filename)) 688 lines.append('') 689 if isinstance(value, unicode): 690 value = value.encode('utf-8') 691 lines.append(value) 692 693 lines.append('--' + BOUNDARY + '--') 694 lines.append('') 695 body = CRLF.join(lines) 696 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY 697 return content_type, body 698
