1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Uploads the results to the flakiness dashboard server.""" 6 7 import logging 8 import os 9 import shutil 10 import subprocess 11 import sys 12 import tempfile 13 import xml 14 15 16 # Include path when ran from a Chromium checkout. 17 sys.path.append( 18 os.path.abspath(os.path.join(os.path.dirname(__file__), 19 os.pardir, os.pardir, os.pardir, os.pardir, 20 'third_party', 'WebKit', 'Tools', 'Scripts'))) 21 22 # Include path when ran from a WebKit checkout. 23 sys.path.append( 24 os.path.abspath(os.path.join(os.path.dirname(__file__), 25 os.pardir, os.pardir, os.pardir, os.pardir, 26 os.pardir, os.pardir, os.pardir, 27 'Tools', 'Scripts'))) 28 29 from webkitpy.common.system import executive, filesystem 30 from webkitpy.layout_tests.layout_package import json_results_generator 31 32 #TODO(craigdh): pylib/utils/ should not depend on pylib/. 33 from pylib import cmd_helper 34 from pylib import constants 35 from pylib.utils import repo_utils 36 37 38 # The JSONResultsGenerator gets the filesystem.join operation from the Port 39 # object. Creating a Port object requires specifying information that only 40 # makes sense for running WebKit layout tests, so we provide a dummy object 41 # that contains the fields required by the generator. 42 class PortDummy(object): 43 def __init__(self): 44 self._executive = executive.Executive() 45 self._filesystem = filesystem.FileSystem() 46 47 48 class JSONResultsGenerator(json_results_generator.JSONResultsGeneratorBase): 49 """Writes test results to a JSON file and handles uploading that file to 50 the test results server. 51 """ 52 def __init__(self, port, builder_name, build_name, build_number, tmp_folder, 53 test_results_map, test_results_server, test_type, master_name): 54 super(JSONResultsGenerator, self).__init__( 55 port=port, 56 builder_name=builder_name, 57 build_name=build_name, 58 build_number=build_number, 59 results_file_base_path=tmp_folder, 60 builder_base_url=None, 61 test_results_map=test_results_map, 62 svn_repositories=(('webkit', 'third_party/WebKit'), 63 ('chrome', '.')), 64 test_results_server=test_results_server, 65 test_type=test_type, 66 master_name=master_name) 67 68 #override 69 def _get_modifier_char(self, test_name): 70 if test_name not in self._test_results_map: 71 return self.__class__.NO_DATA_RESULT 72 73 return self._test_results_map[test_name].modifier 74 75 #override 76 def _get_svn_revision(self, in_directory): 77 """Returns the git/svn revision for the given directory. 78 79 Args: 80 in_directory: The directory relative to src. 81 """ 82 def _is_git_directory(in_directory): 83 """Returns true if the given directory is in a git repository. 84 85 Args: 86 in_directory: The directory path to be tested. 87 """ 88 if os.path.exists(os.path.join(in_directory, '.git')): 89 return True 90 parent = os.path.dirname(in_directory) 91 if parent == constants.DIR_SOURCE_ROOT or parent == in_directory: 92 return False 93 return _is_git_directory(parent) 94 95 in_directory = os.path.join(constants.DIR_SOURCE_ROOT, in_directory) 96 97 if not os.path.exists(os.path.join(in_directory, '.svn')): 98 if _is_git_directory(in_directory): 99 return repo_utils.GetGitHeadSHA1(in_directory) 100 else: 101 return '' 102 103 output = cmd_helper.GetCmdOutput(['svn', 'info', '--xml'], cwd=in_directory) 104 try: 105 dom = xml.dom.minidom.parseString(output) 106 return dom.getElementsByTagName('entry')[0].getAttribute('revision') 107 except xml.parsers.expat.ExpatError: 108 return '' 109 return '' 110 111 112 class ResultsUploader(object): 113 """Handles uploading buildbot tests results to the flakiness dashboard.""" 114 def __init__(self, tests_type): 115 self._build_number = os.environ.get('BUILDBOT_BUILDNUMBER') 116 self._builder_name = os.environ.get('BUILDBOT_BUILDERNAME') 117 self._tests_type = tests_type 118 119 if not self._build_number or not self._builder_name: 120 raise Exception('You should not be uploading tests results to the server' 121 'from your local machine.') 122 123 upstream = (tests_type != 'Chromium_Android_Instrumentation') 124 if upstream: 125 # TODO(frankf): Use factory properties (see buildbot/bb_device_steps.py) 126 # This requires passing the actual master name (e.g. 'ChromiumFYI' not 127 # 'chromium.fyi'). 128 from slave import slave_utils 129 self._build_name = slave_utils.SlaveBuildName(constants.DIR_SOURCE_ROOT) 130 self._master_name = slave_utils.GetActiveMaster() 131 else: 132 self._build_name = 'chromium-android' 133 buildbot_branch = os.environ.get('BUILDBOT_BRANCH') 134 if not buildbot_branch: 135 buildbot_branch = 'master' 136 self._master_name = '%s-%s' % (self._build_name, buildbot_branch) 137 138 self._test_results_map = {} 139 140 def AddResults(self, test_results): 141 # TODO(frankf): Differentiate between fail/crash/timeouts. 142 conversion_map = [ 143 (test_results.GetPass(), False, 144 json_results_generator.JSONResultsGeneratorBase.PASS_RESULT), 145 (test_results.GetFail(), True, 146 json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT), 147 (test_results.GetCrash(), True, 148 json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT), 149 (test_results.GetTimeout(), True, 150 json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT), 151 (test_results.GetUnknown(), True, 152 json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT), 153 ] 154 155 for results_list, failed, modifier in conversion_map: 156 for single_test_result in results_list: 157 test_result = json_results_generator.TestResult( 158 test=single_test_result.GetName(), 159 failed=failed, 160 elapsed_time=single_test_result.GetDur() / 1000) 161 # The WebKit TestResult object sets the modifier it based on test name. 162 # Since we don't use the same test naming convention as WebKit the 163 # modifier will be wrong, so we need to overwrite it. 164 test_result.modifier = modifier 165 166 self._test_results_map[single_test_result.GetName()] = test_result 167 168 def Upload(self, test_results_server): 169 if not self._test_results_map: 170 return 171 172 tmp_folder = tempfile.mkdtemp() 173 174 try: 175 results_generator = JSONResultsGenerator( 176 port=PortDummy(), 177 builder_name=self._builder_name, 178 build_name=self._build_name, 179 build_number=self._build_number, 180 tmp_folder=tmp_folder, 181 test_results_map=self._test_results_map, 182 test_results_server=test_results_server, 183 test_type=self._tests_type, 184 master_name=self._master_name) 185 186 json_files = ["incremental_results.json", "times_ms.json"] 187 results_generator.generate_json_output() 188 results_generator.generate_times_ms_file() 189 results_generator.upload_json_files(json_files) 190 except Exception as e: 191 logging.error("Uploading results to test server failed: %s." % e); 192 finally: 193 shutil.rmtree(tmp_folder) 194 195 196 def Upload(results, flakiness_dashboard_server, test_type): 197 """Reports test results to the flakiness dashboard for Chrome for Android. 198 199 Args: 200 results: test results. 201 flakiness_dashboard_server: the server to upload the results to. 202 test_type: the type of the tests (as displayed by the flakiness dashboard). 203 """ 204 uploader = ResultsUploader(test_type) 205 uploader.AddResults(results) 206 uploader.Upload(flakiness_dashboard_server) 207
