1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Uploads performance data to the performance dashboard. 6 7 Performance tests may output data that needs to be displayed on the performance 8 dashboard. The autotest TKO parser invokes this module with each test 9 associated with a job. If a test has performance data associated with it, it 10 is uploaded to the performance dashboard. The performance dashboard is owned 11 by Chrome team and is available here: https://chromeperf.appspot.com/. Users 12 must be logged in with an @google.com account to view chromeOS perf data there. 13 14 """ 15 16 import httplib, json, math, os, re, urllib, urllib2 17 18 import common 19 from autotest_lib.client.cros import constants 20 from autotest_lib.tko import utils as tko_utils 21 22 _ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 23 _PRESENTATION_CONFIG_FILE = os.path.join( 24 _ROOT_DIR, 'perf_dashboard_config.json') 25 _PRESENTATION_SHADOW_CONFIG_FILE = os.path.join( 26 _ROOT_DIR, 'perf_dashboard_shadow_config.json') 27 _DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point' 28 29 # Format for Chrome and Chrome OS version strings. 30 VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$' 31 32 class PerfUploadingError(Exception): 33 """Exception raised in perf_uploader""" 34 pass 35 36 37 def _aggregate_iterations(perf_values): 38 """Aggregate same measurements from multiple iterations. 39 40 Each perf measurement may exist multiple times across multiple iterations 41 of a test. Here, the results for each unique measured perf metric are 42 aggregated across multiple iterations. 43 44 @param perf_values: A list of tko.models.perf_value_iteration objects. 45 46 @return A dictionary mapping each unique measured perf value (keyed by 47 tuple of its description and graph name) to information about that 48 perf value (in particular, the value is a list of values 49 for each iteration). 50 51 """ 52 perf_data = {} 53 for perf_iteration in perf_values: 54 for perf_dict in perf_iteration.perf_measurements: 55 key = (perf_dict['description'], perf_dict['graph']) 56 if key not in perf_data: 57 perf_data[key] = { 58 'units': perf_dict['units'], 59 'higher_is_better': perf_dict['higher_is_better'], 60 'graph': perf_dict['graph'], 61 'value': [perf_dict['value']], # Note: a list of values. 62 'stddev': perf_dict['stddev'] 63 } 64 else: 65 perf_data[key]['value'].append(perf_dict['value']) 66 # Note: the stddev will be recomputed later when the results 67 # from each of the multiple iterations are averaged together. 68 return perf_data 69 70 71 def _mean_and_stddev(data, precision=4): 72 """Computes mean and standard deviation from a list of numbers. 73 74 Assumes that the list contains at least 2 numbers. 75 76 @param data: A list of numeric values. 77 @param precision: The integer number of decimal places to which to 78 round the results. 79 80 @return A 2-tuple (mean, standard_deviation), in which each value is 81 rounded to |precision| decimal places. 82 83 """ 84 n = len(data) 85 mean = float(sum(data)) / n 86 # Divide by n-1 to compute "sample standard deviation". 87 variance = sum([(elem - mean) ** 2 for elem in data]) / (n - 1) 88 return round(mean, precision), round(math.sqrt(variance), precision) 89 90 91 def _compute_avg_stddev(perf_data): 92 """Compute average and standard deviations as needed for perf measurements. 93 94 For any perf measurement that exists in multiple iterations (has more than 95 one measured value), compute the average and standard deviation for it and 96 then store the updated information in the dictionary. 97 98 @param perf_data: A dictionary of measured perf data as computed by 99 _aggregate_iterations(), except each value is now a single value, not a 100 list of values. 101 102 """ 103 for perf_dict in perf_data.itervalues(): 104 if len(perf_dict['value']) > 1: 105 perf_dict['value'], perf_dict['stddev'] = ( 106 _mean_and_stddev(map(float, perf_dict['value']))) 107 else: 108 perf_dict['value'] = perf_dict['value'][0] # Take out of list. 109 110 111 def _parse_config_file(config_file): 112 """Parses a presentation config file and stores the info into a dict. 113 114 The config file contains information about how to present the perf data 115 on the perf dashboard. This is required if the default presentation 116 settings aren't desired for certain tests. 117 118 @param config_file: Path to the configuration file to be parsed. 119 120 @returns A dictionary mapping each unique autotest name to a dictionary 121 of presentation config information. 122 123 @raises PerfUploadingError if config data or master name for the test 124 is missing from the config file. 125 126 """ 127 json_obj = [] 128 if os.path.exists(config_file): 129 with open(config_file, 'r') as fp: 130 json_obj = json.load(fp) 131 config_dict = {} 132 for entry in json_obj: 133 config_dict[entry['autotest_name']] = entry 134 return config_dict 135 136 137 def _gather_presentation_info(config_data, test_name): 138 """Gathers presentation info from config data for the given test name. 139 140 @param config_data: A dictionary of dashboard presentation info for all 141 tests, as returned by _parse_config_file(). Info is keyed by autotest 142 name. 143 @param test_name: The name of an autotest. 144 145 @return A dictionary containing presentation information extracted from 146 |config_data| for the given autotest name. 147 148 @raises PerfUploadingError if some required data is missing. 149 """ 150 if not test_name in config_data: 151 raise PerfUploadingError( 152 'No config data is specified for test %s in %s.' % 153 (test_name, _PRESENTATION_CONFIG_FILE)) 154 155 presentation_dict = config_data[test_name] 156 try: 157 master_name = presentation_dict['master_name'] 158 except KeyError: 159 raise PerfUploadingError( 160 'No master name is specified for test %s in %s.' % 161 (test_name, _PRESENTATION_CONFIG_FILE)) 162 if 'dashboard_test_name' in presentation_dict: 163 test_name = presentation_dict['dashboard_test_name'] 164 return {'master_name': master_name, 'test_name': test_name} 165 166 167 def _format_for_upload(platform_name, cros_version, chrome_version, 168 hardware_id, variant_name, hardware_hostname, 169 perf_data, presentation_info): 170 """Formats perf data suitably to upload to the perf dashboard. 171 172 The perf dashboard expects perf data to be uploaded as a 173 specially-formatted JSON string. In particular, the JSON object must be a 174 dictionary with key "data", and value being a list of dictionaries where 175 each dictionary contains all the information associated with a single 176 measured perf value: master name, bot name, test name, perf value, error 177 value, units, and build version numbers. 178 179 @param platform_name: The string name of the platform. 180 @param cros_version: The string chromeOS version number. 181 @param chrome_version: The string chrome version number. 182 @param hardware_id: String that identifies the type of hardware the test was 183 executed on. 184 @param variant_name: String that identifies the variant name of the board. 185 @param hardware_hostname: String that identifies the name of the device the 186 test was executed on. 187 @param perf_data: A dictionary of measured perf data as computed by 188 _compute_avg_stddev(). 189 @param presentation_info: A dictionary of dashboard presentation info for 190 the given test, as identified by _gather_presentation_info(). 191 192 @return A dictionary containing the formatted information ready to upload 193 to the performance dashboard. 194 195 """ 196 dash_entries = [] 197 if variant_name: 198 platform_name += '-' + variant_name 199 for (desc, graph), data in perf_data.iteritems(): 200 # Each perf metric is named by a path that encodes the test name, 201 # a graph name (if specified), and a description. This must be defined 202 # according to rules set by the Chrome team, as implemented in: 203 # chromium/tools/build/scripts/slave/results_dashboard.py. 204 if desc.endswith('_ref'): 205 desc = 'ref' 206 desc = desc.replace('_by_url', '') 207 desc = desc.replace('/', '_') 208 if data['graph']: 209 test_path = '%s/%s/%s' % (presentation_info['test_name'], 210 data['graph'], desc) 211 else: 212 test_path = '%s/%s' % (presentation_info['test_name'], desc) 213 214 new_dash_entry = { 215 'master': presentation_info['master_name'], 216 'bot': 'cros-' + platform_name, # Prefix to clarify it's chromeOS. 217 'test': test_path, 218 'value': data['value'], 219 'error': data['stddev'], 220 'units': data['units'], 221 'higher_is_better': data['higher_is_better'], 222 'revision': _get_id_from_version(chrome_version, cros_version), 223 'supplemental_columns': { 224 'r_cros_version': cros_version, 225 'r_chrome_version': chrome_version, 226 'a_default_rev': 'r_chrome_version', 227 'a_hardware_identifier': hardware_id, 228 'a_hardware_hostname': hardware_hostname, 229 } 230 } 231 232 dash_entries.append(new_dash_entry) 233 234 json_string = json.dumps(dash_entries) 235 return {'data': json_string} 236 237 238 def _get_version_numbers(test_attributes): 239 """Gets the version numbers from the test attributes and validates them. 240 241 @param test_attributes: The attributes property (which is a dict) of an 242 autotest tko.models.test object. 243 244 @return A pair of strings (Chrome OS version, Chrome version). 245 246 @raises PerfUploadingError if a version isn't formatted as expected. 247 """ 248 chrome_version = test_attributes.get('CHROME_VERSION', '') 249 cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '') 250 # Prefix the ChromeOS version number with the Chrome milestone. 251 cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version 252 if not re.match(VERSION_REGEXP, cros_version): 253 raise PerfUploadingError('CrOS version "%s" does not match expected ' 254 'format.' % cros_version) 255 if not re.match(VERSION_REGEXP, chrome_version): 256 raise PerfUploadingError('Chrome version "%s" does not match expected ' 257 'format.' % chrome_version) 258 return (cros_version, chrome_version) 259 260 261 def _get_id_from_version(chrome_version, cros_version): 262 """Computes the point ID to use, from Chrome and ChromeOS version numbers. 263 264 For ChromeOS row data, data values are associated with both a Chrome 265 version number and a ChromeOS version number (unlike for Chrome row data 266 that is associated with a single revision number). This function takes 267 both version numbers as input, then computes a single, unique integer ID 268 from them, which serves as a 'fake' revision number that can uniquely 269 identify each ChromeOS data point, and which will allow ChromeOS data points 270 to be sorted by Chrome version number, with ties broken by ChromeOS version 271 number. 272 273 To compute the integer ID, we take the portions of each version number that 274 serve as the shortest unambiguous names for each (as described here: 275 http://www.chromium.org/developers/version-numbers). We then force each 276 component of each portion to be a fixed width (padded by zeros if needed), 277 concatenate all digits together (with those coming from the Chrome version 278 number first), and convert the entire string of digits into an integer. 279 We ensure that the total number of digits does not exceed that which is 280 allowed by AppEngine NDB for an integer (64-bit signed value). 281 282 For example: 283 Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2) 284 ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0) 285 concatenated together with padding for fixed-width columns: 286 ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000' 287 Final integer ID: 14520020390600000 288 289 @param chrome_ver: The Chrome version number as a string. 290 @param cros_ver: The ChromeOS version number as a string. 291 292 @return A unique integer ID associated with the two given version numbers. 293 294 """ 295 296 # Number of digits to use from each part of the version string for Chrome 297 # and Chrome OS versions when building a point ID out of these two versions. 298 chrome_version_col_widths = [0, 0, 5, 3] 299 cros_version_col_widths = [0, 5, 3, 2] 300 301 def get_digits_from_version(version_num, column_widths): 302 if re.match(VERSION_REGEXP, version_num): 303 computed_string = '' 304 version_parts = version_num.split('.') 305 for i, version_part in enumerate(version_parts): 306 if column_widths[i]: 307 computed_string += version_part.zfill(column_widths[i]) 308 return computed_string 309 else: 310 return None 311 312 chrome_digits = get_digits_from_version( 313 chrome_version, chrome_version_col_widths) 314 cros_digits = get_digits_from_version( 315 cros_version, cros_version_col_widths) 316 if not chrome_digits or not cros_digits: 317 return None 318 result_digits = chrome_digits + cros_digits 319 max_digits = sum(chrome_version_col_widths + cros_version_col_widths) 320 if len(result_digits) > max_digits: 321 return None 322 return int(result_digits) 323 324 325 def _send_to_dashboard(data_obj): 326 """Sends formatted perf data to the perf dashboard. 327 328 @param data_obj: A formatted data object as returned by 329 _format_for_upload(). 330 331 @raises PerfUploadingError if an exception was raised when uploading. 332 333 """ 334 encoded = urllib.urlencode(data_obj) 335 req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded) 336 try: 337 urllib2.urlopen(req) 338 except urllib2.HTTPError as e: 339 raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % ( 340 e.code, e.msg, data_obj['data'])) 341 except urllib2.URLError as e: 342 raise PerfUploadingError( 343 'URLError: %s for JSON %s\n' % 344 (str(e.reason), data_obj['data'])) 345 except httplib.HTTPException: 346 raise PerfUploadingError( 347 'HTTPException for JSON %s\n' % data_obj['data']) 348 349 350 def upload_test(job, test): 351 """Uploads any perf data associated with a test to the perf dashboard. 352 353 @param job: An autotest tko.models.job object that is associated with the 354 given |test|. 355 @param test: An autotest tko.models.test object that may or may not be 356 associated with measured perf data. 357 358 """ 359 if not test.perf_values: 360 return 361 362 # Aggregate values from multiple iterations together. 363 perf_data = _aggregate_iterations(test.perf_values) 364 365 # Compute averages and standard deviations as needed for measured perf 366 # values that exist in multiple iterations. Ultimately, we only upload a 367 # single measurement (with standard deviation) for every unique measured 368 # perf metric. 369 _compute_avg_stddev(perf_data) 370 371 # Format the perf data for the upload, then upload it. 372 test_name = test.testname 373 platform_name = job.machine_group 374 hardware_id = test.attributes.get('hwid', '') 375 hardware_hostname = test.machine 376 variant_name = test.attributes.get(constants.VARIANT_KEY, None) 377 config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE) 378 try: 379 shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE) 380 config_data.update(shadow_config_data) 381 except ValueError as e: 382 tko_utils.dprint('Failed to parse config file %s: %s.' % 383 (_PRESENTATION_SHADOW_CONFIG_FILE, e)) 384 try: 385 cros_version, chrome_version = _get_version_numbers(test.attributes) 386 presentation_info = _gather_presentation_info(config_data, test_name) 387 formatted_data = _format_for_upload( 388 platform_name, cros_version, chrome_version, hardware_id, 389 variant_name, hardware_hostname, perf_data, presentation_info) 390 _send_to_dashboard(formatted_data) 391 except PerfUploadingError as e: 392 tko_utils.dprint('Error when uploading perf data to the perf ' 393 'dashboard for test %s: %s' % (test_name, e)) 394 else: 395 tko_utils.dprint('Successfully uploaded perf data to the perf ' 396 'dashboard for test %s.' % test_name) 397
