1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 from datetime import datetime 30 from django.utils import simplejson 31 import logging 32 33 from model.testfile import TestFile 34 35 JSON_RESULTS_FILE = "results.json" 36 JSON_RESULTS_FILE_SMALL = "results-small.json" 37 JSON_RESULTS_PREFIX = "ADD_RESULTS(" 38 JSON_RESULTS_SUFFIX = ");" 39 JSON_RESULTS_VERSION_KEY = "version" 40 JSON_RESULTS_BUILD_NUMBERS = "buildNumbers" 41 JSON_RESULTS_TESTS = "tests" 42 JSON_RESULTS_RESULTS = "results" 43 JSON_RESULTS_TIMES = "times" 44 JSON_RESULTS_PASS = "P" 45 JSON_RESULTS_NO_DATA = "N" 46 JSON_RESULTS_MIN_TIME = 1 47 JSON_RESULTS_VERSION = 3 48 JSON_RESULTS_MAX_BUILDS = 750 49 JSON_RESULTS_MAX_BUILDS_SMALL = 200 50 51 52 class JsonResults(object): 53 @classmethod 54 def _strip_prefix_suffix(cls, data): 55 """Strip out prefix and suffix of json results string. 56 57 Args: 58 data: json file content. 59 60 Returns: 61 json string without prefix and suffix. 62 """ 63 64 assert(data.startswith(JSON_RESULTS_PREFIX)) 65 assert(data.endswith(JSON_RESULTS_SUFFIX)) 66 67 return data[len(JSON_RESULTS_PREFIX): 68 len(data) - len(JSON_RESULTS_SUFFIX)] 69 70 @classmethod 71 def _generate_file_data(cls, json, sort_keys=False): 72 """Given json string, generate file content data by adding 73 prefix and suffix. 74 75 Args: 76 json: json string without prefix and suffix. 77 78 Returns: 79 json file data. 80 """ 81 82 data = simplejson.dumps(json, separators=(',', ':'), 83 sort_keys=sort_keys) 84 return JSON_RESULTS_PREFIX + data + JSON_RESULTS_SUFFIX 85 86 @classmethod 87 def _load_json(cls, file_data): 88 """Load json file to a python object. 89 90 Args: 91 file_data: json file content. 92 93 Returns: 94 json object or 95 None on failure. 96 """ 97 98 json_results_str = cls._strip_prefix_suffix(file_data) 99 if not json_results_str: 100 logging.warning("No json results data.") 101 return None 102 103 try: 104 return simplejson.loads(json_results_str) 105 except Exception, err: 106 logging.debug(json_results_str) 107 logging.error("Failed to load json results: %s", str(err)) 108 return None 109 110 @classmethod 111 def _merge_json(cls, aggregated_json, incremental_json, num_runs): 112 """Merge incremental json into aggregated json results. 113 114 Args: 115 aggregated_json: aggregated json object. 116 incremental_json: incremental json object. 117 num_runs: number of total runs to include. 118 119 Returns: 120 True if merge succeeds or 121 False on failure. 122 """ 123 124 # Merge non tests property data. 125 # Tests properties are merged in _merge_tests. 126 if not cls._merge_non_test_data(aggregated_json, incremental_json, num_runs): 127 return False 128 129 # Merge tests results and times 130 incremental_tests = incremental_json[JSON_RESULTS_TESTS] 131 if incremental_tests: 132 aggregated_tests = aggregated_json[JSON_RESULTS_TESTS] 133 cls._merge_tests(aggregated_tests, incremental_tests, num_runs) 134 135 return True 136 137 @classmethod 138 def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs): 139 """Merge incremental non tests property data into aggregated json results. 140 141 Args: 142 aggregated_json: aggregated json object. 143 incremental_json: incremental json object. 144 num_runs: number of total runs to include. 145 146 Returns: 147 True if merge succeeds or 148 False on failure. 149 """ 150 151 incremental_builds = incremental_json[JSON_RESULTS_BUILD_NUMBERS] 152 aggregated_builds = aggregated_json[JSON_RESULTS_BUILD_NUMBERS] 153 aggregated_build_number = int(aggregated_builds[0]) 154 # Loop through all incremental builds, start from the oldest run. 155 for index in reversed(range(len(incremental_builds))): 156 build_number = int(incremental_builds[index]) 157 logging.debug("Merging build %s, incremental json index: %d.", 158 build_number, index) 159 160 # Return if not all build numbers in the incremental json results 161 # are newer than the most recent build in the aggregated results. 162 # FIXME: make this case work. 163 if build_number < aggregated_build_number: 164 logging.warning(("Build %d in incremental json is older than " 165 "the most recent build in aggregated results: %d"), 166 build_number, aggregated_build_number) 167 return False 168 169 # Return if the build number is duplicated. 170 # FIXME: skip the duplicated build and merge rest of the results. 171 # Need to be careful on skiping the corresponding value in 172 # _merge_tests because the property data for each test could 173 # be accumulated. 174 if build_number == aggregated_build_number: 175 logging.warning("Duplicate build %d in incremental json", 176 build_number) 177 return False 178 179 # Merge this build into aggreagated results. 180 cls._merge_one_build(aggregated_json, incremental_json, index, num_runs) 181 182 return True 183 184 @classmethod 185 def _merge_one_build(cls, aggregated_json, incremental_json, 186 incremental_index, num_runs): 187 """Merge one build of incremental json into aggregated json results. 188 189 Args: 190 aggregated_json: aggregated json object. 191 incremental_json: incremental json object. 192 incremental_index: index of the incremental json results to merge. 193 num_runs: number of total runs to include. 194 """ 195 196 for key in incremental_json.keys(): 197 # Merge json results except "tests" properties (results, times etc). 198 # "tests" properties will be handled separately. 199 if key == JSON_RESULTS_TESTS: 200 continue 201 202 if key in aggregated_json: 203 aggregated_json[key].insert( 204 0, incremental_json[key][incremental_index]) 205 aggregated_json[key] = \ 206 aggregated_json[key][:num_runs] 207 else: 208 aggregated_json[key] = incremental_json[key] 209 210 @classmethod 211 def _merge_tests(cls, aggregated_json, incremental_json, num_runs): 212 """Merge "tests" properties:results, times. 213 214 Args: 215 aggregated_json: aggregated json object. 216 incremental_json: incremental json object. 217 num_runs: number of total runs to include. 218 """ 219 220 all_tests = (set(aggregated_json.iterkeys()) | 221 set(incremental_json.iterkeys())) 222 for test_name in all_tests: 223 if test_name in aggregated_json: 224 aggregated_test = aggregated_json[test_name] 225 if test_name in incremental_json: 226 incremental_test = incremental_json[test_name] 227 results = incremental_test[JSON_RESULTS_RESULTS] 228 times = incremental_test[JSON_RESULTS_TIMES] 229 else: 230 results = [[1, JSON_RESULTS_NO_DATA]] 231 times = [[1, 0]] 232 233 cls._insert_item_run_length_encoded( 234 results, aggregated_test[JSON_RESULTS_RESULTS], num_runs) 235 cls._insert_item_run_length_encoded( 236 times, aggregated_test[JSON_RESULTS_TIMES], num_runs) 237 cls._normalize_results_json(test_name, aggregated_json, num_runs) 238 else: 239 aggregated_json[test_name] = incremental_json[test_name] 240 241 @classmethod 242 def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs): 243 """Inserts the incremental run-length encoded results into the aggregated 244 run-length encoded results. 245 246 Args: 247 incremental_item: incremental run-length encoded results. 248 aggregated_item: aggregated run-length encoded results. 249 num_runs: number of total runs to include. 250 """ 251 252 for item in incremental_item: 253 if len(aggregated_item) and item[1] == aggregated_item[0][1]: 254 aggregated_item[0][0] = min( 255 aggregated_item[0][0] + item[0], num_runs) 256 else: 257 aggregated_item.insert(0, item) 258 259 @classmethod 260 def _normalize_results_json(cls, test_name, aggregated_json, num_runs): 261 """ Prune tests where all runs pass or tests that no longer exist and 262 truncate all results to num_runs. 263 264 Args: 265 test_name: Name of the test. 266 aggregated_json: The JSON object with all the test results for 267 this builder. 268 num_runs: number of total runs to include. 269 """ 270 271 aggregated_test = aggregated_json[test_name] 272 aggregated_test[JSON_RESULTS_RESULTS] = \ 273 cls._remove_items_over_max_number_of_builds( 274 aggregated_test[JSON_RESULTS_RESULTS], num_runs) 275 aggregated_test[JSON_RESULTS_TIMES] = \ 276 cls._remove_items_over_max_number_of_builds( 277 aggregated_test[JSON_RESULTS_TIMES], num_runs) 278 279 is_all_pass = cls._is_results_all_of_type( 280 aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_PASS) 281 is_all_no_data = cls._is_results_all_of_type( 282 aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_NO_DATA) 283 284 max_time = max( 285 [time[1] for time in aggregated_test[JSON_RESULTS_TIMES]]) 286 # Remove all passes/no-data from the results to reduce noise and 287 # filesize. If a test passes every run, but 288 # takes >= JSON_RESULTS_MIN_TIME to run, don't throw away the data. 289 if (is_all_no_data or 290 (is_all_pass and max_time < JSON_RESULTS_MIN_TIME)): 291 del aggregated_json[test_name] 292 293 @classmethod 294 def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs): 295 """Removes items from the run-length encoded list after the final 296 item that exceeds the max number of builds to track. 297 298 Args: 299 encoded_results: run-length encoded results. An array of arrays, e.g. 300 [[3,'A'],[1,'Q']] encodes AAAQ. 301 num_runs: number of total runs to include. 302 """ 303 num_builds = 0 304 index = 0 305 for result in encoded_list: 306 num_builds = num_builds + result[0] 307 index = index + 1 308 if num_builds >= num_runs: 309 return encoded_list[:index] 310 311 return encoded_list 312 313 @classmethod 314 def _is_results_all_of_type(cls, results, type): 315 """Returns whether all the results are of the given type 316 (e.g. all passes). 317 """ 318 319 return len(results) == 1 and results[0][1] == type 320 321 @classmethod 322 def _check_json(cls, builder, json): 323 """Check whether the given json is valid. 324 325 Args: 326 builder: builder name this json is for. 327 json: json object to check. 328 329 Returns: 330 True if the json is valid or 331 False otherwise. 332 """ 333 334 version = json[JSON_RESULTS_VERSION_KEY] 335 if version > JSON_RESULTS_VERSION: 336 logging.error("Results JSON version '%s' is not supported.", 337 version) 338 return False 339 340 if not builder in json: 341 logging.error("Builder '%s' is not in json results.", builder) 342 return False 343 344 results_for_builder = json[builder] 345 if not JSON_RESULTS_BUILD_NUMBERS in results_for_builder: 346 logging.error("Missing build number in json results.") 347 return False 348 349 return True 350 351 @classmethod 352 def merge(cls, builder, aggregated, incremental, num_runs, sort_keys=False): 353 """Merge incremental json file data with aggregated json file data. 354 355 Args: 356 builder: builder name. 357 aggregated: aggregated json file data. 358 incremental: incremental json file data. 359 sort_key: whether or not to sort key when dumping json results. 360 361 Returns: 362 Merged json file data if merge succeeds or 363 None on failure. 364 """ 365 366 if not incremental: 367 logging.warning("Nothing to merge.") 368 return None 369 370 logging.info("Loading incremental json...") 371 incremental_json = cls._load_json(incremental) 372 if not incremental_json: 373 return None 374 375 logging.info("Checking incremental json...") 376 if not cls._check_json(builder, incremental_json): 377 return None 378 379 logging.info("Loading existing aggregated json...") 380 aggregated_json = cls._load_json(aggregated) 381 if not aggregated_json: 382 return incremental 383 384 logging.info("Checking existing aggregated json...") 385 if not cls._check_json(builder, aggregated_json): 386 return incremental 387 388 logging.info("Merging json results...") 389 try: 390 if not cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs): 391 return None 392 except Exception, err: 393 logging.error("Failed to merge json results: %s", str(err)) 394 return None 395 396 aggregated_json[JSON_RESULTS_VERSION_KEY] = JSON_RESULTS_VERSION 397 398 return cls._generate_file_data(aggregated_json, sort_keys) 399 400 @classmethod 401 def update(cls, master, builder, test_type, incremental): 402 """Update datastore json file data by merging it with incremental json 403 file. Writes the large file and a small file. The small file just stores 404 fewer runs. 405 406 Args: 407 master: master name. 408 builder: builder name. 409 test_type: type of test results. 410 incremental: incremental json file data to merge. 411 412 Returns: 413 Large TestFile object if update succeeds or 414 None on failure. 415 """ 416 small_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE_SMALL, JSON_RESULTS_MAX_BUILDS_SMALL) 417 large_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE, JSON_RESULTS_MAX_BUILDS) 418 419 return small_file_updated and large_file_updated 420 421 @classmethod 422 def update_file(cls, master, builder, test_type, incremental, filename, num_runs): 423 files = TestFile.get_files(master, builder, test_type, filename) 424 if files: 425 file = files[0] 426 new_results = cls.merge(builder, file.data, incremental, num_runs) 427 else: 428 # Use the incremental data if there is no aggregated file to merge. 429 file = TestFile() 430 file.master = master 431 file.builder = builder 432 file.test_type = test_type 433 file.name = filename 434 new_results = incremental 435 logging.info("No existing json results, incremental json is saved.") 436 437 if not new_results or not file.save(new_results): 438 logging.info( 439 "Update failed, master: %s, builder: %s, test_type: %s, name: %s." % 440 (master, builder, test_type, filename)) 441 return False 442 443 return True 444 445 @classmethod 446 def get_test_list(cls, builder, json_file_data): 447 """Get list of test names from aggregated json file data. 448 449 Args: 450 json_file_data: json file data that has all test-data and 451 non-test-data. 452 453 Returns: 454 json file with test name list only. The json format is the same 455 as the one saved in datastore, but all non-test-data and test detail 456 results are removed. 457 """ 458 459 logging.debug("Loading test results json...") 460 json = cls._load_json(json_file_data) 461 if not json: 462 return None 463 464 logging.debug("Checking test results json...") 465 if not cls._check_json(builder, json): 466 return None 467 468 test_list_json = {} 469 tests = json[builder][JSON_RESULTS_TESTS] 470 test_list_json[builder] = { 471 "tests": dict.fromkeys(tests, {})} 472 473 return cls._generate_file_data(test_list_json) 474
