1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Module to deal with result cache.""" 5 6 from __future__ import print_function 7 8 import glob 9 import hashlib 10 import os 11 import pickle 12 import re 13 import tempfile 14 import json 15 import sys 16 17 from cros_utils import command_executer 18 from cros_utils import misc 19 20 from image_checksummer import ImageChecksummer 21 22 import results_report 23 import test_flag 24 25 SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 26 RESULTS_FILE = 'results.txt' 27 MACHINE_FILE = 'machine.txt' 28 AUTOTEST_TARBALL = 'autotest.tbz2' 29 PERF_RESULTS_FILE = 'perf-results.txt' 30 CACHE_KEYS_FILE = 'cache_keys.txt' 31 32 33 class Result(object): 34 """Class for holding the results of a single test run. 35 36 This class manages what exactly is stored inside the cache without knowing 37 what the key of the cache is. For runs with perf, it stores perf.data, 38 perf.report, etc. The key generation is handled by the ResultsCache class. 39 """ 40 41 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 42 self.chromeos_root = label.chromeos_root 43 self._logger = logger 44 self.ce = cmd_exec or command_executer.GetCommandExecuter( 45 self._logger, log_level=log_level) 46 self.temp_dir = None 47 self.label = label 48 self.results_dir = None 49 self.log_level = log_level 50 self.machine = machine 51 self.perf_data_files = [] 52 self.perf_report_files = [] 53 self.results_file = [] 54 self.chrome_version = '' 55 self.err = None 56 self.chroot_results_dir = '' 57 self.test_name = '' 58 self.keyvals = None 59 self.board = None 60 self.suite = None 61 self.retval = None 62 self.out = None 63 64 def CopyFilesTo(self, dest_dir, files_to_copy): 65 file_index = 0 66 for file_to_copy in files_to_copy: 67 if not os.path.isdir(dest_dir): 68 command = 'mkdir -p %s' % dest_dir 69 self.ce.RunCommand(command) 70 dest_file = os.path.join(dest_dir, 71 ('%s.%s' % (os.path.basename(file_to_copy), 72 file_index))) 73 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 74 if ret: 75 raise IOError('Could not copy results file: %s' % file_to_copy) 76 77 def CopyResultsTo(self, dest_dir): 78 self.CopyFilesTo(dest_dir, self.perf_data_files) 79 self.CopyFilesTo(dest_dir, self.perf_report_files) 80 if len(self.perf_data_files) or len(self.perf_report_files): 81 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 82 83 def GetNewKeyvals(self, keyvals_dict): 84 # Initialize 'units' dictionary. 85 units_dict = {} 86 for k in keyvals_dict: 87 units_dict[k] = '' 88 results_files = self.GetDataMeasurementsFiles() 89 for f in results_files: 90 # Make sure we can find the results file 91 if os.path.exists(f): 92 data_filename = f 93 else: 94 # Otherwise get the base filename and create the correct 95 # path for it. 96 _, f_base = misc.GetRoot(f) 97 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 98 self.temp_dir, f_base) 99 if data_filename.find('.json') > 0: 100 raw_dict = dict() 101 if os.path.exists(data_filename): 102 with open(data_filename, 'r') as data_file: 103 raw_dict = json.load(data_file) 104 105 if 'charts' in raw_dict: 106 raw_dict = raw_dict['charts'] 107 for k1 in raw_dict: 108 field_dict = raw_dict[k1] 109 for k2 in field_dict: 110 result_dict = field_dict[k2] 111 key = k1 + '__' + k2 112 if 'value' in result_dict: 113 keyvals_dict[key] = result_dict['value'] 114 elif 'values' in result_dict: 115 values = result_dict['values'] 116 if ('type' in result_dict and 117 result_dict['type'] == 'list_of_scalar_values' and values and 118 values != 'null'): 119 keyvals_dict[key] = sum(values) / float(len(values)) 120 else: 121 keyvals_dict[key] = values 122 units_dict[key] = result_dict['units'] 123 else: 124 if os.path.exists(data_filename): 125 with open(data_filename, 'r') as data_file: 126 lines = data_file.readlines() 127 for line in lines: 128 tmp_dict = json.loads(line) 129 graph_name = tmp_dict['graph'] 130 graph_str = (graph_name + '__') if graph_name else '' 131 key = graph_str + tmp_dict['description'] 132 keyvals_dict[key] = tmp_dict['value'] 133 units_dict[key] = tmp_dict['units'] 134 135 return keyvals_dict, units_dict 136 137 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 138 """keyvals_dict is the dict of key-value used to generate Crosperf reports. 139 140 units_dict is a dictionary of the units for the return values in 141 keyvals_dict. We need to associate the units with the return values, 142 for Telemetry tests, so that we can include the units in the reports. 143 This function takes each value in keyvals_dict, finds the corresponding 144 unit in the units_dict, and replaces the old value with a list of the 145 old value and the units. This later gets properly parsed in the 146 ResultOrganizer class, for generating the reports. 147 """ 148 149 results_dict = {} 150 for k in keyvals_dict: 151 # We don't want these lines in our reports; they add no useful data. 152 if k == '' or k == 'telemetry_Crosperf': 153 continue 154 val = keyvals_dict[k] 155 units = units_dict[k] 156 new_val = [val, units] 157 results_dict[k] = new_val 158 return results_dict 159 160 def GetKeyvals(self): 161 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 162 if not self.temp_dir: 163 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 164 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 165 self.ce.RunCommand(command, print_to_console=False) 166 167 command = ('python generate_test_report --no-color --csv %s' % 168 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 169 _, out, _ = self.ce.ChrootRunCommandWOutput( 170 self.chromeos_root, command, print_to_console=False) 171 keyvals_dict = {} 172 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 173 self.temp_dir) 174 for line in out.splitlines(): 175 tokens = re.split('=|,', line) 176 key = tokens[-2] 177 if key.startswith(tmp_dir_in_chroot): 178 key = key[len(tmp_dir_in_chroot) + 1:] 179 value = tokens[-1] 180 keyvals_dict[key] = value 181 182 # Check to see if there is a perf_measurements file and get the 183 # data from it if so. 184 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 185 if self.suite == 'telemetry_Crosperf': 186 # For telemtry_Crosperf results, append the units to the return 187 # results, for use in generating the reports. 188 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 189 return keyvals_dict 190 191 def GetResultsDir(self): 192 mo = re.search(r'Results placed in (\S+)', self.out) 193 if mo: 194 result = mo.group(1) 195 return result 196 raise RuntimeError('Could not find results directory.') 197 198 def FindFilesInResultsDir(self, find_args): 199 if not self.results_dir: 200 return None 201 202 command = 'find %s %s' % (self.results_dir, find_args) 203 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 204 if ret: 205 raise RuntimeError('Could not run find command!') 206 return out 207 208 def GetResultsFile(self): 209 return self.FindFilesInResultsDir('-name results-chart.json').splitlines() 210 211 def GetPerfDataFiles(self): 212 return self.FindFilesInResultsDir('-name perf.data').splitlines() 213 214 def GetPerfReportFiles(self): 215 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 216 217 def GetDataMeasurementsFiles(self): 218 result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() 219 if not result: 220 result = \ 221 self.FindFilesInResultsDir('-name results-chart.json').splitlines() 222 return result 223 224 def GeneratePerfReportFiles(self): 225 perf_report_files = [] 226 for perf_data_file in self.perf_data_files: 227 # Generate a perf.report and store it side-by-side with the perf.data 228 # file. 229 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 230 perf_data_file) 231 perf_report_file = '%s.report' % perf_data_file 232 if os.path.exists(perf_report_file): 233 raise RuntimeError('Perf report file already exists: %s' % 234 perf_report_file) 235 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 236 perf_report_file) 237 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 238 239 perf_file = '/usr/sbin/perf' 240 if os.path.exists(perf_path): 241 perf_file = '/usr/bin/perf' 242 243 command = ('%s report ' 244 '-n ' 245 '--symfs /build/%s ' 246 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 247 '--kallsyms /build/%s/boot/System.map-* ' 248 '-i %s --stdio ' 249 '> %s' % (perf_file, self.board, self.board, self.board, 250 chroot_perf_data_file, chroot_perf_report_file)) 251 self.ce.ChrootRunCommand(self.chromeos_root, command) 252 253 # Add a keyval to the dictionary for the events captured. 254 perf_report_files.append( 255 misc.GetOutsideChrootPath(self.chromeos_root, 256 chroot_perf_report_file)) 257 return perf_report_files 258 259 def GatherPerfResults(self): 260 report_id = 0 261 for perf_report_file in self.perf_report_files: 262 with open(perf_report_file, 'r') as f: 263 report_contents = f.read() 264 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 265 num_events = group[0] 266 event_name = group[1] 267 key = 'perf_%s_%s' % (report_id, event_name) 268 value = str(misc.UnitToNumber(num_events)) 269 self.keyvals[key] = value 270 271 def PopulateFromRun(self, out, err, retval, test, suite): 272 self.board = self.label.board 273 self.out = out 274 self.err = err 275 self.retval = retval 276 self.test_name = test 277 self.suite = suite 278 self.chroot_results_dir = self.GetResultsDir() 279 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 280 self.chroot_results_dir) 281 self.results_file = self.GetResultsFile() 282 self.perf_data_files = self.GetPerfDataFiles() 283 # Include all perf.report data in table. 284 self.perf_report_files = self.GeneratePerfReportFiles() 285 # TODO(asharif): Do something similar with perf stat. 286 287 # Grab keyvals from the directory. 288 self.ProcessResults() 289 290 def ProcessJsonResults(self): 291 # Open and parse the json results file generated by telemetry/test_that. 292 if not self.results_file: 293 raise IOError('No results file found.') 294 filename = self.results_file[0] 295 if not filename.endswith('.json'): 296 raise IOError('Attempt to call json on non-json file: %s' % filename) 297 298 if not os.path.exists(filename): 299 return {} 300 301 keyvals = {} 302 with open(filename, 'r') as f: 303 raw_dict = json.load(f) 304 if 'charts' in raw_dict: 305 raw_dict = raw_dict['charts'] 306 for k, field_dict in raw_dict.iteritems(): 307 for item in field_dict: 308 keyname = k + '__' + item 309 value_dict = field_dict[item] 310 if 'value' in value_dict: 311 result = value_dict['value'] 312 elif 'values' in value_dict: 313 values = value_dict['values'] 314 if not values: 315 continue 316 if ('type' in value_dict and 317 value_dict['type'] == 'list_of_scalar_values' and 318 values != 'null'): 319 result = sum(values) / float(len(values)) 320 else: 321 result = values 322 units = value_dict['units'] 323 new_value = [result, units] 324 keyvals[keyname] = new_value 325 return keyvals 326 327 def ProcessResults(self, use_cache=False): 328 # Note that this function doesn't know anything about whether there is a 329 # cache hit or miss. It should process results agnostic of the cache hit 330 # state. 331 if self.results_file and self.results_file[0].find( 332 'results-chart.json') != -1: 333 self.keyvals = self.ProcessJsonResults() 334 else: 335 if not use_cache: 336 print('\n ** WARNING **: Had to use deprecated output-method to ' 337 'collect results.\n') 338 self.keyvals = self.GetKeyvals() 339 self.keyvals['retval'] = self.retval 340 # Generate report from all perf.data files. 341 # Now parse all perf report files and include them in keyvals. 342 self.GatherPerfResults() 343 344 def GetChromeVersionFromCache(self, cache_dir): 345 # Read chrome_version from keys file, if present. 346 chrome_version = '' 347 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 348 if os.path.exists(keys_file): 349 with open(keys_file, 'r') as f: 350 lines = f.readlines() 351 for l in lines: 352 if l.startswith('Google Chrome '): 353 chrome_version = l 354 if chrome_version.endswith('\n'): 355 chrome_version = chrome_version[:-1] 356 break 357 return chrome_version 358 359 def PopulateFromCacheDir(self, cache_dir, test, suite): 360 self.test_name = test 361 self.suite = suite 362 # Read in everything from the cache directory. 363 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 364 self.out = pickle.load(f) 365 self.err = pickle.load(f) 366 self.retval = pickle.load(f) 367 368 # Untar the tarball to a temporary directory 369 self.temp_dir = tempfile.mkdtemp( 370 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 371 372 command = ('cd %s && tar xf %s' % 373 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 374 ret = self.ce.RunCommand(command, print_to_console=False) 375 if ret: 376 raise RuntimeError('Could not untar cached tarball') 377 self.results_dir = self.temp_dir 378 self.results_file = self.GetDataMeasurementsFiles() 379 self.perf_data_files = self.GetPerfDataFiles() 380 self.perf_report_files = self.GetPerfReportFiles() 381 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 382 self.ProcessResults(use_cache=True) 383 384 def CleanUp(self, rm_chroot_tmp): 385 if rm_chroot_tmp and self.results_dir: 386 dirname, basename = misc.GetRoot(self.results_dir) 387 if basename.find('test_that_results_') != -1: 388 command = 'rm -rf %s' % self.results_dir 389 else: 390 command = 'rm -rf %s' % dirname 391 self.ce.RunCommand(command) 392 if self.temp_dir: 393 command = 'rm -rf %s' % self.temp_dir 394 self.ce.RunCommand(command) 395 396 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 397 # Create the dir if it doesn't exist. 398 temp_dir = tempfile.mkdtemp() 399 400 # Store to the temp directory. 401 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 402 pickle.dump(self.out, f) 403 pickle.dump(self.err, f) 404 pickle.dump(self.retval, f) 405 406 if not test_flag.GetTestMode(): 407 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 408 f.write('%s\n' % self.label.name) 409 f.write('%s\n' % self.label.chrome_version) 410 f.write('%s\n' % self.machine.checksum_string) 411 for k in key_list: 412 f.write(k) 413 f.write('\n') 414 415 if self.results_dir: 416 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 417 command = ('cd %s && ' 418 'tar ' 419 '--exclude=var/spool ' 420 '--exclude=var/log ' 421 '-cjf %s .' % (self.results_dir, tarball)) 422 ret = self.ce.RunCommand(command) 423 if ret: 424 raise RuntimeError("Couldn't store autotest output directory.") 425 # Store machine info. 426 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 427 # this function. 428 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 429 f.write(machine_manager.machine_checksum_string[self.label.name]) 430 431 if os.path.exists(cache_dir): 432 command = 'rm -rf {0}'.format(cache_dir) 433 self.ce.RunCommand(command) 434 435 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 436 command += 'chmod g+x {0} && '.format(temp_dir) 437 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 438 ret = self.ce.RunCommand(command) 439 if ret: 440 command = 'rm -rf {0}'.format(temp_dir) 441 self.ce.RunCommand(command) 442 raise RuntimeError('Could not move dir %s to dir %s' % 443 (temp_dir, cache_dir)) 444 445 @classmethod 446 def CreateFromRun(cls, 447 logger, 448 log_level, 449 label, 450 machine, 451 out, 452 err, 453 retval, 454 test, 455 suite='telemetry_Crosperf'): 456 if suite == 'telemetry': 457 result = TelemetryResult(logger, label, log_level, machine) 458 else: 459 result = cls(logger, label, log_level, machine) 460 result.PopulateFromRun(out, err, retval, test, suite) 461 return result 462 463 @classmethod 464 def CreateFromCacheHit(cls, 465 logger, 466 log_level, 467 label, 468 machine, 469 cache_dir, 470 test, 471 suite='telemetry_Crosperf'): 472 if suite == 'telemetry': 473 result = TelemetryResult(logger, label, log_level, machine) 474 else: 475 result = cls(logger, label, log_level, machine) 476 try: 477 result.PopulateFromCacheDir(cache_dir, test, suite) 478 479 except RuntimeError as e: 480 logger.LogError('Exception while using cache: %s' % e) 481 return None 482 return result 483 484 485 class TelemetryResult(Result): 486 """Class to hold the results of a single Telemetry run.""" 487 488 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 489 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 490 cmd_exec) 491 492 def PopulateFromRun(self, out, err, retval, test, suite): 493 self.out = out 494 self.err = err 495 self.retval = retval 496 497 self.ProcessResults() 498 499 # pylint: disable=arguments-differ 500 def ProcessResults(self): 501 # The output is: 502 # url,average_commit_time (ms),... 503 # www.google.com,33.4,21.2,... 504 # We need to convert to this format: 505 # {"www.google.com:average_commit_time (ms)": "33.4", 506 # "www.google.com:...": "21.2"} 507 # Added note: Occasionally the output comes back 508 # with "JSON.stringify(window.automation.GetResults())" on 509 # the first line, and then the rest of the output as 510 # described above. 511 512 lines = self.out.splitlines() 513 self.keyvals = {} 514 515 if lines: 516 if lines[0].startswith('JSON.stringify'): 517 lines = lines[1:] 518 519 if not lines: 520 return 521 labels = lines[0].split(',') 522 for line in lines[1:]: 523 fields = line.split(',') 524 if len(fields) != len(labels): 525 continue 526 for i in xrange(1, len(labels)): 527 key = '%s %s' % (fields[0], labels[i]) 528 value = fields[i] 529 self.keyvals[key] = value 530 self.keyvals['retval'] = self.retval 531 532 def PopulateFromCacheDir(self, cache_dir, test, suite): 533 self.test_name = test 534 self.suite = suite 535 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 536 self.out = pickle.load(f) 537 self.err = pickle.load(f) 538 self.retval = pickle.load(f) 539 540 self.chrome_version = \ 541 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 542 self.ProcessResults() 543 544 545 class CacheConditions(object): 546 """Various Cache condition values, for export.""" 547 548 # Cache hit only if the result file exists. 549 CACHE_FILE_EXISTS = 0 550 551 # Cache hit if the checksum of cpuinfo and totalmem of 552 # the cached result and the new run match. 553 MACHINES_MATCH = 1 554 555 # Cache hit if the image checksum of the cached result and the new run match. 556 CHECKSUMS_MATCH = 2 557 558 # Cache hit only if the cached result was successful 559 RUN_SUCCEEDED = 3 560 561 # Never a cache hit. 562 FALSE = 4 563 564 # Cache hit if the image path matches the cached image path. 565 IMAGE_PATH_MATCH = 5 566 567 # Cache hit if the uuid of hard disk mataches the cached one 568 569 SAME_MACHINE_MATCH = 6 570 571 572 class ResultsCache(object): 573 """Class to handle the cache for storing/retrieving test run results. 574 575 This class manages the key of the cached runs without worrying about what 576 is exactly stored (value). The value generation is handled by the Results 577 class. 578 """ 579 CACHE_VERSION = 6 580 581 def __init__(self): 582 # Proper initialization happens in the Init function below. 583 self.chromeos_image = None 584 self.chromeos_root = None 585 self.test_name = None 586 self.iteration = None 587 self.test_args = None 588 self.profiler_args = None 589 self.board = None 590 self.cache_conditions = None 591 self.machine_manager = None 592 self.machine = None 593 self._logger = None 594 self.ce = None 595 self.label = None 596 self.share_cache = None 597 self.suite = None 598 self.log_level = None 599 self.show_all = None 600 self.run_local = None 601 602 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 603 profiler_args, machine_manager, machine, board, cache_conditions, 604 logger_to_use, log_level, label, share_cache, suite, 605 show_all_results, run_local): 606 self.chromeos_image = chromeos_image 607 self.chromeos_root = chromeos_root 608 self.test_name = test_name 609 self.iteration = iteration 610 self.test_args = test_args 611 self.profiler_args = profiler_args 612 self.board = board 613 self.cache_conditions = cache_conditions 614 self.machine_manager = machine_manager 615 self.machine = machine 616 self._logger = logger_to_use 617 self.ce = command_executer.GetCommandExecuter( 618 self._logger, log_level=log_level) 619 self.label = label 620 self.share_cache = share_cache 621 self.suite = suite 622 self.log_level = log_level 623 self.show_all = show_all_results 624 self.run_local = run_local 625 626 def GetCacheDirForRead(self): 627 matching_dirs = [] 628 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 629 matching_dirs += glob.glob(glob_path) 630 631 if matching_dirs: 632 # Cache file found. 633 return matching_dirs[0] 634 return None 635 636 def GetCacheDirForWrite(self, get_keylist=False): 637 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 638 if get_keylist: 639 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 640 self.run_local) 641 version, image = results_report.ParseChromeosImage( 642 self.label.chromeos_image) 643 keylist = [ 644 version, image, self.label.board, self.machine.name, self.test_name, 645 str(self.iteration), args_str 646 ] 647 return cache_path, keylist 648 return cache_path 649 650 def FormCacheDir(self, list_of_strings): 651 cache_key = ' '.join(list_of_strings) 652 cache_dir = misc.GetFilenameFromString(cache_key) 653 if self.label.cache_dir: 654 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 655 cache_path = [os.path.join(cache_home, cache_dir)] 656 else: 657 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 658 659 if len(self.share_cache): 660 for path in [x.strip() for x in self.share_cache.split(',')]: 661 if os.path.exists(path): 662 cache_path.append(os.path.join(path, cache_dir)) 663 else: 664 self._logger.LogFatal('Unable to find shared cache: %s' % path) 665 666 return cache_path 667 668 def GetCacheKeyList(self, read): 669 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 670 machine_checksum = '*' 671 else: 672 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 673 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 674 checksum = '*' 675 elif self.label.image_type == 'trybot': 676 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 677 elif self.label.image_type == 'official': 678 checksum = '*' 679 else: 680 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 681 682 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 683 image_path_checksum = '*' 684 else: 685 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 686 687 machine_id_checksum = '' 688 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 689 machine_id_checksum = '*' 690 else: 691 if self.machine and self.machine.name in self.label.remote: 692 machine_id_checksum = self.machine.machine_id_checksum 693 else: 694 for machine in self.machine_manager.GetMachines(self.label): 695 if machine.name == self.label.remote[0]: 696 machine_id_checksum = machine.machine_id_checksum 697 break 698 699 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 700 self.run_local) 701 test_args_checksum = hashlib.md5(temp_test_args).hexdigest() 702 return (image_path_checksum, self.test_name, str(self.iteration), 703 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 704 str(self.CACHE_VERSION)) 705 706 def ReadResult(self): 707 if CacheConditions.FALSE in self.cache_conditions: 708 cache_dir = self.GetCacheDirForWrite() 709 command = 'rm -rf %s' % (cache_dir,) 710 self.ce.RunCommand(command) 711 return None 712 cache_dir = self.GetCacheDirForRead() 713 714 if not cache_dir: 715 return None 716 717 if not os.path.isdir(cache_dir): 718 return None 719 720 if self.log_level == 'verbose': 721 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 722 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 723 self.machine, cache_dir, self.test_name, 724 self.suite) 725 if not result: 726 return None 727 728 if (result.retval == 0 or 729 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 730 return result 731 732 return None 733 734 def StoreResult(self, result): 735 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 736 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 737 738 739 class MockResultsCache(ResultsCache): 740 """Class for mock testing, corresponding to ResultsCache class.""" 741 742 def Init(self, *args): 743 pass 744 745 def ReadResult(self): 746 return None 747 748 def StoreResult(self, result): 749 pass 750 751 752 class MockResult(Result): 753 """Class for mock testing, corresponding to Result class.""" 754 755 def PopulateFromRun(self, out, err, retval, test, suite): 756 self.out = out 757 self.err = err 758 self.retval = retval 759
