Home | History | Annotate | Download | only in result_tools 
      1 # Copyright 2017 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Help functions used by different throttlers."""
      6 
      7 import os
      8 import re
      9 
     10 import utils_lib
     11 
     12 
     13 # A list of file names that should not be throttled, that is, not modified by
     14 # deletion, trimming or compression.
     15 NON_THROTTLEABLE_FILE_NAMES = set([
     16         '.autoserv_execute',
     17         '.parse.lock',
     18         '.parse.log',
     19         '.parser_execute',
     20         'control',
     21         'control.srv',
     22         'host_keyvals',
     23         'job_report.html',
     24         'keyval',
     25         'profiling',
     26         'result_summary.html',
     27         'sponge_invocation.xml',
     28         'status',
     29         'status.log',
     30 
     31         # ACTS related files:
     32         'test_run_details.txt',
     33         'test_run_error.txt',
     34         'test_run_info.txt',
     35         'test_run_summary.json',
     36         ])
     37 
     38 # A list of file name patterns that should not be throttled, that is, not
     39 # modified by deletion, deduping, trimming or compression.
     40 NON_THROTTLEABLE_FILE_PATTERNS = [
     41         '.*/BUILD_INFO-.*',   # ACTS test result files.
     42         '.*/AndroidDevice.*', # ACTS test result files.
     43         ]
     44 
     45 # Regex of result files sorted based on priority. Files can be throttled first
     46 # have higher priority.
     47 RESULT_THROTTLE_PRIORITY = [
     48         '(.*/)?sysinfo/var/log/.*',
     49         '(.*/)?sysinfo/var/log_diff/.*',
     50         '(.*/)?sysinfo/.*',
     51         # The last one matches any file.
     52         '.*',
     53         ]
     54 
     55 # Regex of file names for Autotest debug logs. These files should be preserved
     56 # without throttling if possible.
     57 AUTOTEST_LOG_PATTERN ='.*\.(DEBUG|ERROR|INFO|WARNING)$'
     58 
     59 def _list_files(files, all_files=None):
     60     """Get all files in the given directories.
     61 
     62     @param files: A list of ResultInfo objects for files in a directory.
     63     @param all_files: A list of ResultInfo objects collected so far.
     64     @return: A list of all collected ResultInfo objects.
     65     """
     66     if all_files is None:
     67         all_files = []
     68     for info in files:
     69         if info.is_dir:
     70             _list_files(info.files, all_files)
     71         else:
     72             all_files.append(info)
     73     return all_files
     74 
     75 
     76 def sort_result_files(summary):
     77     """Sort result infos based on priority.
     78 
     79     @param summary: A ResultInfo object containing result summary.
     80     @return: A tuple of (sorted_files, grouped_files)
     81             sorted_files: A list of ResultInfo, sorted based on file size and
     82                 priority based on RESULT_THROTTLE_PRIORITY.
     83             grouped_files: A dictionary of ResultInfo grouped by each item in
     84                 RESULT_THROTTLE_PRIORITY.
     85     """
     86     all_files = _list_files(summary.files)
     87 
     88     # Scan all file paths and group them based on the throttle priority.
     89     grouped_files = {pattern: [] for pattern in RESULT_THROTTLE_PRIORITY}
     90     for info in all_files:
     91         for pattern in RESULT_THROTTLE_PRIORITY:
     92             if re.match(pattern, info.path):
     93                 grouped_files[pattern].append(info)
     94                 break
     95 
     96     sorted_files = []
     97     for pattern in RESULT_THROTTLE_PRIORITY:
     98         # Sort the files in each group by file size, largest first.
     99         infos = grouped_files[pattern]
    100         infos.sort(key=lambda info: -info.trimmed_size)
    101         sorted_files.extend(infos)
    102 
    103     return sorted_files, grouped_files
    104 
    105 
    106 def get_throttleable_files(file_infos, extra_patterns=[]):
    107     """Filter the files can be throttled.
    108 
    109     @param file_infos: A list of ResultInfo objects.
    110     @param extra_patterns: Extra patterns of file path that should not be
    111             throttled.
    112     @yield: ResultInfo objects that can be throttled.
    113     """
    114     for info in file_infos:
    115         # Skip the files being deleted in earlier throttling.
    116         if info.trimmed_size == 0:
    117             continue
    118         if info.name in NON_THROTTLEABLE_FILE_NAMES:
    119             continue
    120         pattern_matched = False
    121         for pattern in extra_patterns + NON_THROTTLEABLE_FILE_PATTERNS:
    122             if re.match(pattern, info.path):
    123                 pattern_matched = True
    124                 break
    125 
    126         if not pattern_matched:
    127             yield info
    128 
    129 
    130 def check_throttle_limit(summary, max_result_size_KB):
    131     """Check if throttling is enough already.
    132 
    133     @param summary: A ResultInfo object containing result summary.
    134     @param max_result_size_KB: Maximum test result size in KB.
    135     @return: True if the result directory has been trimmed to be smaller than
    136             max_result_size_KB.
    137     """
    138     if (summary.trimmed_size <= max_result_size_KB * 1024):
    139         utils_lib.LOG('Maximum result size is reached (current result'
    140                       'size is %s (limit is %s).' %
    141                       (utils_lib.get_size_string(summary.trimmed_size),
    142                        utils_lib.get_size_string(max_result_size_KB * 1024)))
    143         return True
    144     else:
    145         return False
    146 
    147 
    148 def try_delete_file_on_disk(path):
    149     """Try to delete the give file on disk.
    150 
    151     @param path: Path to the file.
    152     @returns: True if the file is deleted, False otherwise.
    153     """
    154     try:
    155         utils_lib.LOG('Deleting file %s.' % path)
    156         os.remove(path)
    157         return True
    158     except OSError as e:
    159         utils_lib.LOG('Failed to delete file %s, Error: %s' % (path, e))