1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Prepare tests that require re-baselining for input to make_expectations.py. 7 8 The regularly running perf-AV tests require re-baselineing of expectations 9 about once a week. The steps involved in rebaselining are: 10 11 1.) Identify the tests to update, based off reported e-mail results. 12 2.) Figure out reva and revb values, which is the starting and ending revision 13 numbers for the range that we should use to obtain new thresholds. 14 3.) Modify lines in perf_expectations.json referring to the tests to be updated, 15 so that they may be used as input to make_expectations.py. 16 17 This script automates the last step above. 18 19 Here's a sample line from perf_expectations.json: 20 21 "win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, \ 22 "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, \ 23 "regress": 0, "sha1": "54d94538"}, 24 25 To get the above test ready for input to make_expectations.py, it should become: 26 27 "win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": <new reva>, \ 28 "revb": <new revb>, "type": "absolute", "better": "higher", "improve": 0, \ 29 "regress": 0}, 30 31 Examples: 32 33 1.) To update the test specified above and get baseline 34 values using the revision range 12345 and 23456, run this script with a command 35 line like this: 36 python update_perf_expectations.py -f \ 37 win-release/media_tests_av_perf/fps/tulip2.m4a --reva 12345 --revb 23456 38 Or, using an input file, 39 where the input file contains a single line with text 40 win-release/media_tests_av_perf/fps/tulip2.m4a 41 run with this command line: 42 python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456 43 44 2.) Let's say you want to update all seek tests on windows, and get baseline 45 values using the revision range 12345 and 23456. 46 Run this script with this command line: 47 python update_perf_expectations.py -f win-release/media_tests_av_perf/seek/ \ 48 --reva 12345 --revb 23456 49 Or: 50 python update_perf_expectations.py -f win-release/.*/seek/ --reva 12345 \ 51 --revb 23456 52 53 Or, using an input file, 54 where the input file contains a single line with text win-release/.*/seek/: 55 python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456 56 57 3.) Similarly, if you want to update seek tests on all platforms 58 python update_perf_expectations.py -f .*-release/.*/seek/ --reva 12345 \ 59 --revb 23456 60 61 """ 62 63 import logging 64 from optparse import OptionParser 65 import os 66 import re 67 68 import make_expectations as perf_ex_lib 69 70 # Default logging is INFO. Use --verbose to enable DEBUG logging. 71 _DEFAULT_LOG_LEVEL = logging.INFO 72 73 74 def GetTestsToUpdate(contents, all_test_keys): 75 """Parses input contents and obtains tests to be re-baselined. 76 77 Args: 78 contents: string containing contents of input file. 79 all_test_keys: list of keys of test dictionary. 80 Returns: 81 A list of keys for tests that should be updated. 82 """ 83 # Each line of the input file specifies a test case to update. 84 tests_list = [] 85 for test_case_filter in contents.splitlines(): 86 # Skip any empty lines. 87 if test_case_filter: 88 # Sample expected line: 89 # win-release/media_tests_av_perf/seek/\ 90 # CACHED_BUFFERED_SEEK_NoConstraints_crowd1080.ogv 91 # Or, if reg-ex, then sample line: 92 # win-release/media-tests_av_perf/seek* 93 # Skip any leading spaces if they exist in the input file. 94 logging.debug('Trying to match %s', test_case_filter) 95 tests_list.extend(GetMatchingTests(test_case_filter.strip(), 96 all_test_keys)) 97 return tests_list 98 99 100 def GetMatchingTests(tests_to_update, all_test_keys): 101 """Parses input reg-ex filter and obtains tests to be re-baselined. 102 103 Args: 104 tests_to_update: reg-ex string specifying tests to be updated. 105 all_test_keys: list of keys of tests dictionary. 106 Returns: 107 A list of keys for tests that should be updated. 108 """ 109 tests_list = [] 110 search_string = re.compile(tests_to_update) 111 # Get matching tests from the dictionary of tests 112 for test_key in all_test_keys: 113 if search_string.match(test_key): 114 tests_list.append(test_key) 115 logging.debug('%s will be updated', test_key) 116 logging.info('%s tests found matching reg-ex: %s', len(tests_list), 117 tests_to_update) 118 return tests_list 119 120 121 def PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb): 122 """Modifies value of tests that are to re-baselined: 123 Set reva and revb values to specified new values. Remove sha1. 124 125 Args: 126 tests_to_update: list of tests to be updated. 127 all_tests: dictionary of all tests. 128 reva: oldest revision in range to use for new values. 129 revb: newest revision in range to use for new values. 130 Raises: 131 ValueError: If reva or revb are not valid ints, or if either 132 of them are negative. 133 """ 134 reva = int(reva) 135 revb = int(revb) 136 137 if reva < 0 or revb < 0: 138 raise ValueError('Revision values should be positive.') 139 # Ensure reva is less than revb. 140 # (this is similar to the check done in make_expectations.py) 141 if revb < reva: 142 temp = revb 143 revb = reva 144 reva = temp 145 for test_key in tests_to_update: 146 # Get original test from the dictionary of tests 147 test_value = all_tests[test_key] 148 if test_value: 149 # Sample line in perf_expectations.json: 150 # "linux-release/media_tests _av_perf/dropped_frames/crowd360.webm":\ 151 # {"reva": 155180, "revb": 155280, "type": "absolute", \ 152 # "better": "lower", "improve": 0, "regress": 3, "sha1": "276ba29c"}, 153 # Set new revision range 154 test_value['reva'] = reva 155 test_value['revb'] = revb 156 # Remove sha1 to indicate this test requires an update 157 # Check first to make sure it exist. 158 if 'sha1' in test_value: 159 del test_value['sha1'] 160 else: 161 logging.warning('%s does not exist.', test_key) 162 logging.info('Done preparing tests for update.') 163 164 165 def GetCommandLineOptions(): 166 """Parse command line arguments. 167 168 Returns: 169 An options object containing command line arguments and their values. 170 """ 171 parser = OptionParser() 172 173 parser.add_option('--reva', dest='reva', type='int', 174 help='Starting revision of new range.', 175 metavar='START_REVISION') 176 parser.add_option('--revb', dest='revb', type='int', 177 help='Ending revision of new range.', 178 metavar='END_REVISION') 179 parser.add_option('-f', dest='tests_filter', 180 help='Regex to use for filtering tests to be updated. ' 181 'At least one of -filter or -input_file must be provided. ' 182 'If both are provided, then input-file is used.', 183 metavar='FILTER', default='') 184 parser.add_option('-i', dest='input_file', 185 help='Optional path to file with reg-exes for tests to' 186 ' update. If provided, it overrides the filter argument.', 187 metavar='INPUT_FILE', default='') 188 parser.add_option('--config', dest='config_file', 189 default=perf_ex_lib.DEFAULT_CONFIG_FILE, 190 help='Set the config file to FILE.', metavar='FILE') 191 parser.add_option('-v', dest='verbose', action='store_true', default=False, 192 help='Enable verbose output.') 193 options = parser.parse_args()[0] 194 return options 195 196 197 def Main(): 198 """Main driver function.""" 199 options = GetCommandLineOptions() 200 201 _SetLogger(options.verbose) 202 # Do some command-line validation 203 if not options.input_file and not options.tests_filter: 204 logging.error('At least one of input-file or test-filter must be provided.') 205 exit(1) 206 if options.input_file and options.tests_filter: 207 logging.error('Specify only one of input file or test-filter.') 208 exit(1) 209 if not options.reva or not options.revb: 210 logging.error('Start and end revision of range must be specified.') 211 exit(1) 212 213 # Load config. 214 config = perf_ex_lib.ConvertJsonIntoDict( 215 perf_ex_lib.ReadFile(options.config_file)) 216 217 # Obtain the perf expectations file from the config file. 218 perf_file = os.path.join( 219 os.path.dirname(options.config_file), config['perf_file']) 220 221 # We should have all the information we require now. 222 # On to the real thang. 223 # First, get all the existing tests from the original perf_expectations file. 224 all_tests = perf_ex_lib.ConvertJsonIntoDict( 225 perf_ex_lib.ReadFile(perf_file)) 226 all_test_keys = all_tests.keys() 227 # Remove the load key, because we don't want to modify it. 228 all_test_keys.remove('load') 229 # Keep tests sorted, like in the original file. 230 all_test_keys.sort() 231 232 # Next, get all tests that have been identified for an update. 233 tests_to_update = [] 234 if options.input_file: 235 # Tests to update have been specified in an input_file. 236 # Get contents of file. 237 tests_filter = perf_ex_lib.ReadFile(options.input_file) 238 elif options.tests_filter: 239 # Tests to update have been specified as a reg-ex filter. 240 tests_filter = options.tests_filter 241 242 # Get tests to update based on filter specified. 243 tests_to_update = GetTestsToUpdate(tests_filter, all_test_keys) 244 logging.info('Done obtaining matching tests.') 245 246 # Now, prepare tests for update. 247 PrepareTestsForUpdate(tests_to_update, all_tests, options.reva, options.revb) 248 249 # Finally, write modified tests back to perf_expectations file. 250 perf_ex_lib.WriteJson(perf_file, all_tests, all_test_keys, 251 calculate_sha1=False) 252 logging.info('Done writing tests for update to %s.', perf_file) 253 254 255 def _SetLogger(verbose): 256 log_level = _DEFAULT_LOG_LEVEL 257 if verbose: 258 log_level = logging.DEBUG 259 logging.basicConfig(level=log_level, format='%(message)s') 260 261 262 if __name__ == '__main__': 263 Main() 264
