1 #!/usr/bin/env python3 2 # 3 # Copyright 2018, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # 18 # 19 # Measure application start-up time by launching applications under various combinations. 20 # See --help for more details. 21 # 22 # 23 # Sample usage: 24 # $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10 -o out.csv 25 # $> ./analyze_metrics.py out.csv 26 # 27 # 28 29 import argparse 30 import csv 31 import itertools 32 import os 33 import subprocess 34 import sys 35 import tempfile 36 from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union 37 38 # The following command line options participate in the combinatorial generation. 39 # All other arguments have a global effect. 40 _COMBINATORIAL_OPTIONS=['packages', 'readaheads', 'compiler_filters'] 41 _TRACING_READAHEADS=['mlock', 'fadvise'] 42 _FORWARD_OPTIONS={'loop_count': '--count'} 43 _RUN_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'run_app_with_prefetch') 44 45 RunCommandArgs = NamedTuple('RunCommandArgs', [('package', str), ('readahead', str), ('compiler_filter', Optional[str])]) 46 CollectorPackageInfo = NamedTuple('CollectorPackageInfo', [('package', str), ('compiler_filter', str)]) 47 _COLLECTOR_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'collector') 48 _COLLECTOR_TIMEOUT_MULTIPLIER = 2 # take the regular --timeout and multiply by 2; systrace starts up slowly. 49 50 _UNLOCK_SCREEN_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'unlock_screen') 51 52 # This must be the only mutable global variable. All other global variables are constants to avoid magic literals. 53 _debug = False # See -d/--debug flag. 54 _DEBUG_FORCE = None # Ignore -d/--debug if this is not none. 55 56 # Type hinting names. 57 T = TypeVar('T') 58 NamedTupleMeta = Callable[..., T] # approximation of a (S : NamedTuple<T> where S() == T) metatype. 59 60 def parse_options(argv: List[str] = None): 61 """Parse command line arguments and return an argparse Namespace object.""" 62 parser = argparse.ArgumentParser(description="Run one or more Android applications under various settings in order to measure startup time.") 63 # argparse considers args starting with - and -- optional in --help, even though required=True. 64 # by using a named argument group --help will clearly say that it's required instead of optional. 65 required_named = parser.add_argument_group('required named arguments') 66 required_named.add_argument('-p', '--package', action='append', dest='packages', help='package of the application', required=True) 67 required_named.add_argument('-r', '--readahead', action='append', dest='readaheads', help='which readahead mode to use', choices=('warm', 'cold', 'mlock', 'fadvise'), required=True) 68 69 # optional arguments 70 # use a group here to get the required arguments to appear 'above' the optional arguments in help. 71 optional_named = parser.add_argument_group('optional named arguments') 72 optional_named.add_argument('-c', '--compiler-filter', action='append', dest='compiler_filters', help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter', choices=('speed', 'speed-profile', 'quicken')) 73 optional_named.add_argument('-s', '--simulate', dest='simulate', action='store_true', help='Print which commands will run, but don\'t run the apps') 74 optional_named.add_argument('-d', '--debug', dest='debug', action='store_true', help='Add extra debugging output') 75 optional_named.add_argument('-o', '--output', dest='output', action='store', help='Write CSV output to file.') 76 optional_named.add_argument('-t', '--timeout', dest='timeout', action='store', type=int, help='Timeout after this many seconds when executing a single run.') 77 optional_named.add_argument('-lc', '--loop-count', dest='loop_count', default=1, type=int, action='store', help='How many times to loop a single run.') 78 optional_named.add_argument('-in', '--inodes', dest='inodes', type=str, action='store', help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)') 79 80 return parser.parse_args(argv) 81 82 # TODO: refactor this with a common library file with analyze_metrics.py 83 def _debug_print(*args, **kwargs): 84 """Print the args to sys.stderr if the --debug/-d flag was passed in.""" 85 if _debug: 86 print(*args, **kwargs, file=sys.stderr) 87 88 def _expand_gen_repr(args): 89 """Like repr but any generator-like object has its iterator consumed 90 and then called repr on.""" 91 new_args_list = [] 92 for i in args: 93 # detect iterable objects that do not have their own override of __str__ 94 if hasattr(i, '__iter__'): 95 to_str = getattr(i, '__str__') 96 if to_str.__objclass__ == object: 97 # the repr for a generator is just type+address, expand it out instead. 98 new_args_list.append([_expand_gen_repr([j])[0] for j in i]) 99 continue 100 # normal case: uses the built-in to-string 101 new_args_list.append(i) 102 return new_args_list 103 104 def _debug_print_gen(*args, **kwargs): 105 """Like _debug_print but will turn any iterable args into a list.""" 106 if not _debug: 107 return 108 109 new_args_list = _expand_gen_repr(args) 110 _debug_print(*new_args_list, **kwargs) 111 112 def _debug_print_nd(*args, **kwargs): 113 """Like _debug_print but will turn any NamedTuple-type args into a string.""" 114 if not _debug: 115 return 116 117 new_args_list = [] 118 for i in args: 119 if hasattr(i, '_field_types'): 120 new_args_list.append("%s: %s" %(i.__name__, i._field_types)) 121 else: 122 new_args_list.append(i) 123 124 _debug_print(*new_args_list, **kwargs) 125 126 def dict_lookup_any_key(dictionary: dict, *keys: List[Any]): 127 for k in keys: 128 if k in dictionary: 129 return dictionary[k] 130 raise KeyError("None of the keys %s were in the dictionary" %(keys)) 131 132 def generate_run_combinations(named_tuple: NamedTupleMeta[T], opts_dict: Dict[str, List[Optional[str]]])\ 133 -> Iterable[T]: 134 """ 135 Create all possible combinations given the values in opts_dict[named_tuple._fields]. 136 137 :type T: type annotation for the named_tuple type. 138 :param named_tuple: named tuple type, whose fields are used to make combinations for 139 :param opts_dict: dictionary of keys to value list. keys correspond to the named_tuple fields. 140 :return: an iterable over named_tuple instances. 141 """ 142 combinations_list = [] 143 for k in named_tuple._fields: 144 # the key can be either singular or plural , e.g. 'package' or 'packages' 145 val = dict_lookup_any_key(opts_dict, k, k + "s") 146 147 # treat {'x': None} key value pairs as if it was [None] 148 # otherwise itertools.product throws an exception about not being able to iterate None. 149 combinations_list.append(val or [None]) 150 151 _debug_print("opts_dict: ", opts_dict) 152 _debug_print_nd("named_tuple: ", named_tuple) 153 _debug_print("combinations_list: ", combinations_list) 154 155 for combo in itertools.product(*combinations_list): 156 yield named_tuple(*combo) 157 158 def key_to_cmdline_flag(key: str) -> str: 159 """Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """ 160 if key.endswith("s"): 161 key = key[:-1] 162 return "--" + key.replace("_", "-") 163 164 def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]: 165 """ 166 Convert a named tuple into a command-line compatible arguments list. 167 168 Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3] 169 """ 170 args = [] 171 for key, value in tpl._asdict().items(): 172 if value is None: 173 continue 174 args.append(key_to_cmdline_flag(key)) 175 args.append(value) 176 return args 177 178 def generate_group_run_combinations(run_combinations: Iterable[NamedTuple], dst_nt: NamedTupleMeta[T])\ 179 -> Iterable[Tuple[T, Iterable[NamedTuple]]]: 180 181 def group_by_keys(src_nt): 182 src_d = src_nt._asdict() 183 # now remove the keys that aren't legal in dst. 184 for illegal_key in set(src_d.keys()) - set(dst_nt._fields): 185 if illegal_key in src_d: 186 del src_d[illegal_key] 187 188 return dst_nt(**src_d) 189 190 for args_list_it in itertools.groupby(run_combinations, group_by_keys): 191 (group_key_value, args_it) = args_list_it 192 yield (group_key_value, args_it) 193 194 def parse_run_script_csv_file(csv_file: TextIO) -> List[int]: 195 """Parse a CSV file full of integers into a flat int list.""" 196 csv_reader = csv.reader(csv_file) 197 arr = [] 198 for row in csv_reader: 199 for i in row: 200 if i: 201 arr.append(int(i)) 202 return arr 203 204 def make_script_command_with_temp_output(script: str, args: List[str], **kwargs)\ 205 -> Tuple[str, TextIO]: 206 """ 207 Create a command to run a script given the args. 208 Appends --count <loop_count> --output <tmp-file-name>. 209 Returns a tuple (cmd, tmp_file) 210 """ 211 tmp_output_file = tempfile.NamedTemporaryFile(mode='r') 212 cmd = [script] + args 213 for key, value in kwargs.items(): 214 cmd += ['--%s' %(key), "%s" %(value)] 215 if _debug: 216 cmd += ['--verbose'] 217 cmd = cmd + ["--output", tmp_output_file.name] 218 return cmd, tmp_output_file 219 220 def execute_arbitrary_command(cmd: List[str], simulate: bool, timeout: int) -> Tuple[bool, str]: 221 if simulate: 222 print(" ".join(cmd)) 223 return (True, "") 224 else: 225 _debug_print("[EXECUTE]", cmd) 226 proc = subprocess.Popen(cmd, 227 stderr=subprocess.STDOUT, 228 stdout=subprocess.PIPE, 229 universal_newlines=True) 230 try: 231 script_output = proc.communicate(timeout=timeout)[0] 232 except subprocess.TimeoutExpired: 233 print("[TIMEDOUT]") 234 proc.kill() 235 script_output = proc.communicate()[0] 236 237 _debug_print("[STDOUT]", script_output) 238 return_code = proc.wait() 239 passed = (return_code == 0) 240 _debug_print("[$?]", return_code) 241 if not passed: 242 print("[FAILED, code:%s]" %(return_code), script_output, file=sys.stderr) 243 244 return (passed, script_output) 245 246 def execute_run_combos(grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]], simulate: bool, inodes_path: str, timeout: int, loop_count: int, need_trace: bool): 247 # nothing will work if the screen isn't unlocked first. 248 execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT], simulate, timeout) 249 250 for collector_info, run_combos in grouped_run_combos: 251 #collector_args = ["--package", package_name] 252 collector_args = as_run_command(collector_info) 253 # TODO: forward --wait_time for how long systrace runs? 254 # TODO: forward --trace_buffer_size for size of systrace buffer size? 255 collector_cmd, collector_tmp_output_file = make_script_command_with_temp_output(_COLLECTOR_SCRIPT, collector_args, inodes=inodes_path) 256 257 with collector_tmp_output_file: 258 collector_passed = True 259 if need_trace: 260 collector_timeout = timeout and _COLLECTOR_TIMEOUT_MULTIPLIER * timeout 261 (collector_passed, collector_script_output) = execute_arbitrary_command(collector_cmd, simulate, collector_timeout) 262 # TODO: consider to print a ; collector wrote file to <...> into the CSV file so we know it was ran. 263 264 for combos in run_combos: 265 args = as_run_command(combos) 266 267 cmd, tmp_output_file = make_script_command_with_temp_output(_RUN_SCRIPT, args, count=loop_count, input=collector_tmp_output_file.name) 268 with tmp_output_file: 269 (passed, script_output) = execute_arbitrary_command(cmd, simulate, timeout) 270 parsed_output = simulate and [1,2,3] or parse_run_script_csv_file(tmp_output_file) 271 yield (passed, script_output, parsed_output) 272 273 def gather_results(commands: Iterable[Tuple[bool, str, List[int]]], key_list: List[str], value_list: List[Tuple[str, ...]]): 274 _debug_print("gather_results: key_list = ", key_list) 275 yield key_list + ["time(ms)"] 276 277 stringify_none = lambda s: s is None and "<none>" or s 278 279 for ((passed, script_output, run_result_list), values) in itertools.zip_longest(commands, value_list): 280 if not passed: 281 continue 282 for result in run_result_list: 283 yield [stringify_none(i) for i in values] + [result] 284 285 yield ["; avg(%s), min(%s), max(%s), count(%s)" %(sum(run_result_list, 0.0) / len(run_result_list), min(run_result_list), max(run_result_list), len(run_result_list)) ] 286 287 def eval_and_save_to_csv(output, annotated_result_values): 288 csv_writer = csv.writer(output) 289 for row in annotated_result_values: 290 csv_writer.writerow(row) 291 output.flush() # see the output live. 292 293 def main(): 294 global _debug 295 296 opts = parse_options() 297 _debug = opts.debug 298 if _DEBUG_FORCE is not None: 299 _debug = _DEBUG_FORCE 300 _debug_print("parsed options: ", opts) 301 need_trace = not not set(opts.readaheads).intersection(set(_TRACING_READAHEADS)) 302 if need_trace and not opts.inodes: 303 print("Error: Missing -in/--inodes, required when using a readahead of %s" %(_TRACING_READAHEADS), file=sys.stderr) 304 return 1 305 306 output_file = opts.output and open(opts.output, 'w') or sys.stdout 307 308 combos = lambda: generate_run_combinations(RunCommandArgs, vars(opts)) 309 _debug_print_gen("run combinations: ", combos()) 310 311 grouped_combos = lambda: generate_group_run_combinations(combos(), CollectorPackageInfo) 312 _debug_print_gen("grouped run combinations: ", grouped_combos()) 313 314 exec = execute_run_combos(grouped_combos(), opts.simulate, opts.inodes, opts.timeout, opts.loop_count, need_trace) 315 results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos()) 316 eval_and_save_to_csv(output_file, results) 317 318 return 0 319 320 321 if __name__ == '__main__': 322 sys.exit(main()) 323
