1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import datetime 6 import json 7 import logging 8 import sys 9 10 from lib.bucket import BUCKET_ID, COMMITTED 11 from lib.pageframe import PFNCounts 12 from lib.policy import PolicySet 13 from lib.subcommand import SubCommand 14 15 16 LOGGER = logging.getLogger('dmprof') 17 18 19 class PolicyCommands(SubCommand): 20 def __init__(self, command): 21 super(PolicyCommands, self).__init__( 22 'Usage: %%prog %s [-p POLICY] <first-dump> [shared-first-dumps...]' % 23 command) 24 self._parser.add_option('-p', '--policy', type='string', dest='policy', 25 help='profile with POLICY', metavar='POLICY') 26 self._parser.add_option('--alternative-dirs', dest='alternative_dirs', 27 metavar='/path/on/target@/path/on/host[:...]', 28 help='Read files in /path/on/host/ instead of ' 29 'files in /path/on/target/.') 30 31 def _set_up(self, sys_argv): 32 options, args = self._parse_args(sys_argv, 1) 33 dump_path = args[1] 34 shared_first_dump_paths = args[2:] 35 alternative_dirs_dict = {} 36 if options.alternative_dirs: 37 for alternative_dir_pair in options.alternative_dirs.split(':'): 38 target_path, host_path = alternative_dir_pair.split('@', 1) 39 alternative_dirs_dict[target_path] = host_path 40 (bucket_set, dumps) = SubCommand.load_basic_files( 41 dump_path, True, alternative_dirs=alternative_dirs_dict) 42 43 pfn_counts_dict = {} 44 for shared_first_dump_path in shared_first_dump_paths: 45 shared_dumps = SubCommand._find_all_dumps(shared_first_dump_path) 46 for shared_dump in shared_dumps: 47 pfn_counts = PFNCounts.load(shared_dump) 48 if pfn_counts.pid not in pfn_counts_dict: 49 pfn_counts_dict[pfn_counts.pid] = [] 50 pfn_counts_dict[pfn_counts.pid].append(pfn_counts) 51 52 policy_set = PolicySet.load(SubCommand._parse_policy_list(options.policy)) 53 return policy_set, dumps, pfn_counts_dict, bucket_set 54 55 @staticmethod 56 def _apply_policy(dump, pfn_counts_dict, policy, bucket_set, first_dump_time): 57 """Aggregates the total memory size of each component. 58 59 Iterate through all stacktraces and attribute them to one of the components 60 based on the policy. It is important to apply policy in right order. 61 62 Args: 63 dump: A Dump object. 64 pfn_counts_dict: A dict mapping a pid to a list of PFNCounts. 65 policy: A Policy object. 66 bucket_set: A BucketSet object. 67 first_dump_time: An integer representing time when the first dump is 68 dumped. 69 70 Returns: 71 A dict mapping components and their corresponding sizes. 72 """ 73 LOGGER.info(' %s' % dump.path) 74 all_pfn_dict = {} 75 if pfn_counts_dict: 76 LOGGER.info(' shared with...') 77 for pid, pfnset_list in pfn_counts_dict.iteritems(): 78 closest_pfnset_index = None 79 closest_pfnset_difference = 1024.0 80 for index, pfnset in enumerate(pfnset_list): 81 time_difference = pfnset.time - dump.time 82 if time_difference >= 3.0: 83 break 84 elif ((time_difference < 0.0 and pfnset.reason != 'Exiting') or 85 (0.0 <= time_difference and time_difference < 3.0)): 86 closest_pfnset_index = index 87 closest_pfnset_difference = time_difference 88 elif time_difference < 0.0 and pfnset.reason == 'Exiting': 89 closest_pfnset_index = None 90 break 91 if closest_pfnset_index: 92 for pfn, count in pfnset_list[closest_pfnset_index].iter_pfn: 93 all_pfn_dict[pfn] = all_pfn_dict.get(pfn, 0) + count 94 LOGGER.info(' %s (time difference = %f)' % 95 (pfnset_list[closest_pfnset_index].path, 96 closest_pfnset_difference)) 97 else: 98 LOGGER.info(' (no match with pid:%d)' % pid) 99 100 sizes = dict((c, 0) for c in policy.components) 101 102 PolicyCommands._accumulate_malloc(dump, policy, bucket_set, sizes) 103 verify_global_stats = PolicyCommands._accumulate_maps( 104 dump, all_pfn_dict, policy, bucket_set, sizes) 105 106 # TODO(dmikurube): Remove the verifying code when GLOBAL_STATS is removed. 107 # http://crbug.com/245603. 108 for verify_key, verify_value in verify_global_stats.iteritems(): 109 dump_value = dump.global_stat('%s_committed' % verify_key) 110 if dump_value != verify_value: 111 LOGGER.warn('%25s: %12d != %d (%d)' % ( 112 verify_key, dump_value, verify_value, dump_value - verify_value)) 113 114 sizes['mmap-no-log'] = ( 115 dump.global_stat('profiled-mmap_committed') - 116 sizes['mmap-total-log']) 117 sizes['mmap-total-record'] = dump.global_stat('profiled-mmap_committed') 118 sizes['mmap-total-record-vm'] = dump.global_stat('profiled-mmap_virtual') 119 120 sizes['tc-no-log'] = ( 121 dump.global_stat('profiled-malloc_committed') - 122 sizes['tc-total-log']) 123 sizes['tc-total-record'] = dump.global_stat('profiled-malloc_committed') 124 sizes['tc-unused'] = ( 125 sizes['mmap-tcmalloc'] - 126 dump.global_stat('profiled-malloc_committed')) 127 if sizes['tc-unused'] < 0: 128 LOGGER.warn(' Assuming tc-unused=0 as it is negative: %d (bytes)' % 129 sizes['tc-unused']) 130 sizes['tc-unused'] = 0 131 sizes['tc-total'] = sizes['mmap-tcmalloc'] 132 133 # TODO(dmikurube): global_stat will be deprecated. 134 # See http://crbug.com/245603. 135 for key, value in { 136 'total': 'total_committed', 137 'filemapped': 'file_committed', 138 'absent': 'absent_committed', 139 'file-exec': 'file-exec_committed', 140 'file-nonexec': 'file-nonexec_committed', 141 'anonymous': 'anonymous_committed', 142 'stack': 'stack_committed', 143 'other': 'other_committed', 144 'unhooked-absent': 'nonprofiled-absent_committed', 145 'total-vm': 'total_virtual', 146 'filemapped-vm': 'file_virtual', 147 'anonymous-vm': 'anonymous_virtual', 148 'other-vm': 'other_virtual' }.iteritems(): 149 if key in sizes: 150 sizes[key] = dump.global_stat(value) 151 152 if 'mustbezero' in sizes: 153 removed_list = ( 154 'profiled-mmap_committed', 155 'nonprofiled-absent_committed', 156 'nonprofiled-anonymous_committed', 157 'nonprofiled-file-exec_committed', 158 'nonprofiled-file-nonexec_committed', 159 'nonprofiled-stack_committed', 160 'nonprofiled-other_committed') 161 sizes['mustbezero'] = ( 162 dump.global_stat('total_committed') - 163 sum(dump.global_stat(removed) for removed in removed_list)) 164 if 'total-exclude-profiler' in sizes: 165 sizes['total-exclude-profiler'] = ( 166 dump.global_stat('total_committed') - 167 (sizes['mmap-profiler'] + sizes['mmap-type-profiler'])) 168 if 'hour' in sizes: 169 sizes['hour'] = (dump.time - first_dump_time) / 60.0 / 60.0 170 if 'minute' in sizes: 171 sizes['minute'] = (dump.time - first_dump_time) / 60.0 172 if 'second' in sizes: 173 sizes['second'] = dump.time - first_dump_time 174 175 return sizes 176 177 @staticmethod 178 def _accumulate_malloc(dump, policy, bucket_set, sizes): 179 for line in dump.iter_stacktrace: 180 words = line.split() 181 bucket = bucket_set.get(int(words[BUCKET_ID])) 182 if not bucket or bucket.allocator_type == 'malloc': 183 component_match = policy.find_malloc(bucket) 184 elif bucket.allocator_type == 'mmap': 185 continue 186 else: 187 assert False 188 sizes[component_match] += int(words[COMMITTED]) 189 190 assert not component_match.startswith('mmap-') 191 if component_match.startswith('tc-'): 192 sizes['tc-total-log'] += int(words[COMMITTED]) 193 else: 194 sizes['other-total-log'] += int(words[COMMITTED]) 195 196 @staticmethod 197 def _accumulate_maps(dump, pfn_dict, policy, bucket_set, sizes): 198 # TODO(dmikurube): Remove the dict when GLOBAL_STATS is removed. 199 # http://crbug.com/245603. 200 global_stats = { 201 'total': 0, 202 'file-exec': 0, 203 'file-nonexec': 0, 204 'anonymous': 0, 205 'stack': 0, 206 'other': 0, 207 'nonprofiled-file-exec': 0, 208 'nonprofiled-file-nonexec': 0, 209 'nonprofiled-anonymous': 0, 210 'nonprofiled-stack': 0, 211 'nonprofiled-other': 0, 212 'profiled-mmap': 0, 213 } 214 215 for key, value in dump.iter_map: 216 # TODO(dmikurube): Remove the subtotal code when GLOBAL_STATS is removed. 217 # It's temporary verification code for transition described in 218 # http://crbug.com/245603. 219 committed = 0 220 if 'committed' in value[1]: 221 committed = value[1]['committed'] 222 global_stats['total'] += committed 223 key = 'other' 224 name = value[1]['vma']['name'] 225 if name.startswith('/'): 226 if value[1]['vma']['executable'] == 'x': 227 key = 'file-exec' 228 else: 229 key = 'file-nonexec' 230 elif name == '[stack]': 231 key = 'stack' 232 elif name == '': 233 key = 'anonymous' 234 global_stats[key] += committed 235 if value[0] == 'unhooked': 236 global_stats['nonprofiled-' + key] += committed 237 if value[0] == 'hooked': 238 global_stats['profiled-mmap'] += committed 239 240 if value[0] == 'unhooked': 241 if pfn_dict and dump.pageframe_length: 242 for pageframe in value[1]['pageframe']: 243 component_match = policy.find_unhooked(value, pageframe, pfn_dict) 244 sizes[component_match] += pageframe.size 245 else: 246 component_match = policy.find_unhooked(value) 247 sizes[component_match] += int(value[1]['committed']) 248 elif value[0] == 'hooked': 249 if pfn_dict and dump.pageframe_length: 250 for pageframe in value[1]['pageframe']: 251 component_match, _ = policy.find_mmap( 252 value, bucket_set, pageframe, pfn_dict) 253 sizes[component_match] += pageframe.size 254 assert not component_match.startswith('tc-') 255 if component_match.startswith('mmap-'): 256 sizes['mmap-total-log'] += pageframe.size 257 else: 258 sizes['other-total-log'] += pageframe.size 259 else: 260 component_match, _ = policy.find_mmap(value, bucket_set) 261 sizes[component_match] += int(value[1]['committed']) 262 if component_match.startswith('mmap-'): 263 sizes['mmap-total-log'] += int(value[1]['committed']) 264 else: 265 sizes['other-total-log'] += int(value[1]['committed']) 266 else: 267 LOGGER.error('Unrecognized mapping status: %s' % value[0]) 268 269 return global_stats 270 271 272 class CSVCommand(PolicyCommands): 273 def __init__(self): 274 super(CSVCommand, self).__init__('csv') 275 276 def do(self, sys_argv): 277 policy_set, dumps, pfn_counts_dict, bucket_set = self._set_up(sys_argv) 278 return CSVCommand._output( 279 policy_set, dumps, pfn_counts_dict, bucket_set, sys.stdout) 280 281 @staticmethod 282 def _output(policy_set, dumps, pfn_counts_dict, bucket_set, out): 283 max_components = 0 284 for label in policy_set: 285 max_components = max(max_components, len(policy_set[label].components)) 286 287 for label in sorted(policy_set): 288 components = policy_set[label].components 289 if len(policy_set) > 1: 290 out.write('%s%s\n' % (label, ',' * (max_components - 1))) 291 out.write('%s%s\n' % ( 292 ','.join(components), ',' * (max_components - len(components)))) 293 294 LOGGER.info('Applying a policy %s to...' % label) 295 for dump in dumps: 296 component_sizes = PolicyCommands._apply_policy( 297 dump, pfn_counts_dict, policy_set[label], bucket_set, dumps[0].time) 298 s = [] 299 for c in components: 300 if c in ('hour', 'minute', 'second'): 301 s.append('%05.5f' % (component_sizes[c])) 302 else: 303 s.append('%05.5f' % (component_sizes[c] / 1024.0 / 1024.0)) 304 out.write('%s%s\n' % ( 305 ','.join(s), ',' * (max_components - len(components)))) 306 307 bucket_set.clear_component_cache() 308 309 return 0 310 311 312 class JSONCommand(PolicyCommands): 313 def __init__(self): 314 super(JSONCommand, self).__init__('json') 315 316 def do(self, sys_argv): 317 policy_set, dumps, pfn_counts_dict, bucket_set = self._set_up(sys_argv) 318 return JSONCommand._output( 319 policy_set, dumps, pfn_counts_dict, bucket_set, sys.stdout) 320 321 @staticmethod 322 def _output(policy_set, dumps, pfn_counts_dict, bucket_set, out): 323 json_base = { 324 'version': 'JSON_DEEP_2', 325 'policies': {}, 326 } 327 328 for label in sorted(policy_set): 329 json_base['policies'][label] = { 330 'legends': policy_set[label].components, 331 'snapshots': [], 332 } 333 334 LOGGER.info('Applying a policy %s to...' % label) 335 for dump in dumps: 336 component_sizes = PolicyCommands._apply_policy( 337 dump, pfn_counts_dict, policy_set[label], bucket_set, dumps[0].time) 338 component_sizes['dump_path'] = dump.path 339 component_sizes['dump_time'] = datetime.datetime.fromtimestamp( 340 dump.time).strftime('%Y-%m-%d %H:%M:%S') 341 json_base['policies'][label]['snapshots'].append(component_sizes) 342 343 bucket_set.clear_component_cache() 344 345 json.dump(json_base, out, indent=2, sort_keys=True) 346 347 return 0 348 349 350 class ListCommand(PolicyCommands): 351 def __init__(self): 352 super(ListCommand, self).__init__('list') 353 354 def do(self, sys_argv): 355 policy_set, dumps, pfn_counts_dict, bucket_set = self._set_up(sys_argv) 356 return ListCommand._output( 357 policy_set, dumps, pfn_counts_dict, bucket_set, sys.stdout) 358 359 @staticmethod 360 def _output(policy_set, dumps, pfn_counts_dict, bucket_set, out): 361 for label in sorted(policy_set): 362 LOGGER.info('Applying a policy %s to...' % label) 363 for dump in dumps: 364 component_sizes = PolicyCommands._apply_policy( 365 dump, pfn_counts_dict, policy_set[label], bucket_set, dump.time) 366 out.write('%s for %s:\n' % (label, dump.path)) 367 for c in policy_set[label].components: 368 if c in ['hour', 'minute', 'second']: 369 out.write('%40s %12.3f\n' % (c, component_sizes[c])) 370 else: 371 out.write('%40s %12d\n' % (c, component_sizes[c])) 372 373 bucket_set.clear_component_cache() 374 375 return 0 376
