1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import cStringIO 6 import json 7 import logging 8 import os 9 import re 10 11 12 LOGGER = logging.getLogger('dmprof') 13 14 BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 15 16 DEFAULT_SORTERS = [ 17 os.path.join(BASE_PATH, 'sorter.malloc-component.json'), 18 os.path.join(BASE_PATH, 'sorter.malloc-type.json'), 19 os.path.join(BASE_PATH, 'sorter.vm-map.json'), 20 os.path.join(BASE_PATH, 'sorter.vm-sharing.json'), 21 ] 22 23 DEFAULT_TEMPLATES = os.path.join(BASE_PATH, 'templates.json') 24 25 26 class Unit(object): 27 """Represents a minimum unit of memory usage categorization. 28 29 It is supposed to be inherited for some different spaces like the entire 30 virtual memory and malloc arena. Such different spaces are called "worlds" 31 in dmprof. (For example, the "vm" world and the "malloc" world.) 32 """ 33 def __init__(self, unit_id, size): 34 self._unit_id = unit_id 35 self._size = size 36 37 @property 38 def unit_id(self): 39 return self._unit_id 40 41 @property 42 def size(self): 43 return self._size 44 45 46 class VMUnit(Unit): 47 """Represents a Unit for a memory region on virtual memory.""" 48 def __init__(self, unit_id, committed, reserved, mmap, region, 49 pageframe=None, group_pfn_counts=None): 50 super(VMUnit, self).__init__(unit_id, committed) 51 self._reserved = reserved 52 self._mmap = mmap 53 self._region = region 54 self._pageframe = pageframe 55 self._group_pfn_counts = group_pfn_counts 56 57 @property 58 def committed(self): 59 return self._size 60 61 @property 62 def reserved(self): 63 return self._reserved 64 65 @property 66 def mmap(self): 67 return self._mmap 68 69 @property 70 def region(self): 71 return self._region 72 73 @property 74 def pageframe(self): 75 return self._pageframe 76 77 @property 78 def group_pfn_counts(self): 79 return self._group_pfn_counts 80 81 82 class MMapUnit(VMUnit): 83 """Represents a Unit for a mmap'ed region.""" 84 def __init__(self, unit_id, committed, reserved, region, bucket_set, 85 pageframe=None, group_pfn_counts=None): 86 super(MMapUnit, self).__init__(unit_id, committed, reserved, True, 87 region, pageframe, group_pfn_counts) 88 self._bucket_set = bucket_set 89 90 def __repr__(self): 91 return str(self.region) 92 93 @property 94 def bucket_set(self): 95 return self._bucket_set 96 97 98 class UnhookedUnit(VMUnit): 99 """Represents a Unit for a non-mmap'ed memory region on virtual memory.""" 100 def __init__(self, unit_id, committed, reserved, region, 101 pageframe=None, group_pfn_counts=None): 102 super(UnhookedUnit, self).__init__(unit_id, committed, reserved, False, 103 region, pageframe, group_pfn_counts) 104 105 def __repr__(self): 106 return str(self.region) 107 108 109 class MallocUnit(Unit): 110 """Represents a Unit for a malloc'ed memory block.""" 111 def __init__(self, unit_id, size, alloc_count, free_count, bucket): 112 super(MallocUnit, self).__init__(unit_id, size) 113 self._bucket = bucket 114 self._alloc_count = alloc_count 115 self._free_count = free_count 116 117 def __repr__(self): 118 return str(self.bucket) 119 120 @property 121 def bucket(self): 122 return self._bucket 123 124 @property 125 def alloc_count(self): 126 return self._alloc_count 127 128 @property 129 def free_count(self): 130 return self._free_count 131 132 133 class UnitSet(object): 134 """Represents an iterable set of Units.""" 135 def __init__(self, world): 136 self._units = {} 137 self._world = world 138 139 def __repr__(self): 140 return str(self._units) 141 142 def __iter__(self): 143 for unit_id in sorted(self._units): 144 yield self._units[unit_id] 145 146 def append(self, unit, overwrite=False): 147 if not overwrite and unit.unit_id in self._units: 148 LOGGER.error('The unit id=%s already exists.' % str(unit.unit_id)) 149 self._units[unit.unit_id] = unit 150 151 152 class AbstractRule(object): 153 """An abstract class for rules to be matched with units.""" 154 def __init__(self, dct): 155 self._name = dct['name'] 156 self._hidden = dct.get('hidden', False) 157 self._subs = dct.get('subs', []) 158 159 def match(self, unit): 160 raise NotImplementedError() 161 162 @property 163 def name(self): 164 return self._name 165 166 @property 167 def hidden(self): 168 return self._hidden 169 170 def iter_subs(self): 171 for sub in self._subs: 172 yield sub 173 174 175 class VMRule(AbstractRule): 176 """Represents a Rule to match with virtual memory regions.""" 177 def __init__(self, dct): 178 super(VMRule, self).__init__(dct) 179 self._backtrace_function = dct.get('backtrace_function', None) 180 if self._backtrace_function: 181 self._backtrace_function = re.compile(self._backtrace_function) 182 self._backtrace_sourcefile = dct.get('backtrace_sourcefile', None) 183 if self._backtrace_sourcefile: 184 self._backtrace_sourcefile = re.compile(self._backtrace_sourcefile) 185 self._mmap = dct.get('mmap', None) 186 self._sharedwith = dct.get('sharedwith', []) 187 self._mapped_pathname = dct.get('mapped_pathname', None) 188 if self._mapped_pathname: 189 self._mapped_pathname = re.compile(self._mapped_pathname) 190 self._mapped_permission = dct.get('mapped_permission', None) 191 if self._mapped_permission: 192 self._mapped_permission = re.compile(self._mapped_permission) 193 194 def __repr__(self): 195 result = cStringIO.StringIO() 196 result.write('{"%s"=>' % self._name) 197 attributes = [] 198 attributes.append('mmap: %s' % self._mmap) 199 if self._backtrace_function: 200 attributes.append('backtrace_function: "%s"' % 201 self._backtrace_function.pattern) 202 if self._sharedwith: 203 attributes.append('sharedwith: "%s"' % self._sharedwith) 204 if self._mapped_pathname: 205 attributes.append('mapped_pathname: "%s"' % self._mapped_pathname.pattern) 206 if self._mapped_permission: 207 attributes.append('mapped_permission: "%s"' % 208 self._mapped_permission.pattern) 209 result.write('%s}' % ', '.join(attributes)) 210 return result.getvalue() 211 212 def match(self, unit): 213 if unit.mmap: 214 assert unit.region[0] == 'hooked' 215 bucket = unit.bucket_set.get(unit.region[1]['bucket_id']) 216 assert bucket 217 assert bucket.allocator_type == 'mmap' 218 219 stackfunction = bucket.symbolized_joined_stackfunction 220 stacksourcefile = bucket.symbolized_joined_stacksourcefile 221 222 # TODO(dmikurube): Support shared memory. 223 sharedwith = None 224 225 if self._mmap == False: # (self._mmap == None) should go through. 226 return False 227 if (self._backtrace_function and 228 not self._backtrace_function.match(stackfunction)): 229 return False 230 if (self._backtrace_sourcefile and 231 not self._backtrace_sourcefile.match(stacksourcefile)): 232 return False 233 if (self._mapped_pathname and 234 not self._mapped_pathname.match(unit.region[1]['vma']['name'])): 235 return False 236 if (self._mapped_permission and 237 not self._mapped_permission.match( 238 unit.region[1]['vma']['readable'] + 239 unit.region[1]['vma']['writable'] + 240 unit.region[1]['vma']['executable'] + 241 unit.region[1]['vma']['private'])): 242 return False 243 if (self._sharedwith and 244 unit.pageframe and sharedwith not in self._sharedwith): 245 return False 246 247 return True 248 249 else: 250 assert unit.region[0] == 'unhooked' 251 252 # TODO(dmikurube): Support shared memory. 253 sharedwith = None 254 255 if self._mmap == True: # (self._mmap == None) should go through. 256 return False 257 if (self._mapped_pathname and 258 not self._mapped_pathname.match(unit.region[1]['vma']['name'])): 259 return False 260 if (self._mapped_permission and 261 not self._mapped_permission.match( 262 unit.region[1]['vma']['readable'] + 263 unit.region[1]['vma']['writable'] + 264 unit.region[1]['vma']['executable'] + 265 unit.region[1]['vma']['private'])): 266 return False 267 if (self._sharedwith and 268 unit.pageframe and sharedwith not in self._sharedwith): 269 return False 270 271 return True 272 273 274 class MallocRule(AbstractRule): 275 """Represents a Rule to match with malloc'ed blocks.""" 276 def __init__(self, dct): 277 super(MallocRule, self).__init__(dct) 278 self._backtrace_function = dct.get('backtrace_function', None) 279 if self._backtrace_function: 280 self._backtrace_function = re.compile(self._backtrace_function) 281 self._backtrace_sourcefile = dct.get('backtrace_sourcefile', None) 282 if self._backtrace_sourcefile: 283 self._backtrace_sourcefile = re.compile(self._backtrace_sourcefile) 284 self._typeinfo = dct.get('typeinfo', None) 285 if self._typeinfo: 286 self._typeinfo = re.compile(self._typeinfo) 287 288 def __repr__(self): 289 result = cStringIO.StringIO() 290 result.write('{"%s"=>' % self._name) 291 attributes = [] 292 if self._backtrace_function: 293 attributes.append('backtrace_function: "%s"' % self._backtrace_function) 294 if self._typeinfo: 295 attributes.append('typeinfo: "%s"' % self._typeinfo) 296 result.write('%s}' % ', '.join(attributes)) 297 return result.getvalue() 298 299 def match(self, unit): 300 assert unit.bucket.allocator_type == 'malloc' 301 302 stackfunction = unit.bucket.symbolized_joined_stackfunction 303 stacksourcefile = unit.bucket.symbolized_joined_stacksourcefile 304 typeinfo = unit.bucket.symbolized_typeinfo 305 if typeinfo.startswith('0x'): 306 typeinfo = unit.bucket.typeinfo_name 307 308 return ((not self._backtrace_function or 309 self._backtrace_function.match(stackfunction)) and 310 (not self._backtrace_sourcefile or 311 self._backtrace_sourcefile.match(stacksourcefile)) and 312 (not self._typeinfo or self._typeinfo.match(typeinfo))) 313 314 315 class NoBucketMallocRule(MallocRule): 316 """Represents a Rule that small ignorable units match with.""" 317 def __init__(self): 318 super(NoBucketMallocRule, self).__init__({'name': 'tc-no-bucket'}) 319 self._no_bucket = True 320 321 @property 322 def no_bucket(self): 323 return self._no_bucket 324 325 326 class AbstractSorter(object): 327 """An abstract class for classifying Units with a set of Rules.""" 328 def __init__(self, dct): 329 self._type = 'sorter' 330 self._version = dct['version'] 331 self._world = dct['world'] 332 self._name = dct['name'] 333 self._root = dct.get('root', False) 334 self._order = dct['order'] 335 336 self._rules = [] 337 for rule in dct['rules']: 338 if dct['world'] == 'vm': 339 self._rules.append(VMRule(rule)) 340 elif dct['world'] == 'malloc': 341 self._rules.append(MallocRule(rule)) 342 else: 343 LOGGER.error('Unknown sorter world type') 344 345 def __repr__(self): 346 result = cStringIO.StringIO() 347 result.write('world=%s' % self._world) 348 result.write('order=%s' % self._order) 349 result.write('rules:') 350 for rule in self._rules: 351 result.write(' %s' % rule) 352 return result.getvalue() 353 354 @staticmethod 355 def load(filename): 356 with open(filename) as sorter_f: 357 sorter_dict = json.load(sorter_f) 358 if sorter_dict['world'] == 'vm': 359 return VMSorter(sorter_dict) 360 elif sorter_dict['world'] == 'malloc': 361 return MallocSorter(sorter_dict) 362 else: 363 LOGGER.error('Unknown sorter world type') 364 return None 365 366 @property 367 def world(self): 368 return self._world 369 370 @property 371 def name(self): 372 return self._name 373 374 @property 375 def root(self): 376 return self._root 377 378 def find(self, unit): 379 raise NotImplementedError() 380 381 def find_rule(self, name): 382 """Finds a rule whose name is |name|. """ 383 for rule in self._rules: 384 if rule.name == name: 385 return rule 386 return None 387 388 389 class VMSorter(AbstractSorter): 390 """Represents a Sorter for memory regions on virtual memory.""" 391 def __init__(self, dct): 392 assert dct['world'] == 'vm' 393 super(VMSorter, self).__init__(dct) 394 395 def find(self, unit): 396 for rule in self._rules: 397 if rule.match(unit): 398 return rule 399 assert False 400 401 402 class MallocSorter(AbstractSorter): 403 """Represents a Sorter for malloc'ed blocks.""" 404 def __init__(self, dct): 405 assert dct['world'] == 'malloc' 406 super(MallocSorter, self).__init__(dct) 407 self._no_bucket_rule = NoBucketMallocRule() 408 409 def find(self, unit): 410 if not unit.bucket: 411 return self._no_bucket_rule 412 assert unit.bucket.allocator_type == 'malloc' 413 414 if unit.bucket.component_cache: 415 return unit.bucket.component_cache 416 417 for rule in self._rules: 418 if rule.match(unit): 419 unit.bucket.component_cache = rule 420 return rule 421 assert False 422 423 424 class SorterTemplates(object): 425 """Represents a template for sorters.""" 426 def __init__(self, dct): 427 self._dict = dct 428 429 def as_dict(self): 430 return self._dict 431 432 @staticmethod 433 def load(filename): 434 with open(filename) as templates_f: 435 templates_dict = json.load(templates_f) 436 return SorterTemplates(templates_dict) 437 438 439 class SorterSet(object): 440 """Represents an iterable set of Sorters.""" 441 def __init__(self, additional=None, default=None): 442 if not additional: 443 additional = [] 444 if not default: 445 default = DEFAULT_SORTERS 446 self._sorters = {} 447 for filename in default + additional: 448 sorter = AbstractSorter.load(filename) 449 if sorter.world not in self._sorters: 450 self._sorters[sorter.world] = [] 451 self._sorters[sorter.world].append(sorter) 452 self._templates = SorterTemplates.load(DEFAULT_TEMPLATES) 453 454 def __repr__(self): 455 result = cStringIO.StringIO() 456 result.write(self._sorters) 457 return result.getvalue() 458 459 def __iter__(self): 460 for sorters in self._sorters.itervalues(): 461 for sorter in sorters: 462 yield sorter 463 464 def iter_world(self, world): 465 for sorter in self._sorters.get(world, []): 466 yield sorter 467 468 @property 469 def templates(self): 470 return self._templates 471
