Home | History | Annotate | Download | only in scheduler 
      1 # Compute and gather statistics about garbage collection in this process.
      2 # This module depends on the CPython gc module and garbage collection behavior.
      3 
      4 import gc, logging, pprint
      5 
      6 
      7 verbose = False
      8 
      9 
     10 # A mapping from type objects to a count of instances of those types in the
     11 # garbage collectors all objects list on the previous call to
     12 # _log_garbage_collector_stats().
     13 _previous_obj_type_map = {}
     14 
     15 
     16 # A set of object ids for everything in the all objects list on the
     17 # previous call to _log_garbage_collector_stats().
     18 _previous_obj_ids = set()
     19 
     20 
     21 def _log_garbage_collector_stats(minimum_count=10):
     22     """
     23     Log statistics about how many of what type of Python object exist in this
     24     process.
     25 
     26     @param minimum_count: The minimum number of instances of a type for it
     27             to be considered worthy of logging.
     28     """
     29     global _previous_obj_type_map
     30     global _previous_obj_ids
     31 
     32     # We get all objects -before- creating any new objects within this function.
     33     # to avoid having our own local instances in the list.
     34     all_objects = gc.get_objects()
     35     obj = None
     36     new_objects = []
     37     try:
     38         obj_type_map = {}
     39         object_ids = set()
     40         for obj in all_objects:
     41             obj_type = type(obj)
     42             obj_type_map.setdefault(obj_type, 0)
     43             obj_type_map[obj_type] += 1
     44             object_ids.add(id(obj))
     45         whats_new_big_str = ''
     46         if verbose and _previous_obj_ids:
     47             new_object_ids = object_ids - _previous_obj_ids
     48             for obj in all_objects:
     49                 if id(obj) in new_object_ids:
     50                     new_objects.append(obj)
     51             whats_new_big_str = pprint.pformat(new_objects, indent=1)
     52     finally:
     53         # Never keep references to stuff returned by gc.get_objects() around
     54         # or it'll just make the future cyclic gc runs more difficult.
     55         del all_objects
     56         del obj
     57         del new_objects
     58 
     59 
     60     delta = {}
     61     for obj_type, count in obj_type_map.iteritems():
     62         if obj_type not in _previous_obj_type_map:
     63             delta[obj_type] = count
     64         elif _previous_obj_type_map[obj_type] != count:
     65             delta[obj_type] = count - _previous_obj_type_map[obj_type]
     66 
     67     sorted_stats = reversed(sorted(
     68             (count, obj_type) for obj_type, count in obj_type_map.iteritems()))
     69     sorted_delta = reversed(sorted(
     70             (count, obj_type) for obj_type, count in delta.iteritems()))
     71 
     72     logging.debug('Garbage collector object type counts:')
     73     for count, obj_type in sorted_stats:
     74         if count >= minimum_count:
     75             logging.debug('  %d\t%s', count, obj_type)
     76 
     77     logging.info('Change in object counts since previous GC stats:')
     78     for change, obj_type in sorted_delta:
     79         if obj_type_map[obj_type] > minimum_count:
     80             logging.info('  %+d\t%s\tto %d', change, obj_type,
     81                          obj_type_map[obj_type])
     82 
     83     if verbose and whats_new_big_str:
     84         logging.debug('Pretty printed representation of the new objects:')
     85         logging.debug(whats_new_big_str)
     86 
     87     _previous_obj_type_map = obj_type_map
     88     if verbose:
     89         _previous_obj_ids = object_ids
     90