1 # Copyright 2017 ARM Limited 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import re 16 import sys 17 import logging 18 import os.path 19 from collections import defaultdict 20 21 import devlib 22 from devlib.exception import TargetError 23 from devlib.module import Module 24 from devlib.platform import Platform 25 from devlib.platform.gem5 import Gem5SimulationPlatform 26 from devlib.utils.gem5 import iter_statistics_dump, GEM5STATS_ROI_NUMBER, GEM5STATS_DUMP_TAIL 27 28 29 class Gem5ROI: 30 def __init__(self, number, target): 31 self.target = target 32 self.number = number 33 self.running = False 34 self.field = 'ROI::{}'.format(number) 35 36 def start(self): 37 if self.running: 38 return False 39 self.target.execute('m5 roistart {}'.format(self.number)) 40 self.running = True 41 return True 42 43 def stop(self): 44 if not self.running: 45 return False 46 self.target.execute('m5 roiend {}'.format(self.number)) 47 self.running = False 48 return True 49 50 class Gem5StatsModule(Module): 51 ''' 52 Module controlling Region of Interest (ROIs) markers, satistics dump 53 frequency and parsing statistics log file when using gem5 platforms. 54 55 ROIs are identified by user-defined labels and need to be booked prior to 56 use. The translation of labels into gem5 ROI numbers will be performed 57 internally in order to avoid conflicts between multiple clients. 58 ''' 59 name = 'gem5stats' 60 61 @staticmethod 62 def probe(target): 63 return isinstance(target.platform, Gem5SimulationPlatform) 64 65 def __init__(self, target): 66 super(Gem5StatsModule, self).__init__(target) 67 self._current_origin = 0 68 self._stats_file_path = os.path.join(target.platform.gem5_out_dir, 69 'stats.txt') 70 self.rois = {} 71 self._dump_pos_cache = {0: 0} 72 73 def book_roi(self, label): 74 if label in self.rois: 75 raise KeyError('ROI label {} already used'.format(label)) 76 if len(self.rois) >= GEM5STATS_ROI_NUMBER: 77 raise RuntimeError('Too many ROIs reserved') 78 all_rois = set(xrange(GEM5STATS_ROI_NUMBER)) 79 used_rois = set([roi.number for roi in self.rois.values()]) 80 avail_rois = all_rois - used_rois 81 self.rois[label] = Gem5ROI(list(avail_rois)[0], self.target) 82 83 def free_roi(self, label): 84 if label not in self.rois: 85 raise KeyError('ROI label {} not reserved yet'.format(label)) 86 self.rois[label].stop() 87 del self.rois[label] 88 89 def roi_start(self, label): 90 if label not in self.rois: 91 raise KeyError('Incorrect ROI label: {}'.format(label)) 92 if not self.rois[label].start(): 93 raise TargetError('ROI {} was already running'.format(label)) 94 95 def roi_end(self, label): 96 if label not in self.rois: 97 raise KeyError('Incorrect ROI label: {}'.format(label)) 98 if not self.rois[label].stop(): 99 raise TargetError('ROI {} was not running'.format(label)) 100 101 def start_periodic_dump(self, delay_ns=0, period_ns=10000000): 102 # Default period is 10ms because it's roughly what's needed to have 103 # accurate power estimations 104 if delay_ns < 0 or period_ns < 0: 105 msg = 'Delay ({}) and period ({}) for periodic dumps must be positive' 106 raise ValueError(msg.format(delay_ns, period_ns)) 107 self.target.execute('m5 dumpresetstats {} {}'.format(delay_ns, period_ns)) 108 109 def match(self, keys, rois_labels, base_dump=0): 110 ''' 111 Extract specific values from the statistics log file of gem5 112 113 :param keys: a list of key name or regular expression patterns that 114 will be matched in the fields of the statistics file. ``match()`` 115 returns only the values of fields matching at least one these 116 keys. 117 :type keys: list 118 119 :param rois_labels: list of ROIs labels. ``match()`` returns the 120 values of the specified fields only during dumps spanned by at 121 least one of these ROIs. 122 :type rois_label: list 123 124 :param base_dump: dump number from which ``match()`` should operate. By 125 specifying a non-zero dump number, one can virtually truncate 126 the head of the stats file and ignore all dumps before a specific 127 instant. The value of ``base_dump`` will typically (but not 128 necessarily) be the result of a previous call to ``next_dump_no``. 129 Default value is 0. 130 :type base_dump: int 131 132 :returns: a dict indexed by key parameters containing a dict indexed by 133 ROI labels containing an in-order list of records for the key under 134 consideration during the active intervals of the ROI. 135 136 Example of return value: 137 * Result of match(['sim_'],['roi_1']): 138 { 139 'sim_inst': 140 { 141 'roi_1': [265300176, 267975881] 142 } 143 'sim_ops': 144 { 145 'roi_1': [324395787, 327699419] 146 } 147 'sim_seconds': 148 { 149 'roi_1': [0.199960, 0.199897] 150 } 151 'sim_freq': 152 { 153 'roi_1': [1000000000000, 1000000000000] 154 } 155 'sim_ticks': 156 { 157 'roi_1': [199960234227, 199896897330] 158 } 159 } 160 ''' 161 records = defaultdict(lambda : defaultdict(list)) 162 for record, active_rois in self.match_iter(keys, rois_labels, base_dump): 163 for key in record: 164 for roi_label in active_rois: 165 records[key][roi_label].append(record[key]) 166 return records 167 168 def match_iter(self, keys, rois_labels, base_dump=0): 169 ''' 170 Yield specific values dump-by-dump from the statistics log file of gem5 171 172 :param keys: same as ``match()`` 173 :param rois_labels: same as ``match()`` 174 :param base_dump: same as ``match()`` 175 :returns: a pair containing: 176 1. a dict storing the values corresponding to each of the found keys 177 2. the list of currently active ROIs among those passed as parameters 178 179 Example of return value: 180 * Result of match_iter(['sim_'],['roi_1', 'roi_2']).next() 181 ( 182 { 183 'sim_inst': 265300176, 184 'sim_ops': 324395787, 185 'sim_seconds': 0.199960, 186 'sim_freq': 1000000000000, 187 'sim_ticks': 199960234227, 188 }, 189 [ 'roi_1 ' ] 190 ) 191 ''' 192 for label in rois_labels: 193 if label not in self.rois: 194 raise KeyError('Impossible to match ROI label {}'.format(label)) 195 if self.rois[label].running: 196 self.logger.warning('Trying to match records in statistics file' 197 ' while ROI {} is running'.format(label)) 198 199 # Construct one large regex that concatenates all keys because 200 # matching one large expression is more efficient than several smaller 201 all_keys_re = re.compile('|'.join(keys)) 202 203 def roi_active(roi_label, dump): 204 roi = self.rois[roi_label] 205 return (roi.field in dump) and (int(dump[roi.field]) == 1) 206 207 with open(self._stats_file_path, 'r') as stats_file: 208 self._goto_dump(stats_file, base_dump) 209 for dump in iter_statistics_dump(stats_file): 210 active_rois = [l for l in rois_labels if roi_active(l, dump)] 211 if active_rois: 212 rec = {k: dump[k] for k in dump if all_keys_re.search(k)} 213 yield (rec, active_rois) 214 215 def next_dump_no(self): 216 ''' 217 Returns the number of the next dump to be written to the stats file. 218 219 For example, if next_dump_no is called while there are 5 (0 to 4) full 220 dumps in the stats file, it will return 5. This will be usefull to know 221 from which dump one should match() in the future to get only data from 222 now on. 223 ''' 224 with open(self._stats_file_path, 'r') as stats_file: 225 # _goto_dump reach EOF and returns the total number of dumps + 1 226 return self._goto_dump(stats_file, sys.maxint) 227 228 def _goto_dump(self, stats_file, target_dump): 229 if target_dump < 0: 230 raise HostError('Cannot go to dump {}'.format(target_dump)) 231 232 # Go to required dump quickly if it was visited before 233 if target_dump in self._dump_pos_cache: 234 stats_file.seek(self._dump_pos_cache[target_dump]) 235 return target_dump 236 # Or start from the closest dump already visited before the required one 237 prev_dumps = filter(lambda x: x < target_dump, self._dump_pos_cache.keys()) 238 curr_dump = max(prev_dumps) 239 curr_pos = self._dump_pos_cache[curr_dump] 240 stats_file.seek(curr_pos) 241 242 # And iterate until target_dump 243 dump_iterator = iter_statistics_dump(stats_file) 244 while curr_dump < target_dump: 245 try: 246 dump = dump_iterator.next() 247 except StopIteration: 248 break 249 # End of passed dump is beginning og next one 250 curr_pos = stats_file.tell() 251 curr_dump += 1 252 self._dump_pos_cache[curr_dump] = curr_pos 253 return curr_dump 254 255
