1 # Copyright 2015-2017 ARM Limited 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # 15 16 """This module provides the Constraint class for handling 17 filters and pivots in a modular fashion. This enable easy 18 constraint application. 19 20 An implementation of :mod:`trappy.plotter.AbstractDataPlotter` 21 is expected to use the :mod:`trappy.plotter.Constraint.ConstraintManager` 22 class to pivot and filter data and handle multiple column, 23 trace and event inputs. 24 25 The underlying object that encapsulates a unique set of 26 a data column, data event and the requisite filters is 27 :mod:`trappy.plotter.Constraint.Constraint` 28 """ 29 # pylint: disable=R0913 30 from trappy.plotter.Utils import decolonize, normalize_list 31 from trappy.utils import listify 32 from trappy.plotter import AttrConf 33 34 35 class Constraint(object): 36 37 """ 38 What is a Constraint? 39 It is collection of data based on two rules: 40 41 - A Pivot 42 43 - A Set of Filters 44 45 - A Data Column 46 47 For Example a :mod:`pandas.DataFrame` 48 49 ===== ======== ========= 50 Time CPU Latency 51 ===== ======== ========= 52 1 x <val> 53 2 y <val> 54 3 z <val> 55 4 a <val> 56 ===== ======== ========= 57 58 The resultant data will be split for each unique pivot value 59 with the filters applied 60 :: 61 62 result["x"] = pd.Series.filtered() 63 result["y"] = pd.Series.filtered() 64 result["z"] = pd.Series.filtered() 65 result["a"] = pd.Series.filtered() 66 67 68 :param trappy_trace: Input Data 69 :type trappy_trace: :mod:`pandas.DataFrame` or a class derived from 70 :mod:`trappy.trace.BareTrace` 71 72 :param column: The data column 73 :type column: str 74 75 :param template: TRAPpy Event 76 :type template: :mod:`trappy.base.Base` event 77 78 :param trace_index: The index of the trace/data in the overall constraint 79 data 80 :type trace_index: int 81 82 :param filters: A dictionary of filter values 83 :type filters: dict 84 85 :param window: A time window to apply to the constraint. 86 E.g. window=(5, 20) will constraint to events that happened 87 between Time=5 to Time=20. 88 :type window: tuple of two ints 89 90 """ 91 92 def __init__(self, trappy_trace, pivot, column, template, trace_index, 93 filters, window): 94 self._trappy_trace = trappy_trace 95 self._filters = filters 96 self._pivot = pivot 97 self.column = column 98 self._template = template 99 self._dup_resolved = False 100 self._data = self.populate_data_frame() 101 102 if window: 103 # We want to include the previous value before the window 104 # and the next after the window in the dataset 105 min_idx = self._data.loc[:window[0]].index.max() 106 max_idx = self._data.loc[window[1]:].index.min() 107 self._data = self._data.loc[min_idx:max_idx] 108 109 self.result = self._apply() 110 self.trace_index = trace_index 111 112 def _apply(self): 113 """This method applies the filter on the resultant data 114 on the input column. 115 """ 116 data = self._data 117 result = {} 118 119 try: 120 values = data[self.column] 121 except KeyError: 122 return result 123 124 if self._pivot == AttrConf.PIVOT: 125 pivot_vals = [AttrConf.PIVOT_VAL] 126 else: 127 pivot_vals = self.pivot_vals(data) 128 129 for pivot_val in pivot_vals: 130 criterion = values.map(lambda x: True) 131 132 for key in self._filters.keys(): 133 if key != self._pivot and key in data.columns: 134 criterion = criterion & data[key].map( 135 lambda x: x in self._filters[key]) 136 137 if pivot_val != AttrConf.PIVOT_VAL: 138 criterion &= data[self._pivot] == pivot_val 139 140 val_series = values[criterion] 141 if len(val_series) != 0: 142 result[pivot_val] = val_series 143 144 return result 145 146 def _uses_trappy_trace(self): 147 if not self._template: 148 return False 149 else: 150 return True 151 152 def populate_data_frame(self): 153 """Return the populated :mod:`pandas.DataFrame`""" 154 if not self._uses_trappy_trace(): 155 return self._trappy_trace 156 157 data_container = getattr( 158 self._trappy_trace, 159 decolonize(self._template.name)) 160 return data_container.data_frame 161 162 def pivot_vals(self, data): 163 """This method returns the unique pivot values for the 164 Constraint's pivot and the column 165 166 :param data: Input Data 167 :type data: :mod:`pandas.DataFrame` 168 """ 169 if self._pivot == AttrConf.PIVOT: 170 return AttrConf.PIVOT_VAL 171 172 if self._pivot not in data.columns: 173 return [] 174 175 pivot_vals = set(data[self._pivot]) 176 if self._pivot in self._filters: 177 pivot_vals = pivot_vals & set(self._filters[self._pivot]) 178 179 return list(pivot_vals) 180 181 def __str__(self): 182 183 name = self.get_data_name() 184 185 if not self._uses_trappy_trace(): 186 return name + ":" + str(self.column) 187 188 return name + ":" + \ 189 self._template.name + ":" + self.column 190 191 192 def get_data_name(self): 193 """Get name for the data member. This method 194 relies on the "name" attribute for the name. 195 If the name attribute is absent, it associates 196 a numeric name to the respective data element 197 198 :returns: The name of the data member 199 """ 200 if self._uses_trappy_trace(): 201 if self._trappy_trace.name != "": 202 return self._trappy_trace.name 203 else: 204 return "Trace {}".format(self.trace_index) 205 else: 206 return "DataFrame {}".format(self.trace_index) 207 208 class ConstraintManager(object): 209 210 """A class responsible for converting inputs 211 to constraints and also ensuring sanity 212 213 214 :param traces: Input Trace data 215 :type traces: :mod:`trappy.trace.BareTrace`, list(:mod:`trappy.trace.BareTrace`) 216 (or a class derived from :mod:`trappy.trace.BareTrace`) 217 :param columns: The column values from the corresponding 218 :mod:`pandas.DataFrame` 219 :type columns: str, list(str) 220 :param pivot: The column around which the data will be 221 pivoted: 222 :type pivot: str 223 :param templates: TRAPpy events 224 :type templates: :mod:`trappy.base.Base` 225 :param filters: A dictionary of values to be applied on the 226 respective columns 227 :type filters: dict 228 :param window: A time window to apply to the constraints 229 :type window: tuple of ints 230 :param zip_constraints: Permutes the columns and traces instead 231 of a one-to-one correspondence 232 :type zip_constraints: bool 233 """ 234 235 def __init__(self, traces, columns, templates, pivot, filters, 236 window=None, zip_constraints=True): 237 238 self._ip_vec = [] 239 self._ip_vec.append(listify(traces)) 240 self._ip_vec.append(listify(columns)) 241 self._ip_vec.append(listify(templates)) 242 243 self._lens = map(len, self._ip_vec) 244 self._max_len = max(self._lens) 245 self._pivot = pivot 246 self._filters = filters 247 self.window = window 248 self._constraints = [] 249 250 self._trace_expanded = False 251 self._expand() 252 if zip_constraints: 253 self._populate_zip_constraints() 254 else: 255 self._populate_constraints() 256 257 def _expand(self): 258 """This is really important. We need to 259 meet the following criteria for constraint 260 expansion: 261 :: 262 263 Len[traces] == Len[columns] == Len[templates] 264 265 Or: 266 :: 267 268 Permute( 269 Len[traces] = 1 270 Len[columns] = 1 271 Len[templates] != 1 272 ) 273 274 Permute( 275 Len[traces] = 1 276 Len[columns] != 1 277 Len[templates] != 1 278 ) 279 """ 280 min_len = min(self._lens) 281 max_pos_comp = [ 282 i for i, 283 j in enumerate( 284 self._lens) if j != self._max_len] 285 286 if self._max_len == 1 and min_len != 1: 287 raise RuntimeError("Essential Arg Missing") 288 289 if self._max_len > 1: 290 291 # Are they all equal? 292 if len(set(self._lens)) == 1: 293 return 294 295 if min_len > 1: 296 raise RuntimeError("Cannot Expand a list of Constraints") 297 298 for val in max_pos_comp: 299 if val == 0: 300 self._trace_expanded = True 301 self._ip_vec[val] = normalize_list(self._max_len, 302 self._ip_vec[val]) 303 304 def _populate_constraints(self): 305 """Populate the constraints creating one for each column in 306 each trace 307 308 In a multi-trace, multicolumn scenario, constraints are created for 309 all the columns in each of the traces. _populate_constraints() 310 creates one constraint for the first trace and first column, the 311 next for the second trace and second column,... This function 312 creates a constraint for every combination of traces and columns 313 possible. 314 """ 315 316 for trace_idx, trace in enumerate(self._ip_vec[0]): 317 for col in self._ip_vec[1]: 318 template = self._ip_vec[2][trace_idx] 319 constraint = Constraint(trace, self._pivot, col, template, 320 trace_idx, self._filters, self.window) 321 self._constraints.append(constraint) 322 323 def get_column_index(self, constraint): 324 return self._ip_vec[1].index(constraint.column) 325 326 def _populate_zip_constraints(self): 327 """Populate the expanded constraints 328 329 In a multitrace, multicolumn scenario, create constraints for 330 the first trace and the first column, second trace and second 331 column,... that is, as if you run zip(traces, columns) 332 """ 333 334 for idx in range(self._max_len): 335 if self._trace_expanded: 336 trace_idx = 0 337 else: 338 trace_idx = idx 339 340 trace = self._ip_vec[0][idx] 341 col = self._ip_vec[1][idx] 342 template = self._ip_vec[2][idx] 343 self._constraints.append( 344 Constraint(trace, self._pivot, col, template, trace_idx, 345 self._filters, self.window)) 346 347 def generate_pivots(self, permute=False): 348 """Return a union of the pivot values 349 350 :param permute: Permute the Traces and Columns 351 :type permute: bool 352 """ 353 pivot_vals = [] 354 for constraint in self._constraints: 355 pivot_vals += constraint.result.keys() 356 357 p_list = list(set(pivot_vals)) 358 traces = range(self._lens[0]) 359 360 try: 361 sorted_plist = sorted(p_list, key=int) 362 except (ValueError, TypeError): 363 try: 364 sorted_plist = sorted(p_list, key=lambda x: int(x, 16)) 365 except (ValueError, TypeError): 366 sorted_plist = sorted(p_list) 367 368 if permute: 369 pivot_gen = ((trace_idx, pivot) for trace_idx in traces for pivot in sorted_plist) 370 return pivot_gen, len(sorted_plist) * self._lens[0] 371 else: 372 return sorted_plist, len(sorted_plist) 373 374 def constraint_labels(self): 375 """ 376 :return: string to represent the 377 set of Constraints 378 379 """ 380 return map(str, self._constraints) 381 382 def __len__(self): 383 return len(self._constraints) 384 385 def __iter__(self): 386 return iter(self._constraints) 387
