1 #!/usr/bin/python 2 # Copyright (C) 2015 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import argparse, json, math, re, sys, zipfile 18 import xml.etree.ElementTree as ET 19 from collections import defaultdict, namedtuple 20 21 22 class Size(namedtuple('Size', ['width', 'height'])): 23 """A namedtuple with width and height fields.""" 24 def __str__(self): 25 return '%dx%d' % (self.width, self.height) 26 27 def nicekey(v): 28 """Returns a nicer sort key for sorting strings. 29 30 This sorts using lower case, with numbers in numerical order first.""" 31 key = [] 32 num = False 33 for p in re.split('(\d+)', v.lower()): 34 if num: 35 key.append(('0', int(p))) 36 elif p: 37 key.append((p, 0)) 38 num = not num 39 return key + [(v, 0)] 40 41 def nice(v): 42 """Returns a nicer representation for objects in debug messages. 43 44 Dictionaries are sorted, size is WxH, unicode removed, and floats have 1 digit precision.""" 45 if isinstance(v, dict): 46 return 'dict(' + ', '.join(k + '=' + nice(v) for k, v in sorted(v.items(), key=lambda i: nicekey(i[0]))) + ')' 47 if isinstance(v, str): 48 return repr(v) 49 if isinstance(v, int): 50 return str(v) 51 if isinstance(v, Size): 52 return repr(str(v)) 53 if isinstance(v, float): 54 return '%.1f' % v 55 if isinstance(v, type(u'')): 56 return repr(str(v)) 57 raise ValueError(v) 58 59 class ResultParser: 60 @staticmethod 61 def _intify(value): 62 """Returns a value converted to int if possible, else the original value.""" 63 try: 64 return int(value) 65 except ValueError: 66 return value 67 68 def _parseDict(self, value): 69 """Parses a MediaFormat from its string representation sans brackets.""" 70 return dict((k, self._intify(v)) 71 for k, v in re.findall(r'([^ =]+)=([^ [=]+(?:|\[[^\]]+\]))(?:, |$)', value)) 72 73 def _cleanFormat(self, format): 74 """Removes internal fields from a parsed MediaFormat.""" 75 format.pop('what', None) 76 format.pop('image-data', None) 77 78 MESSAGE_PATTERN = r'(?P<key>\w+)=(?P<value>\{[^}]*\}|[^ ,{}]+)' 79 80 def _parsePartialResult(self, message_match): 81 """Parses a partial test result conforming to the message pattern. 82 83 Returns: 84 A tuple of string key and int, string or dict value, where dict has 85 string keys mapping to int or string values. 86 """ 87 key, value = message_match.group('key', 'value') 88 if value.startswith('{'): 89 value = self._parseDict(value[1:-1]) 90 if key.endswith('Format'): 91 self._cleanFormat(value) 92 else: 93 value = self._intify(value) 94 return key, value 95 96 97 def perc(data, p, fn=round): 98 """Returns a percentile value from a sorted array. 99 100 Arguments: 101 data: sorted data 102 p: percentile value (0-100) 103 fn: method used for rounding the percentile to an integer index in data 104 """ 105 return data[int(fn((len(data) - 1) * p / 100))] 106 107 108 def genXml(data, A=None): 109 yield '<?xml version="1.0" encoding="utf-8" ?>' 110 yield '<!-- Copyright 2016 The Android Open Source Project' 111 yield '' 112 yield ' Licensed under the Apache License, Version 2.0 (the "License");' 113 yield ' you may not use this file except in compliance with the License.' 114 yield ' You may obtain a copy of the License at' 115 yield '' 116 yield ' http://www.apache.org/licenses/LICENSE-2.0' 117 yield '' 118 yield ' Unless required by applicable law or agreed to in writing, software' 119 yield ' distributed under the License is distributed on an "AS IS" BASIS,' 120 yield ' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.' 121 yield ' See the License for the specific language governing permissions and' 122 yield ' limitations under the License.' 123 yield '-->' 124 yield '' 125 yield '<MediaCodecs>' 126 last_section = None 127 from collections import namedtuple 128 Comp = namedtuple('Comp', 'is_decoder google mime name') 129 Result = namedtuple('Result', 'mn mx p95 med geo p5') 130 for comp_, cdata in sorted(data.items()): 131 comp = Comp(*comp_) 132 section = 'Decoders' if comp.is_decoder else 'Encoders' 133 if section != last_section: 134 if last_section: 135 yield ' </%s>' % last_section 136 yield ' <%s>' % section 137 last_section = section 138 yield ' <MediaCodec name="%s" type="%s" update="true">' % (comp.name, comp.mime) 139 for size, sdata in sorted(cdata.items()): 140 data = sorted(sdata) 141 N = len(data) 142 mn, mx = data[0], data[-1] 143 144 if N < 20 and not A.ignore: 145 raise ValueError("need at least 20 data points for %s size %s; have %s" % 146 (comp.name, size, N)) 147 148 TO = 2.2 # tolerance with margin 149 T = TO / 1.1 # tolerance without margin 150 151 Final = namedtuple('Final', 'comment c2 var qual') 152 lastFinal = None 153 for RG in (10, 15, 20, 25, 30, 40, 50): 154 P = 50./RG 155 quality = 0 156 p95, med, p5 = perc(data, P, math.floor), perc(data, 50, round), perc(data, 100 - P, math.ceil) 157 geo = math.sqrt(p5 * p95) 158 comment = '' 159 pub_lo, pub_hi = min(int(p95 * T), round(geo)), max(math.ceil(p5 / T), round(geo)) 160 if pub_lo > med: 161 if pub_lo > med * 1.1: 162 quality += 0.5 163 comment += ' SLOW' 164 pub_lo = int(med) 165 if N < 2 * RG: 166 comment += ' N=%d' % N 167 quality += 2 168 RGVAR = False 169 if p5 / p95 > T ** 3: 170 quality += 3 171 RGVAR = True 172 if pub_hi > pub_lo * TO: 173 quality += 1 174 if RG == 10: 175 # find best pub_lo and pub_hi 176 for i in range(N / 2): 177 pub_lo_, pub_hi_ = min(int(data[N / 2 - i - 1] * T), round(geo), int(med)), max(math.ceil(data[N / 2 + i] / T), round(geo)) 178 if pub_hi_ > pub_lo_ * TO: 179 # ??? 180 pub_lo = min(pub_lo, math.ceil(pub_hi_ / TO)) 181 break 182 pub_lo, pub_hi = pub_lo_, pub_hi_ 183 if mn < pub_lo / T or mx > pub_hi * T or pub_lo <= pub_hi / T: 184 quality += 1 185 comment += ' FLAKY(' 186 if round(mn, 1) < pub_lo / T: 187 comment += 'mn=%.1f < ' % mn 188 comment += 'RANGE' 189 if round(mx, 1) > pub_hi * T: 190 comment += ' < mx=%.1f' % mx 191 comment += ')' 192 if False: 193 comment += ' DATA(mn=%1.f p%d=%1.f accept=%1.f-%1.f p50=%1.f p%d=%1.f mx=%1.f)' % ( 194 mn, 100-P, p95, pub_lo / T, pub_hi * T, med, P, p5, mx) 195 var = math.sqrt(p5/p95) 196 if p95 < geo / T or p5 > geo * T: 197 if RGVAR: 198 comment += ' RG.VARIANCE:%.1f' % ((p5/p95) ** (1./3)) 199 else: 200 comment += ' variance:%.1f' % var 201 comment = comment.replace('RANGE', '%d - %d' % (math.ceil(pub_lo / T), int(pub_hi * T))) 202 c2 = '' 203 if N >= 2 * RG: 204 c2 += ' N=%d' % N 205 if var <= T or p5 / p95 > T ** 3: 206 c2 += ' v%d%%=%.1f' % (round(100 - 2 * P), var) 207 if A and A.dbg: 208 c2 += ' E=%s' % (str(quality)) 209 if c2: 210 c2 = ' <!--%s -->' % c2 211 212 if comment: 213 comment = ' <!-- measured %d%%:%d-%d med:%d%s -->' % (round(100 - 2 * P), int(p95), math.ceil(p5), int(round(med)), comment) 214 if A and A.dbg: yield '<!-- --> %s%s' % (comment, c2) 215 c2 = ' <Limit name="measured-frame-rate-%s" range="%d-%d" />%s' % (size, pub_lo, pub_hi, c2) 216 final = Final(comment, c2, var, quality) 217 if lastFinal and final.var > lastFinal.var * math.sqrt(1.3): 218 if A and A.dbg: yield '<!-- RANGE JUMP -->' 219 break 220 elif not lastFinal or quality <= lastFinal.qual: 221 lastFinal = final 222 if N < 2 * RG or quality >= 4: 223 break 224 comment, c2, var, quality = lastFinal 225 226 if comment: 227 yield comment 228 yield c2 229 yield ' </MediaCodec>' 230 if last_section: 231 yield ' </%s>' % last_section 232 yield '</MediaCodecs>' 233 234 235 class Data: 236 def __init__(self): 237 self.data = set() 238 self.kind = {} 239 self.devices = set() 240 self.parser = ResultParser() 241 242 def summarize(self, A=None): 243 devs = sorted(self.devices) 244 # device > (not encoder,goog,mime,codec) > size > fps 245 xmlInfo = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) 246 247 for mime, encoder, goog in sorted(set(self.kind.values())): 248 for dev, build, codec, size, num, std, avg, p0, p5, p10, p20, p30, p40, p50, p60, p70, p80, p90, p95, p100 in self.data: 249 if self.kind[codec] != (mime, encoder, goog): 250 continue 251 252 if p95 > 2: # ignore measurements at or below 2fps 253 xmlInfo[dev][(not encoder, goog, mime, codec)][size].append(p95) 254 else: 255 print >> sys.stderr, "warning: p95 value is suspiciously low: %s" % ( 256 nice(dict(config=dict(dev=dev, codec=codec, size=str(size), N=num), 257 data=dict(std=std, avg=avg, p0=p0, p5=p5, p10=p10, p20=p20, p30=p30, p40=p40, 258 p50=p50, p60=p60, p70=p70, p80=p80, p90=p90, p95=p95, p100=p100)))) 259 for dev, ddata in xmlInfo.items(): 260 outFile = '{}.media_codecs_performance.xml'.format(dev) 261 print >> sys.stderr, "generating", outFile 262 with open(outFile, "wt") as out: 263 for l in genXml(ddata, A=A): 264 out.write(l + '\n') 265 print l 266 print >> sys.stderr, "generated", outFile 267 268 def parse_fmt(self, fmt): 269 return self.parser._parseDict(fmt) 270 271 def parse_perf(self, a, device, build): 272 def rateFn(i): 273 if i is None: 274 return i 275 elif i == 0: 276 return 1e6 277 return 1000. / i 278 279 points = ('avg', 'min', 'p5', 'p10', 'p20', 'p30', 'p40', 'p50', 'p60', 'p70', 'p80', 'p90', 'p95', 'max') 280 a = dict(a) 281 codec = a['codec_name'] + '' 282 mime = a['mime_type'] 283 size = Size(a['width'], a['height']) 284 if 'decode_to' in a: 285 fmt = self.parse_fmt(a['output_format']) 286 ofmt = self.parse_fmt(a['input_format']) 287 else: 288 fmt = self.parse_fmt(a['input_format']) 289 ofmt = self.parse_fmt(a['output_format']) 290 size = Size(max(fmt['width'], ofmt['width']), max(fmt['height'], ofmt['height'])) 291 292 try: 293 prefix = 'time_avg_stats_' 294 if prefix + 'stdev' in a and a[prefix + 'avg']: 295 stdev = (a[prefix + 'stdev'] * 1e3 / a[prefix + 'avg'] ** 2) 296 data = ((device, build, codec, size, a[prefix + 'num'], stdev) + 297 tuple(rateFn(a.get(prefix + i)) for i in points)) 298 self.data.add(data) 299 self.kind[codec] = (mime, 'decode_to' not in a, codec.lower().startswith('omx.google.')) 300 self.devices.add(data[0]) 301 except (KeyError, ZeroDivisionError): 302 print >> sys.stderr, a 303 raise 304 305 def parse_json(self, json, device, build): 306 for test, results in json: 307 if test in ("video_encoder_performance", "video_decoder_performance"): 308 try: 309 if isinstance(results, list) and len(results[0]) and len(results[0][0]) == 2 and len(results[0][0][0]): 310 for result in results: 311 self.parse_perf(result, device, build) 312 else: 313 self.parse_perf(results, device, build) 314 except KeyboardInterrupt: 315 raise 316 317 def parse_result(self, result): 318 device, build = '', '' 319 if not result.endswith('.zip'): 320 print >> sys.stderr, "cannot parse %s" % result 321 return 322 323 try: 324 with zipfile.ZipFile(result) as zip: 325 resultInfo, testInfos = None, [] 326 for info in zip.infolist(): 327 if re.search(r'/GenericDeviceInfo.deviceinfo.json$', info.filename): 328 resultInfo = info 329 elif re.search(r'/Cts(Media|Video)TestCases\.reportlog\.json$', info.filename): 330 testInfos.append(info) 331 if resultInfo: 332 try: 333 jsonFile = zip.open(resultInfo) 334 jsonData = json.load(jsonFile, encoding='utf-8') 335 device, build = jsonData['build_device'], jsonData['build_id'] 336 except ValueError: 337 print >> sys.stderr, "could not parse %s" % resultInfo.filename 338 for info in testInfos: 339 jsonFile = zip.open(info) 340 try: 341 jsonData = json.load(jsonFile, encoding='utf-8', object_pairs_hook=lambda items: items) 342 except ValueError: 343 print >> sys.stderr, "cannot parse JSON in %s" % info.filename 344 self.parse_json(jsonData, device, build) 345 346 except zipfile.BadZipfile: 347 raise ValueError('bad zipfile') 348 349 350 P = argparse.ArgumentParser("gar_v2") 351 P.add_argument("--dbg", "-v", action='store_true', help="dump debug info into xml") 352 P.add_argument("--ignore", "-I", action='store_true', help="ignore minimum sample count") 353 P.add_argument("result_zip", nargs="*") 354 A = P.parse_args() 355 356 D = Data() 357 for res in A.result_zip: 358 D.parse_result(res) 359 D.summarize(A=A) 360
