1 #!/usr/bin/python 2 3 import optparse 4 import sys 5 import sqlite3 6 import scipy.stats 7 import numpy 8 from math import log10, floor 9 import matplotlib 10 11 matplotlib.use("Agg") 12 13 import matplotlib.pyplot as plt 14 import pylab 15 16 import adbutil 17 from devices import DEVICES 18 19 DB_PATH="/data/data/com.android.benchmark/databases/BenchmarkResults" 20 OUT_PATH = "db/" 21 22 QUERY_BAD_FRAME = ("select run_id, name, iteration, total_duration from ui_results " 23 "where total_duration >= 16 order by run_id, name, iteration") 24 QUERY_PERCENT_JANK = ("select run_id, name, iteration, sum(jank_frame) as jank_count, count (*) as total " 25 "from ui_results group by run_id, name, iteration") 26 27 SKIP_TESTS = [ 28 # "BMUpload", 29 # "Low-hitrate text render", 30 # "High-hitrate text render", 31 # "Edit Text Input", 32 # "List View Fling" 33 ] 34 35 INCLUDE_TESTS = [ 36 #"BMUpload" 37 #"Shadow Grid Fling" 38 #"Image List View Fling" 39 #"Edit Text Input" 40 ] 41 42 class IterationResult: 43 def __init__(self): 44 self.durations = [] 45 self.jank_count = 0 46 self.total_count = 0 47 48 49 def get_scoremap(dbpath): 50 db = sqlite3.connect(dbpath) 51 rows = db.execute(QUERY_BAD_FRAME) 52 53 scoremap = {} 54 for row in rows: 55 run_id = row[0] 56 name = row[1] 57 iteration = row[2] 58 total_duration = row[3] 59 60 if not run_id in scoremap: 61 scoremap[run_id] = {} 62 63 if not name in scoremap[run_id]: 64 scoremap[run_id][name] = {} 65 66 if not iteration in scoremap[run_id][name]: 67 scoremap[run_id][name][iteration] = IterationResult() 68 69 scoremap[run_id][name][iteration].durations.append(float(total_duration)) 70 71 for row in db.execute(QUERY_PERCENT_JANK): 72 run_id = row[0] 73 name = row[1] 74 iteration = row[2] 75 jank_count = row[3] 76 total_count = row[4] 77 78 if run_id in scoremap.keys() and name in scoremap[run_id].keys() and iteration in scoremap[run_id][name].keys(): 79 scoremap[run_id][name][iteration].jank_count = long(jank_count) 80 scoremap[run_id][name][iteration].total_count = long(total_count) 81 82 db.close() 83 return scoremap 84 85 def round_to_2(val): 86 return val 87 if val == 0: 88 return val 89 return round(val , -int(floor(log10(abs(val)))) + 1) 90 91 def score_device(name, serial, pull = False, verbose = False): 92 dbpath = OUT_PATH + name + ".db" 93 94 if pull: 95 adbutil.root(serial) 96 adbutil.pull(serial, DB_PATH, dbpath) 97 98 scoremap = None 99 try: 100 scoremap = get_scoremap(dbpath) 101 except sqlite3.DatabaseError: 102 print "Database corrupt, fetching..." 103 adbutil.root(serial) 104 adbutil.pull(serial, DB_PATH, dbpath) 105 scoremap = get_scoremap(dbpath) 106 107 per_test_score = {} 108 per_test_sample_count = {} 109 global_overall = {} 110 111 for run_id in iter(scoremap): 112 overall = [] 113 if len(scoremap[run_id]) < 1: 114 if verbose: 115 print "Skipping short run %s" % run_id 116 continue 117 print "Run: %s" % run_id 118 for test in iter(scoremap[run_id]): 119 if test in SKIP_TESTS: 120 continue 121 if INCLUDE_TESTS and test not in INCLUDE_TESTS: 122 continue 123 if verbose: 124 print "\t%s" % test 125 scores = [] 126 means = [] 127 stddevs = [] 128 pjs = [] 129 sample_count = 0 130 hit_min_count = 0 131 # try pooling together all iterations 132 for iteration in iter(scoremap[run_id][test]): 133 res = scoremap[run_id][test][iteration] 134 stddev = round_to_2(numpy.std(res.durations)) 135 mean = round_to_2(numpy.mean(res.durations)) 136 sample_count += len(res.durations) 137 pj = round_to_2(100 * res.jank_count / float(res.total_count)) 138 score = stddev * mean * pj 139 score = 100 * len(res.durations) / float(res.total_count) 140 if score == 0: 141 score = 1 142 scores.append(score) 143 means.append(mean) 144 stddevs.append(stddev) 145 pjs.append(pj) 146 if verbose: 147 print "\t%s: Score = %f x %f x %f = %f (%d samples)" % (iteration, stddev, mean, pj, score, len(res.durations)) 148 149 if verbose: 150 print "\tHit min: %d" % hit_min_count 151 print "\tMean Variation: %0.2f%%" % (100 * scipy.stats.variation(means)) 152 print "\tStdDev Variation: %0.2f%%" % (100 * scipy.stats.variation(stddevs)) 153 print "\tPJ Variation: %0.2f%%" % (100 * scipy.stats.variation(pjs)) 154 155 geo_run = numpy.mean(scores) 156 if test not in per_test_score: 157 per_test_score[test] = [] 158 159 if test not in per_test_sample_count: 160 per_test_sample_count[test] = [] 161 162 sample_count /= len(scoremap[run_id][test]) 163 164 per_test_score[test].append(geo_run) 165 per_test_sample_count[test].append(int(sample_count)) 166 overall.append(geo_run) 167 168 if not verbose: 169 print "\t%s:\t%0.2f (%0.2f avg. sample count)" % (test, geo_run, sample_count) 170 else: 171 print "\tOverall:\t%0.2f (%0.2f avg. sample count)" % (geo_run, sample_count) 172 print "" 173 174 global_overall[run_id] = scipy.stats.gmean(overall) 175 print "Run Overall: %f" % global_overall[run_id] 176 print "" 177 178 print "" 179 print "Variability (CV) - %s:" % name 180 181 worst_offender_test = None 182 worst_offender_variation = 0 183 for test in per_test_score: 184 variation = 100 * scipy.stats.variation(per_test_score[test]) 185 if worst_offender_variation < variation: 186 worst_offender_test = test 187 worst_offender_variation = variation 188 print "\t%s:\t%0.2f%% (%0.2f avg sample count)" % (test, variation, numpy.mean(per_test_sample_count[test])) 189 190 print "\tOverall: %0.2f%%" % (100 * scipy.stats.variation([x for x in global_overall.values()])) 191 print "" 192 193 return { 194 "overall": global_overall.values(), 195 "worst_offender_test": (name, worst_offender_test, worst_offender_variation) 196 } 197 198 def parse_options(argv): 199 usage = 'Usage: %prog [options]' 200 desc = 'Example: %prog' 201 parser = optparse.OptionParser(usage=usage, description=desc) 202 parser.add_option("-p", dest='pull', action="store_true") 203 parser.add_option("-d", dest='device', action="store") 204 parser.add_option("-v", dest='verbose', action="store_true") 205 options, categories = parser.parse_args(argv[1:]) 206 return options 207 208 def main(): 209 options = parse_options(sys.argv) 210 if options.device != None: 211 score_device(options.device, DEVICES[options.device], options.pull, options.verbose) 212 else: 213 device_scores = [] 214 worst_offenders = [] 215 for name, serial in DEVICES.iteritems(): 216 print "======== %s =========" % name 217 result = score_device(name, serial, options.pull, options.verbose) 218 device_scores.append((name, result["overall"])) 219 worst_offenders.append(result["worst_offender_test"]) 220 221 222 device_scores.sort(cmp=(lambda x, y: cmp(x[1], y[1]))) 223 print "Ranking by max overall score:" 224 for name, score in device_scores: 225 plt.plot([0, 1, 2, 3, 4, 5], score, label=name) 226 print "\t%s: %s" % (name, score) 227 228 plt.ylabel("Jank %") 229 plt.xlabel("Iteration") 230 plt.title("Jank Percentage") 231 plt.legend() 232 pylab.savefig("holy.png", bbox_inches="tight") 233 234 print "Worst offender tests:" 235 for device, test, variation in worst_offenders: 236 print "\t%s: %s %.2f%%" % (device, test, variation) 237 238 if __name__ == "__main__": 239 main() 240 241
