1 #!/usr/bin/env python 2 # SPDX-License-Identifier: Apache-2.0 3 # 4 # Copyright (C) 2017, ARM Limited, Google, and contributors. 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 # not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 # 18 # 19 # Analysis of framestats deltas between 2+ runs 20 # This analysis outputs potential regressions when comparing 21 # a baseline folder output of microbenchmarks like UiBench with 22 # a test folder with similar stats. The base and test folders 23 # can contain the output of one or more runs of run_uibench.py or 24 # similar script. 25 26 # For example: 27 # python framestats_analysis.py --baseline_dir=/usr/local/google/home/andresoportus/b/master2/external/lisa/results/UiBench_walleye_em_baseline_b/ --results_dir=/usr/local/google/home/andresoportus/b/master2/external/lisa/results/UiBench_walleye_em_first/ --threshold_ave 0.322 28 # No handlers could be found for logger "EnergyModel" 29 # Potential regression in: 30 # avg-frame-time-50 avg-frame-time-90 avg-frame-time-95 avg-frame-time-99 31 # UiBenchJankTests#testTrivialAnimation 32 # all base [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5.2 5.1 5.2 5.9 5.2 5.2] 33 # all test [ 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5.] [ 5.4 6. 5.3 5.8 6. 5.1] 34 # ave base [ 5.] [ 5.] [ 5.] [ 5.25714286] 35 # ave test [ 5.] [ 5.] [ 5.] [ 5.6] 36 # avg delta [ 0.] [ 0.] [ 0.] [ 0.34285714] 37 38 import os 39 import sys 40 import argparse 41 import numpy as np 42 import pandas as pd 43 from collections import OrderedDict 44 from android.workloads.uibench import UiBench 45 46 averages = ['avg-frame-time-50', 'avg-frame-time-90', 47 'avg-frame-time-95', 'avg-frame-time-99'] 48 stats_file = 'framestats.txt' 49 50 """ 51 Parses a directory to find stats 52 53 :param stats_dir: path to stats dir 54 :type stats_dir: str 55 56 :param stats_file: name of stats file 57 :type stats_file: str 58 """ 59 def parse_stats_dir(stats_dir, stats_file): 60 df = {} 61 run_name = os.path.basename(os.path.normpath(stats_dir)) 62 for root, dirs, files in os.walk(stats_dir): 63 if stats_file in files: 64 test_name = os.path.basename(os.path.normpath(root)) 65 if test_name not in df: 66 df[test_name] = pd.DataFrame() 67 df[test_name] = df[test_name].append(UiBench.get_results(root)) 68 return df 69 70 """ 71 Prints a header line 72 73 :param df: pandas dataframe to extract columns names from 74 :type df: pd.DataFrame 75 """ 76 def print_header(df): 77 sys.stdout.write('{:16}'.format('')) 78 for c in df.columns: 79 sys.stdout.write(' {:32}'.format(str(c))) 80 sys.stdout.write('\n') 81 82 """ 83 Prints a pandas DataFrame as a row 84 85 :param df: pandas dataframe to extract values from 86 :type df: pd.DataFrame 87 """ 88 def print_as_row(name, df): 89 sys.stdout.write('{:16}'.format(name)) 90 for c in df.columns: 91 sys.stdout.write(' {:32}'.format(str(df[c].values))) 92 sys.stdout.write('\n') 93 94 def main(): 95 header_printed = False 96 pd.set_option('precision', 2) 97 base = parse_stats_dir(args.baseline_dir, stats_file) 98 test = parse_stats_dir(args.results_dir, stats_file) 99 for name in base: 100 try: 101 # DataFrame created from the Series output of mean() needs to 102 # be transposed to make averages be columns again 103 ave_base = pd.DataFrame(base[name][averages].mean()).T 104 ave_test = pd.DataFrame(test[name][averages].mean()).T 105 if args.verbose or \ 106 ((ave_test - ave_base) > args.threshold_ave).iloc[0].any(): 107 if not header_printed: 108 if not args.verbose: 109 print "Potential regression in:" 110 print_header(base[name][averages]) 111 header_printed = True 112 print name 113 print_as_row('all base', base[name][averages]) 114 print_as_row('all test', test[name][averages]) 115 print_as_row('ave base', ave_base) 116 print_as_row('ave test', ave_test) 117 print_as_row('avg delta', ave_test - ave_base) 118 except KeyError: 119 sys.stdout.write('\n') 120 if not header_printed: 121 print "No regression found" 122 123 if __name__ == "__main__": 124 parser = argparse.ArgumentParser( 125 description="Framestats analysis") 126 127 parser.add_argument("--results_dir", "-d", type=str, 128 default=os.path.join(os.environ["LISA_HOME"], 129 "results/UiBench_default"), 130 help="The test directory to read from. (default \ 131 LISA_HOME/restuls/UiBench_deafult)") 132 parser.add_argument("--baseline_dir", "-b", type=str, required=True, 133 help="The directory that provides baseline test to \ 134 compare against") 135 parser.add_argument("--threshold_ave", "-t", type=float, default=0.99, 136 help="Amount to filter noise in average values") 137 parser.add_argument("--verbose", "-v", action='store_true', 138 help="Verbose output") 139 args = parser.parse_args() 140 main() 141
