1 #!/usr/bin/env python 2 # SPDX-License-Identifier: Apache-2.0 3 # 4 # Copyright (C) 2017, ARM Limited, Google, and contributors. 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 # not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 # 18 19 import os 20 import pandas as pd 21 import sqlite3 22 import argparse 23 import shutil 24 25 # Run a comparison experiment between 2 Jankbench result directories 26 # containing multiple subtests. This makes it easy to compare results 27 # between 2 jankbench runs. 28 # 29 # Sample run: 30 # ./compare_jankbench.py --baseline='./results/Jankbench_baseline' 31 # --compare-with='./results/Jankbench_kernelchange' 32 # 33 # The output will be something like (only showing 25% and 50%): 34 # 25% compare 25%_diff 50%_compare 50%_diff 35 # test_name 36 # image_list_view 2.11249 0.0178108 5.7952 0.0242445 37 # list_view 2.02227 -3.65839 5.74957 -0.095421 38 # shadow_grid 6.00877 -0.000898 6.23746 -0.0057695 39 # high_hitrate_text 5.81625 0.0264913 6.03504 0.0017795 40 # 41 # (Note that baseline_df is only used for calculations. 42 43 JANKBENCH_DB_NAME = 'BenchmarkResults' 44 45 def get_results(out_dir): 46 """ 47 Extract data from results db and return as a pandas dataframe 48 49 :param out_dir: Output directory for a run of the Jankbench workload 50 :type out_dir: str 51 """ 52 path = os.path.join(out_dir, JANKBENCH_DB_NAME) 53 columns = ['_id', 'name', 'run_id', 'iteration', 'total_duration', 'jank_frame'] 54 data = [] 55 conn = sqlite3.connect(path) 56 for row in conn.execute('SELECT {} FROM ui_results'.format(','.join(columns))): 57 data.append(row) 58 return pd.DataFrame(data, columns=columns) 59 60 def build_stats_df(test_outdir): 61 """ 62 Build a .describe() df with statistics 63 """ 64 stats_dfs = [] 65 for t in tests: 66 test_dir = os.path.join(test_outdir, t) 67 res_df = get_results(test_dir).describe(percentiles=[0.25,0.5,0.75,0.9,0.95,0.99]) 68 stats_df = res_df['total_duration'] 69 stats_df['test_name'] = t 70 stats_dfs.append(stats_df) 71 fdf = pd.concat(stats_dfs, axis = 1).T 72 fdf.set_index('test_name', inplace=True) 73 return fdf 74 75 76 parser = argparse.ArgumentParser(description='Jankbench comparisons') 77 78 parser.add_argument('--baseline', dest='baseline', action='store', default='default', 79 required=True, help='baseline out directory') 80 81 parser.add_argument('--compare-with', dest='compare_with', action='store', default='default', 82 required=True, help='out directory to compare with baseline') 83 84 args = parser.parse_args() 85 86 # Get list of Jankbench tests available 87 tests = os.listdir(args.baseline) 88 tests = [t for t in tests if os.path.isdir(os.path.join(args.baseline, t))] 89 90 # Build a baseline df (experiment baseline - say without kernel change) 91 # compare df (experiment with change) 92 # diff (difference in stats between compare and baseline) 93 94 baseline_df = build_stats_df(args.baseline) 95 compare_df = build_stats_df(args.compare_with) 96 diff = compare_df - baseline_df 97 98 diff.columns = [str(col) + '_diff' for col in diff.columns] 99 baseline_df.columns = [str(col) + '_baseline' for col in baseline_df.columns] 100 compare_df.columns = [str(col) + '_compare' for col in compare_df.columns] 101 102 final_df = pd.concat([compare_df, diff], axis=1) 103 final_df = final_df.reindex_axis(sorted(final_df.columns), axis=1) 104 105 # Print the results 106 print final_df 107
