1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the License); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an AS IS BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.benchmark.results; 18 19 import android.content.ContentValues; 20 import android.content.Context; 21 import android.database.Cursor; 22 import android.database.sqlite.SQLiteDatabase; 23 import android.database.sqlite.SQLiteOpenHelper; 24 import android.view.FrameMetrics; 25 import android.widget.Toast; 26 27 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; 28 29 import java.io.FileWriter; 30 import java.io.IOException; 31 import java.text.DateFormat; 32 import java.util.ArrayList; 33 import java.util.Date; 34 import java.util.HashMap; 35 36 public class GlobalResultsStore extends SQLiteOpenHelper { 37 private static final int VERSION = 2; 38 39 private static GlobalResultsStore sInstance; 40 private static final String UI_RESULTS_TABLE = "ui_results"; 41 42 private final Context mContext; 43 44 private GlobalResultsStore(Context context) { 45 super(context, "BenchmarkResults", null, VERSION); 46 mContext = context; 47 } 48 49 public static GlobalResultsStore getInstance(Context context) { 50 if (sInstance == null) { 51 sInstance = new GlobalResultsStore(context.getApplicationContext()); 52 } 53 54 return sInstance; 55 } 56 57 @Override 58 public void onCreate(SQLiteDatabase sqLiteDatabase) { 59 sqLiteDatabase.execSQL("CREATE TABLE " + UI_RESULTS_TABLE + " (" + 60 " _id INTEGER PRIMARY KEY AUTOINCREMENT," + 61 " name TEXT," + 62 " run_id INTEGER," + 63 " iteration INTEGER," + 64 " timestamp TEXT," + 65 " unknown_delay REAL," + 66 " input REAL," + 67 " animation REAL," + 68 " layout REAL," + 69 " draw REAL," + 70 " sync REAL," + 71 " command_issue REAL," + 72 " swap_buffers REAL," + 73 " total_duration REAL," + 74 " jank_frame BOOLEAN, " + 75 " device_charging INTEGER);"); 76 } 77 78 public void storeRunResults(String testName, int runId, int iteration, 79 UiBenchmarkResult result) { 80 SQLiteDatabase db = getWritableDatabase(); 81 db.beginTransaction(); 82 83 try { 84 String date = DateFormat.getDateTimeInstance().format(new Date()); 85 int jankIndexIndex = 0; 86 int[] sortedJankIndices = result.getSortedJankFrameIndices(); 87 int totalFrameCount = result.getTotalFrameCount(); 88 for (int frameIdx = 0; frameIdx < totalFrameCount; frameIdx++) { 89 ContentValues cv = new ContentValues(); 90 cv.put("name", testName); 91 cv.put("run_id", runId); 92 cv.put("iteration", iteration); 93 cv.put("timestamp", date); 94 cv.put("unknown_delay", 95 result.getMetricAtIndex(frameIdx, FrameMetrics.UNKNOWN_DELAY_DURATION)); 96 cv.put("input", 97 result.getMetricAtIndex(frameIdx, FrameMetrics.INPUT_HANDLING_DURATION)); 98 cv.put("animation", 99 result.getMetricAtIndex(frameIdx, FrameMetrics.ANIMATION_DURATION)); 100 cv.put("layout", 101 result.getMetricAtIndex(frameIdx, FrameMetrics.LAYOUT_MEASURE_DURATION)); 102 cv.put("draw", 103 result.getMetricAtIndex(frameIdx, FrameMetrics.DRAW_DURATION)); 104 cv.put("sync", 105 result.getMetricAtIndex(frameIdx, FrameMetrics.SYNC_DURATION)); 106 cv.put("command_issue", 107 result.getMetricAtIndex(frameIdx, FrameMetrics.COMMAND_ISSUE_DURATION)); 108 cv.put("swap_buffers", 109 result.getMetricAtIndex(frameIdx, FrameMetrics.SWAP_BUFFERS_DURATION)); 110 cv.put("total_duration", 111 result.getMetricAtIndex(frameIdx, FrameMetrics.TOTAL_DURATION)); 112 if (jankIndexIndex < sortedJankIndices.length && 113 sortedJankIndices[jankIndexIndex] == frameIdx) { 114 jankIndexIndex++; 115 cv.put("jank_frame", true); 116 } else { 117 cv.put("jank_frame", false); 118 } 119 db.insert(UI_RESULTS_TABLE, null, cv); 120 } 121 db.setTransactionSuccessful(); 122 Toast.makeText(mContext, "Score: " + result.getScore() 123 + " Jank: " + (100 * sortedJankIndices.length) / (float) totalFrameCount + "%", 124 Toast.LENGTH_LONG).show(); 125 } finally { 126 db.endTransaction(); 127 } 128 129 } 130 131 public ArrayList<UiBenchmarkResult> loadTestResults(String testName, int runId) { 132 SQLiteDatabase db = getReadableDatabase(); 133 ArrayList<UiBenchmarkResult> resultList = new ArrayList<>(); 134 try { 135 String[] columnsToQuery = new String[] { 136 "name", 137 "run_id", 138 "iteration", 139 "unknown_delay", 140 "input", 141 "animation", 142 "layout", 143 "draw", 144 "sync", 145 "command_issue", 146 "swap_buffers", 147 "total_duration", 148 }; 149 150 Cursor cursor = db.query( 151 UI_RESULTS_TABLE, columnsToQuery, "run_id=? AND name=?", 152 new String[] { Integer.toString(runId), testName }, null, null, "iteration"); 153 154 double[] values = new double[columnsToQuery.length - 3]; 155 156 while (cursor.moveToNext()) { 157 int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration")); 158 159 values[0] = cursor.getDouble( 160 cursor.getColumnIndexOrThrow("unknown_delay")); 161 values[1] = cursor.getDouble( 162 cursor.getColumnIndexOrThrow("input")); 163 values[2] = cursor.getDouble( 164 cursor.getColumnIndexOrThrow("animation")); 165 values[3] = cursor.getDouble( 166 cursor.getColumnIndexOrThrow("layout")); 167 values[4] = cursor.getDouble( 168 cursor.getColumnIndexOrThrow("draw")); 169 values[5] = cursor.getDouble( 170 cursor.getColumnIndexOrThrow("sync")); 171 values[6] = cursor.getDouble( 172 cursor.getColumnIndexOrThrow("command_issue")); 173 values[7] = cursor.getDouble( 174 cursor.getColumnIndexOrThrow("swap_buffers")); 175 values[8] = cursor.getDouble( 176 cursor.getColumnIndexOrThrow("total_duration")); 177 178 UiBenchmarkResult iterationResult; 179 if (resultList.size() == iteration) { 180 iterationResult = new UiBenchmarkResult(values); 181 resultList.add(iteration, iterationResult); 182 } else { 183 iterationResult = resultList.get(iteration); 184 iterationResult.update(values); 185 } 186 } 187 188 cursor.close(); 189 } finally { 190 db.close(); 191 } 192 193 int total = resultList.get(0).getTotalFrameCount(); 194 for (int i = 0; i < total; i++) { 195 System.out.println(""+ resultList.get(0).getMetricAtIndex(0, FrameMetrics.TOTAL_DURATION)); 196 } 197 198 return resultList; 199 } 200 201 public HashMap<String, ArrayList<UiBenchmarkResult>> loadDetailedResults(int runId) { 202 SQLiteDatabase db = getReadableDatabase(); 203 HashMap<String, ArrayList<UiBenchmarkResult>> results = new HashMap<>(); 204 try { 205 String[] columnsToQuery = new String[] { 206 "name", 207 "run_id", 208 "iteration", 209 "unknown_delay", 210 "input", 211 "animation", 212 "layout", 213 "draw", 214 "sync", 215 "command_issue", 216 "swap_buffers", 217 "total_duration", 218 }; 219 220 Cursor cursor = db.query( 221 UI_RESULTS_TABLE, columnsToQuery, "run_id=?", 222 new String[] { Integer.toString(runId) }, null, null, "name, iteration"); 223 224 double[] values = new double[columnsToQuery.length - 3]; 225 while (cursor.moveToNext()) { 226 int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration")); 227 String name = cursor.getString(cursor.getColumnIndexOrThrow("name")); 228 ArrayList<UiBenchmarkResult> resultList = results.get(name); 229 if (resultList == null) { 230 resultList = new ArrayList<>(); 231 results.put(name, resultList); 232 } 233 234 values[0] = cursor.getDouble( 235 cursor.getColumnIndexOrThrow("unknown_delay")); 236 values[1] = cursor.getDouble( 237 cursor.getColumnIndexOrThrow("input")); 238 values[2] = cursor.getDouble( 239 cursor.getColumnIndexOrThrow("animation")); 240 values[3] = cursor.getDouble( 241 cursor.getColumnIndexOrThrow("layout")); 242 values[4] = cursor.getDouble( 243 cursor.getColumnIndexOrThrow("draw")); 244 values[5] = cursor.getDouble( 245 cursor.getColumnIndexOrThrow("sync")); 246 values[6] = cursor.getDouble( 247 cursor.getColumnIndexOrThrow("command_issue")); 248 values[7] = cursor.getDouble( 249 cursor.getColumnIndexOrThrow("swap_buffers")); 250 values[8] = cursor.getDouble( 251 cursor.getColumnIndexOrThrow("total_duration")); 252 values[8] = cursor.getDouble( 253 cursor.getColumnIndexOrThrow("total_duration")); 254 255 UiBenchmarkResult iterationResult; 256 if (resultList.size() == iteration) { 257 iterationResult = new UiBenchmarkResult(values); 258 resultList.add(iterationResult); 259 } else { 260 iterationResult = resultList.get(iteration); 261 iterationResult.update(values); 262 } 263 } 264 265 cursor.close(); 266 } finally { 267 db.close(); 268 } 269 270 return results; 271 } 272 273 public void exportToCsv() throws IOException { 274 String path = mContext.getFilesDir() + "/results-" + System.currentTimeMillis() + ".csv"; 275 SQLiteDatabase db = getReadableDatabase(); 276 277 // stats across metrics for each run and each test 278 HashMap<String, DescriptiveStatistics> stats = new HashMap<>(); 279 280 Cursor runIdCursor = db.query( 281 UI_RESULTS_TABLE, new String[] { "run_id" }, null, null, "run_id", null, null); 282 283 while (runIdCursor.moveToNext()) { 284 285 int runId = runIdCursor.getInt(runIdCursor.getColumnIndexOrThrow("run_id")); 286 HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults = 287 loadDetailedResults(runId); 288 289 writeRawResults(runId, detailedResults); 290 291 DescriptiveStatistics overall = new DescriptiveStatistics(); 292 try (FileWriter writer = new FileWriter(path, true)) { 293 writer.write("Run ID, " + runId + "\n"); 294 writer.write("Test, Iteration, Score, Jank Penalty, Consistency Bonus, 95th, " + 295 "90th\n"); 296 for (String testName : detailedResults.keySet()) { 297 ArrayList<UiBenchmarkResult> results = detailedResults.get(testName); 298 DescriptiveStatistics scoreStats = new DescriptiveStatistics(); 299 DescriptiveStatistics jankPenalty = new DescriptiveStatistics(); 300 DescriptiveStatistics consistencyBonus = new DescriptiveStatistics(); 301 for (int i = 0; i < results.size(); i++) { 302 UiBenchmarkResult result = results.get(i); 303 int score = result.getScore(); 304 scoreStats.addValue(score); 305 overall.addValue(score); 306 jankPenalty.addValue(result.getJankPenalty()); 307 consistencyBonus.addValue(result.getConsistencyBonus()); 308 309 writer.write(testName); 310 writer.write(","); 311 writer.write("" + i); 312 writer.write(","); 313 writer.write("" + score); 314 writer.write(","); 315 writer.write("" + result.getJankPenalty()); 316 writer.write(","); 317 writer.write("" + result.getConsistencyBonus()); 318 writer.write(","); 319 writer.write(Double.toString( 320 result.getPercentile(FrameMetrics.TOTAL_DURATION, 95))); 321 writer.write(","); 322 writer.write(Double.toString( 323 result.getPercentile(FrameMetrics.TOTAL_DURATION, 90))); 324 writer.write("\n"); 325 } 326 327 writer.write("Score CV," + 328 (100 * scoreStats.getStandardDeviation() 329 / scoreStats.getMean()) + "%\n"); 330 writer.write("Jank Penalty CV, " + 331 (100 * jankPenalty.getStandardDeviation() 332 / jankPenalty.getMean()) + "%\n"); 333 writer.write("Consistency Bonus CV, " + 334 (100 * consistencyBonus.getStandardDeviation() 335 / consistencyBonus.getMean()) + "%\n"); 336 writer.write("\n"); 337 } 338 339 writer.write("Overall Score CV," + 340 (100 * overall.getStandardDeviation() / overall.getMean()) + "%\n"); 341 writer.flush(); 342 } 343 } 344 345 runIdCursor.close(); 346 } 347 348 private void writeRawResults(int runId, 349 HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults) { 350 StringBuilder path = new StringBuilder(); 351 path.append(mContext.getFilesDir()); 352 path.append("/"); 353 path.append(Integer.toString(runId)); 354 path.append(".csv"); 355 try (FileWriter writer = new FileWriter(path.toString())) { 356 for (String test : detailedResults.keySet()) { 357 writer.write("Test, " + test + "\n"); 358 writer.write("iteration, unknown delay, input, animation, layout, draw, sync, " + 359 "command issue, swap buffers\n"); 360 ArrayList<UiBenchmarkResult> runs = detailedResults.get(test); 361 for (int i = 0; i < runs.size(); i++) { 362 UiBenchmarkResult run = runs.get(i); 363 for (int j = 0; j < run.getTotalFrameCount(); j++) { 364 writer.write(Integer.toString(i) + "," + 365 run.getMetricAtIndex(j, FrameMetrics.UNKNOWN_DELAY_DURATION) + "," + 366 run.getMetricAtIndex(j, FrameMetrics.INPUT_HANDLING_DURATION) + "," + 367 run.getMetricAtIndex(j, FrameMetrics.ANIMATION_DURATION) + "," + 368 run.getMetricAtIndex(j, FrameMetrics.LAYOUT_MEASURE_DURATION) + "," + 369 run.getMetricAtIndex(j, FrameMetrics.DRAW_DURATION) + "," + 370 run.getMetricAtIndex(j, FrameMetrics.SYNC_DURATION) + "," + 371 run.getMetricAtIndex(j, FrameMetrics.COMMAND_ISSUE_DURATION) + "," + 372 run.getMetricAtIndex(j, FrameMetrics.SWAP_BUFFERS_DURATION) + "," + 373 run.getMetricAtIndex(j, FrameMetrics.TOTAL_DURATION) + "\n"); 374 } 375 } 376 } 377 } catch (IOException e) { 378 e.printStackTrace(); 379 } 380 } 381 382 @Override 383 public void onUpgrade(SQLiteDatabase sqLiteDatabase, int oldVersion, int currentVersion) { 384 if (oldVersion < VERSION) { 385 sqLiteDatabase.execSQL("ALTER TABLE " 386 + UI_RESULTS_TABLE + " ADD COLUMN timestamp TEXT;"); 387 } 388 } 389 } 390
