Home | History | Annotate | Download | only in hwuicompare 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.test.hwuicompare;
     18 
     19 import java.io.File;
     20 import java.io.FileInputStream;
     21 import java.io.FileOutputStream;
     22 import java.io.IOException;
     23 import java.util.ArrayList;
     24 import java.util.Comparator;
     25 import java.util.HashMap;
     26 import java.util.TreeSet;
     27 
     28 import org.json.JSONException;
     29 import org.json.JSONObject;
     30 
     31 import android.os.Bundle;
     32 import android.os.Environment;
     33 import android.os.Trace;
     34 import android.util.Log;
     35 import android.widget.ImageView;
     36 import android.widget.Toast;
     37 
     38 public class AutomaticActivity extends CompareActivity {
     39     private static final String LOG_TAG = "AutomaticActivity";
     40     private static final float ERROR_DISPLAY_THRESHOLD = 0.01f;
     41     protected static final boolean DRAW_BITMAPS = false;
     42 
     43     /**
     44      * Threshold of error change required to consider a test regressed/improved
     45      */
     46     private static final float ERROR_CHANGE_THRESHOLD = 0.001f;
     47 
     48     private static final float[] ERROR_CUTOFFS = {
     49             0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f
     50     };
     51 
     52     private final float[] mErrorRates = new float[ERROR_CUTOFFS.length];
     53     private float mTotalTests = 0;
     54     private float mTotalError = 0;
     55     private int mTestsRegressed = 0;
     56     private int mTestsImproved = 0;
     57 
     58     private ImageView mSoftwareImageView = null;
     59     private ImageView mHardwareImageView = null;
     60 
     61 
     62     public abstract static class FinalCallback {
     63         abstract void report(String name, float value);
     64         void complete() {};
     65     }
     66 
     67     private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>();
     68 
     69     Runnable mRunnable = new Runnable() {
     70         @Override
     71         public void run() {
     72             loadBitmaps();
     73             if (mSoftwareBitmap == null || mHardwareBitmap == null) {
     74                 Log.e(LOG_TAG, "bitmap is null");
     75                 return;
     76             }
     77 
     78             if (DRAW_BITMAPS) {
     79                 mSoftwareImageView.setImageBitmap(mSoftwareBitmap);
     80                 mHardwareImageView.setImageBitmap(mHardwareBitmap);
     81             }
     82 
     83             Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError");
     84             float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap);
     85             Trace.traceEnd(Trace.TRACE_TAG_ALWAYS);
     86 
     87             final String[] modifierNames = DisplayModifier.getLastAppliedModifications();
     88             handleError(modifierNames, error);
     89 
     90             if (DisplayModifier.step()) {
     91                 finishTest();
     92             } else {
     93                 mHardwareView.invalidate();
     94                 if (DRAW_BITMAPS) {
     95                     mSoftwareImageView.invalidate();
     96                     mHardwareImageView.invalidate();
     97                 }
     98             }
     99             mHandler.removeCallbacks(mRunnable);
    100         }
    101     };
    102 
    103     @Override
    104     protected void onPause() {
    105         super.onPause();
    106         mHandler.removeCallbacks(mRunnable);
    107     };
    108 
    109     @Override
    110     protected void onCreate(Bundle savedInstanceState) {
    111         super.onCreate(savedInstanceState);
    112         setContentView(R.layout.automatic_layout);
    113 
    114         mSoftwareImageView = (ImageView) findViewById(R.id.software_image_view);
    115         mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view);
    116 
    117         onCreateCommon(mRunnable);
    118         beginTest();
    119     }
    120 
    121     private static class TestResult {
    122         TestResult(String label, float error) {
    123             mLabel = label;
    124             mTotalError = error;
    125             mCount = 1;
    126         }
    127         public void addInto(float error) {
    128             mTotalError += error;
    129             mCount++;
    130         }
    131         public float getAverage() {
    132             return mTotalError / mCount;
    133         }
    134         final String mLabel;
    135         float mTotalError;
    136         int mCount;
    137     }
    138 
    139     JSONObject mOutputJson = null;
    140     JSONObject mInputJson = null;
    141     final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>();
    142     final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>();
    143     final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>();
    144     final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>();
    145     private void beginTest() {
    146         mFinalCallbacks.add(new FinalCallback() {
    147             @Override
    148             void report(String name, float value) {
    149                 Log.d(LOG_TAG, name + " " + value);
    150             };
    151         });
    152 
    153         File inputFile = new File(Environment.getExternalStorageDirectory(),
    154                 "CanvasCompareInput.json");
    155         if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) {
    156             try {
    157                 FileInputStream inputStream = new FileInputStream(inputFile);
    158                 Log.d(LOG_TAG, "Parsing input file...");
    159                 StringBuffer content = new StringBuffer((int)inputFile.length());
    160                 byte[] buffer = new byte[1024];
    161                 while (inputStream.read(buffer) != -1) {
    162                     content.append(new String(buffer));
    163                 }
    164                 mInputJson = new JSONObject(content.toString());
    165                 inputStream.close();
    166                 Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries");
    167             } catch (JSONException e) {
    168                 Log.e(LOG_TAG, "error parsing input json", e);
    169             } catch (IOException e) {
    170                 Log.e(LOG_TAG, "error reading input json from sd", e);
    171             }
    172         }
    173 
    174         mOutputJson = new JSONObject();
    175     }
    176 
    177     private static void logTestResultHash(String label, HashMap<String, TestResult> map) {
    178         Log.d(LOG_TAG, "---------------");
    179         Log.d(LOG_TAG, label + ":");
    180         Log.d(LOG_TAG, "---------------");
    181         TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() {
    182             @Override
    183             public int compare(TestResult lhs, TestResult rhs) {
    184                 if (lhs == rhs) return 0; // don't need to worry about complex equality
    185 
    186                 int cmp = Float.compare(lhs.getAverage(), rhs.getAverage());
    187                 if (cmp != 0) {
    188                     return cmp;
    189                 }
    190                 return lhs.mLabel.compareTo(rhs.mLabel);
    191             }
    192         });
    193 
    194         for (TestResult t : map.values()) {
    195             set.add(t);
    196         }
    197 
    198         for (TestResult t : set.descendingSet()) {
    199             if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) {
    200                 Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel));
    201             }
    202         }
    203         Log.d(LOG_TAG, "");
    204     }
    205 
    206     private void finishTest() {
    207         for (FinalCallback c : mFinalCallbacks) {
    208             c.report("averageError", (mTotalError / mTotalTests));
    209             for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
    210                 c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]),
    211                         mErrorRates[i]);
    212             }
    213             if (mInputJson != null) {
    214                 c.report("tests regressed", mTestsRegressed);
    215                 c.report("tests improved", mTestsImproved);
    216             }
    217             c.complete();
    218         }
    219 
    220         try {
    221             if (mOutputJson != null) {
    222                 String outputString = mOutputJson.toString(4);
    223                 File outputFile = new File(Environment.getExternalStorageDirectory(),
    224                         "CanvasCompareOutput.json");
    225                 FileOutputStream outputStream = new FileOutputStream(outputFile);
    226                 outputStream.write(outputString.getBytes());
    227                 outputStream.close();
    228                 Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries");
    229             }
    230         } catch (JSONException e) {
    231             Log.e(LOG_TAG, "error during JSON stringify", e);
    232         } catch (IOException e) {
    233             Log.e(LOG_TAG, "error storing JSON output on sd", e);
    234         }
    235 
    236         logTestResultHash("Modifier change vs previous", mModifierDiffResults);
    237         logTestResultHash("Invidual test change vs previous", mIndividualDiffResults);
    238         logTestResultHash("Modifier average test results", mModifierResults);
    239         logTestResultHash("Individual test results", mIndividualResults);
    240 
    241         Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
    242         finish();
    243     }
    244 
    245     /**
    246      * Inserts the error value into all TestResult objects, associated with each of its modifiers
    247      */
    248     private static void addForAllModifiers(String fullName, float error, String[] modifierNames,
    249             HashMap<String, TestResult> modifierResults) {
    250         for (String modifierName : modifierNames) {
    251             TestResult r = modifierResults.get(fullName);
    252             if (r == null) {
    253                 modifierResults.put(modifierName, new TestResult(modifierName, error));
    254             } else {
    255                 r.addInto(error);
    256             }
    257         }
    258     }
    259 
    260     private void handleError(final String[] modifierNames, final float error) {
    261         String fullName = "";
    262         for (String s : modifierNames) {
    263             fullName = fullName.concat("." + s);
    264         }
    265         fullName = fullName.substring(1);
    266 
    267         float deltaError = 0;
    268         if (mInputJson != null) {
    269             try {
    270                 deltaError = error - (float)mInputJson.getDouble(fullName);
    271             } catch (JSONException e) {
    272                 Log.w(LOG_TAG, "Warning: unable to read from input json", e);
    273             }
    274             if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++;
    275             if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++;
    276             mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError));
    277             addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults);
    278         }
    279 
    280         mIndividualResults.put(fullName, new TestResult(fullName, error));
    281         addForAllModifiers(fullName, error, modifierNames, mModifierResults);
    282 
    283         try {
    284             if (mOutputJson != null) {
    285                 mOutputJson.put(fullName, error);
    286             }
    287         } catch (JSONException e) {
    288             Log.e(LOG_TAG, "exception during JSON recording", e);
    289             mOutputJson = null;
    290         }
    291 
    292         for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
    293             if (error <= ERROR_CUTOFFS[i]) break;
    294             mErrorRates[i]++;
    295         }
    296         mTotalError += error;
    297         mTotalTests++;
    298     }
    299 
    300     @Override
    301     protected boolean forceRecreateBitmaps() {
    302         // disable, unless needed for drawing into imageviews
    303         return DRAW_BITMAPS;
    304     }
    305 
    306     // FOR TESTING
    307     public void setFinalCallback(FinalCallback c) {
    308         mFinalCallbacks.add(c);
    309     }
    310 }
    311