1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 // Takes sharpness score, rates the image good if above 10, bad otherwise 17 18 package androidx.media.filterfw.samples.simplecamera; 19 20 import android.graphics.Bitmap; 21 import android.os.AsyncTask; 22 import android.util.Log; 23 import android.widget.ImageView; 24 import androidx.media.filterfw.Filter; 25 import androidx.media.filterfw.FrameImage2D; 26 import androidx.media.filterfw.FrameType; 27 import androidx.media.filterfw.FrameValue; 28 import androidx.media.filterfw.MffContext; 29 import androidx.media.filterfw.OutputPort; 30 import androidx.media.filterfw.Signature; 31 32 public class ImageGoodnessFilter extends Filter { 33 34 private static final String TAG = "ImageGoodnessFilter"; 35 private static boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 36 37 private final static String GREAT = "Great Picture!"; 38 private final static String GOOD = "Good Picture!"; 39 private final static String OK = "Ok Picture"; 40 private final static String BAD = "Bad Picture"; 41 private final static String AWFUL = "Awful Picture"; 42 private final static float SMALL_SCORE_INC = 0.25f; 43 private final static float BIG_SCORE_INC = 0.5f; 44 private final static float LOW_VARIANCE = 0.1f; 45 private final static float MEDIUM_VARIANCE = 10; 46 private final static float HIGH_VARIANCE = 100; 47 private float sharpnessMean = 0; 48 private float sharpnessVar = 0; 49 private float underExposureMean = 0; 50 private float underExposureVar = 0; 51 private float overExposureMean = 0; 52 private float overExposureVar = 0; 53 private float contrastMean = 0; 54 private float contrastVar = 0; 55 private float colorfulnessMean = 0; 56 private float colorfulnessVar = 0; 57 private float brightnessMean = 0; 58 private float brightnessVar = 0; 59 60 private float motionMean = 0; 61 private float scoreMean = 0; 62 private static final float DECAY = 0.03f; 63 /** 64 * @param context 65 * @param name 66 */ 67 public ImageGoodnessFilter(MffContext context, String name) { 68 super(context, name); 69 } 70 71 @Override 72 public Signature getSignature() { 73 FrameType floatT = FrameType.single(float.class); 74 FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU); 75 76 return new Signature() 77 .addInputPort("sharpness", Signature.PORT_REQUIRED, floatT) 78 .addInputPort("overExposure", Signature.PORT_REQUIRED, floatT) 79 .addInputPort("underExposure", Signature.PORT_REQUIRED, floatT) 80 .addInputPort("colorfulness", Signature.PORT_REQUIRED, floatT) 81 .addInputPort("contrastRating", Signature.PORT_REQUIRED, floatT) 82 .addInputPort("motionValues", Signature.PORT_REQUIRED, FrameType.array(float.class)) 83 .addInputPort("brightness", Signature.PORT_REQUIRED, floatT) 84 .addInputPort("capturing", Signature.PORT_REQUIRED, FrameType.single(boolean.class)) 85 .addInputPort("image", Signature.PORT_REQUIRED, imageIn) 86 .addOutputPort("goodOrBadPic", Signature.PORT_REQUIRED, 87 FrameType.single(String.class)) 88 .addOutputPort("score", Signature.PORT_OPTIONAL, floatT) 89 .disallowOtherPorts(); 90 } 91 92 /** 93 * @see androidx.media.filterfw.Filter#onProcess() 94 */ 95 @Override 96 protected void onProcess() { 97 FrameValue sharpnessFrameValue = 98 getConnectedInputPort("sharpness").pullFrame().asFrameValue(); 99 float sharpness = ((Float)sharpnessFrameValue.getValue()).floatValue(); 100 101 FrameValue overExposureFrameValue = 102 getConnectedInputPort("overExposure").pullFrame().asFrameValue(); 103 float overExposure = ((Float)overExposureFrameValue.getValue()).floatValue(); 104 105 FrameValue underExposureFrameValue = 106 getConnectedInputPort("underExposure").pullFrame().asFrameValue(); 107 float underExposure = ((Float)underExposureFrameValue.getValue()).floatValue(); 108 109 FrameValue colorfulnessFrameValue = 110 getConnectedInputPort("colorfulness").pullFrame().asFrameValue(); 111 float colorfulness = ((Float)colorfulnessFrameValue.getValue()).floatValue(); 112 113 FrameValue contrastRatingFrameValue = 114 getConnectedInputPort("contrastRating").pullFrame().asFrameValue(); 115 float contrastRating = ((Float)contrastRatingFrameValue.getValue()).floatValue(); 116 117 FrameValue brightnessFrameValue = 118 getConnectedInputPort("brightness").pullFrame().asFrameValue(); 119 float brightness = ((Float)brightnessFrameValue.getValue()).floatValue(); 120 121 FrameValue motionValuesFrameValue = 122 getConnectedInputPort("motionValues").pullFrame().asFrameValue(); 123 float[] motionValues = (float[]) motionValuesFrameValue.getValue(); 124 125 126 float vectorAccel = (float) Math.sqrt(Math.pow(motionValues[0], 2) + 127 Math.pow(motionValues[1], 2) + Math.pow(motionValues[2], 2)); 128 String outStr; 129 130 FrameValue capturingFrameValue = 131 getConnectedInputPort("capturing").pullFrame().asFrameValue(); 132 boolean capturing = (Boolean) capturingFrameValue.getValue(); 133 134 FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D(); 135 136 137 // TODO: get rid of magic numbers 138 float score = 0.0f; 139 score = computePictureScore(vectorAccel, sharpness, underExposure, overExposure, 140 contrastRating, colorfulness, brightness); 141 if (scoreMean == 0) scoreMean = score; 142 else scoreMean = scoreMean * (1 - DECAY) + score * DECAY; 143 144 if (motionMean == 0) motionMean = vectorAccel; 145 else motionMean = motionMean * (1 - DECAY) + vectorAccel * DECAY; 146 147 float classifierScore = classifierComputeScore(vectorAccel, sharpness, underExposure, 148 colorfulness, contrastRating, score); 149 150 // Log.v(TAG, "ClassifierScore:: " + classifierScore); 151 final float GREAT_SCORE = 3.5f; 152 final float GOOD_SCORE = 2.5f; 153 final float OK_SCORE = 1.5f; 154 final float BAD_SCORE = 0.5f; 155 156 if (score >= GREAT_SCORE) { 157 outStr = GREAT; 158 } else if (score >= GOOD_SCORE) { 159 outStr = GOOD; 160 } else if (score >= OK_SCORE) { 161 outStr = OK; 162 } else if (score >= BAD_SCORE) { 163 outStr = BAD; 164 } else { 165 outStr = AWFUL; 166 } 167 168 if(capturing) { 169 if (outStr.equals(GREAT)) { 170 // take a picture 171 Bitmap bitmap = inputImage.toBitmap(); 172 173 new AsyncOperation().execute(bitmap); 174 final float RESET_FEATURES = 0.01f; 175 sharpnessMean = RESET_FEATURES; 176 underExposureMean = RESET_FEATURES; 177 overExposureMean = RESET_FEATURES; 178 contrastMean = RESET_FEATURES; 179 colorfulnessMean = RESET_FEATURES; 180 brightnessMean = RESET_FEATURES; 181 } 182 } 183 184 OutputPort outPort = getConnectedOutputPort("goodOrBadPic"); 185 FrameValue stringFrame = outPort.fetchAvailableFrame(null).asFrameValue(); 186 stringFrame.setValue(outStr); 187 outPort.pushFrame(stringFrame); 188 189 OutputPort scoreOutPort = getConnectedOutputPort("score"); 190 FrameValue scoreFrame = scoreOutPort.fetchAvailableFrame(null).asFrameValue(); 191 scoreFrame.setValue(score); 192 scoreOutPort.pushFrame(scoreFrame); 193 194 } 195 196 private class AsyncOperation extends AsyncTask<Bitmap, Void, String> { 197 private Bitmap b; 198 protected void onPostExecute(String result) { 199 ImageView view = SmartCamera.getImageView(); 200 view.setImageBitmap(b); 201 } 202 203 @Override 204 protected String doInBackground(Bitmap... params) { 205 // TODO Auto-generated method stub 206 b = params[0]; 207 return null; 208 } 209 210 } 211 // Returns a number between -1 and 1 212 private float classifierComputeScore(float vectorAccel, float sharpness, float underExposure, 213 float colorfulness, float contrast, float score) { 214 float result = (-0.0223f * sharpness + -0.0563f * underExposure + 0.0137f * colorfulness 215 + 0.3102f * contrast + 0.0314f * vectorAccel + -0.0094f * score + 0.0227f * 216 sharpnessMean + 0.0459f * underExposureMean + -0.3934f * contrastMean + 217 -0.0697f * motionMean + 0.0091f * scoreMean + -0.0152f); 218 return result; 219 } 220 221 // Returns a number between -1 and 4 representing the score for this picture 222 private float computePictureScore(float vector_accel, float sharpness, 223 float underExposure, float overExposure, float contrastRating, float colorfulness, 224 float brightness) { 225 final float ACCELERATION_THRESHOLD_VERY_STEADY = 0.1f; 226 final float ACCELERATION_THRESHOLD_STEADY = 0.3f; 227 final float ACCELERATION_THRESHOLD_MOTION = 2f; 228 229 float score = 0.0f; 230 if (vector_accel > ACCELERATION_THRESHOLD_MOTION) { 231 score -= (BIG_SCORE_INC + BIG_SCORE_INC); // set score to -1, bad pic 232 } else if (vector_accel > ACCELERATION_THRESHOLD_STEADY) { 233 score -= BIG_SCORE_INC; 234 score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, 235 colorfulness, brightness, score); 236 } else if (vector_accel < ACCELERATION_THRESHOLD_VERY_STEADY) { 237 score += BIG_SCORE_INC; 238 score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, 239 colorfulness, brightness, score); 240 } else { 241 score = subComputeScore(sharpness, underExposure, overExposure, contrastRating, 242 colorfulness, brightness, score); 243 } 244 return score; 245 } 246 247 // Changes the score by at most +/- 3.5 248 private float subComputeScore(float sharpness, float underExposure, float overExposure, 249 float contrastRating, float colorfulness, float brightness, float score) { 250 // The score methods return values -0.5 to 0.5 251 final float SHARPNESS_WEIGHT = 2; 252 score += SHARPNESS_WEIGHT * sharpnessScore(sharpness); 253 score += underExposureScore(underExposure); 254 score += overExposureScore(overExposure); 255 score += contrastScore(contrastRating); 256 score += colorfulnessScore(colorfulness); 257 score += brightnessScore(brightness); 258 return score; 259 } 260 261 private float sharpnessScore(float sharpness) { 262 if (sharpnessMean == 0) { 263 sharpnessMean = sharpness; 264 sharpnessVar = 0; 265 return 0; 266 } else { 267 sharpnessMean = sharpnessMean * (1 - DECAY) + sharpness * DECAY; 268 sharpnessVar = sharpnessVar * (1 - DECAY) + (sharpness - sharpnessMean) * 269 (sharpness - sharpnessMean) * DECAY; 270 if (sharpnessVar < LOW_VARIANCE) { 271 return BIG_SCORE_INC; 272 } else if (sharpness < sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) { 273 return -BIG_SCORE_INC; 274 } else if (sharpness < sharpnessMean) { 275 return -SMALL_SCORE_INC; 276 } else if (sharpness > sharpnessMean && sharpnessVar > HIGH_VARIANCE) { 277 return 0; 278 } else if (sharpness > sharpnessMean && sharpnessVar > MEDIUM_VARIANCE) { 279 return SMALL_SCORE_INC; 280 } else { 281 return BIG_SCORE_INC; // low variance, sharpness above the mean 282 } 283 } 284 } 285 286 private float underExposureScore(float underExposure) { 287 if (underExposureMean == 0) { 288 underExposureMean = underExposure; 289 underExposureVar = 0; 290 return 0; 291 } else { 292 underExposureMean = underExposureMean * (1 - DECAY) + underExposure * DECAY; 293 underExposureVar = underExposureVar * (1 - DECAY) + (underExposure - underExposureMean) 294 * (underExposure - underExposureMean) * DECAY; 295 if (underExposureVar < LOW_VARIANCE) { 296 return BIG_SCORE_INC; 297 } else if (underExposure > underExposureMean && underExposureVar > MEDIUM_VARIANCE) { 298 return -BIG_SCORE_INC; 299 } else if (underExposure > underExposureMean) { 300 return -SMALL_SCORE_INC; 301 } else if (underExposure < underExposureMean && underExposureVar > HIGH_VARIANCE) { 302 return 0; 303 } else if (underExposure < underExposureMean && underExposureVar > MEDIUM_VARIANCE) { 304 return SMALL_SCORE_INC; 305 } else { 306 return BIG_SCORE_INC; // low variance, underExposure below the mean 307 } 308 } 309 } 310 311 private float overExposureScore(float overExposure) { 312 if (overExposureMean == 0) { 313 overExposureMean = overExposure; 314 overExposureVar = 0; 315 return 0; 316 } else { 317 overExposureMean = overExposureMean * (1 - DECAY) + overExposure * DECAY; 318 overExposureVar = overExposureVar * (1 - DECAY) + (overExposure - overExposureMean) * 319 (overExposure - overExposureMean) * DECAY; 320 if (overExposureVar < LOW_VARIANCE) { 321 return BIG_SCORE_INC; 322 } else if (overExposure > overExposureMean && overExposureVar > MEDIUM_VARIANCE) { 323 return -BIG_SCORE_INC; 324 } else if (overExposure > overExposureMean) { 325 return -SMALL_SCORE_INC; 326 } else if (overExposure < overExposureMean && overExposureVar > HIGH_VARIANCE) { 327 return 0; 328 } else if (overExposure < overExposureMean && overExposureVar > MEDIUM_VARIANCE) { 329 return SMALL_SCORE_INC; 330 } else { 331 return BIG_SCORE_INC; // low variance, overExposure below the mean 332 } 333 } 334 } 335 336 private float contrastScore(float contrast) { 337 if (contrastMean == 0) { 338 contrastMean = contrast; 339 contrastVar = 0; 340 return 0; 341 } else { 342 contrastMean = contrastMean * (1 - DECAY) + contrast * DECAY; 343 contrastVar = contrastVar * (1 - DECAY) + (contrast - contrastMean) * 344 (contrast - contrastMean) * DECAY; 345 if (contrastVar < LOW_VARIANCE) { 346 return BIG_SCORE_INC; 347 } else if (contrast < contrastMean && contrastVar > MEDIUM_VARIANCE) { 348 return -BIG_SCORE_INC; 349 } else if (contrast < contrastMean) { 350 return -SMALL_SCORE_INC; 351 } else if (contrast > contrastMean && contrastVar > 100) { 352 return 0; 353 } else if (contrast > contrastMean && contrastVar > MEDIUM_VARIANCE) { 354 return SMALL_SCORE_INC; 355 } else { 356 return BIG_SCORE_INC; // low variance, contrast above the mean 357 } 358 } 359 } 360 361 private float colorfulnessScore(float colorfulness) { 362 if (colorfulnessMean == 0) { 363 colorfulnessMean = colorfulness; 364 colorfulnessVar = 0; 365 return 0; 366 } else { 367 colorfulnessMean = colorfulnessMean * (1 - DECAY) + colorfulness * DECAY; 368 colorfulnessVar = colorfulnessVar * (1 - DECAY) + (colorfulness - colorfulnessMean) * 369 (colorfulness - colorfulnessMean) * DECAY; 370 if (colorfulnessVar < LOW_VARIANCE) { 371 return BIG_SCORE_INC; 372 } else if (colorfulness < colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) { 373 return -BIG_SCORE_INC; 374 } else if (colorfulness < colorfulnessMean) { 375 return -SMALL_SCORE_INC; 376 } else if (colorfulness > colorfulnessMean && colorfulnessVar > 100) { 377 return 0; 378 } else if (colorfulness > colorfulnessMean && colorfulnessVar > MEDIUM_VARIANCE) { 379 return SMALL_SCORE_INC; 380 } else { 381 return BIG_SCORE_INC; // low variance, colorfulness above the mean 382 } 383 } 384 } 385 386 private float brightnessScore(float brightness) { 387 if (brightnessMean == 0) { 388 brightnessMean = brightness; 389 brightnessVar = 0; 390 return 0; 391 } else { 392 brightnessMean = brightnessMean * (1 - DECAY) + brightness * DECAY; 393 brightnessVar = brightnessVar * (1 - DECAY) + (brightness - brightnessMean) * 394 (brightness - brightnessMean) * DECAY; 395 if (brightnessVar < LOW_VARIANCE) { 396 return BIG_SCORE_INC; 397 } else if (brightness < brightnessMean && brightnessVar > MEDIUM_VARIANCE) { 398 return -BIG_SCORE_INC; 399 } else if (brightness < brightnessMean) { 400 return -SMALL_SCORE_INC; 401 } else if (brightness > brightnessMean && brightnessVar > 100) { 402 return 0; 403 } else if (brightness > brightnessMean && brightnessVar > MEDIUM_VARIANCE) { 404 return SMALL_SCORE_INC; 405 } else { 406 return BIG_SCORE_INC; // low variance, brightness above the mean 407 } 408 } 409 } 410 } 411
