1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.filterpacks.videoproc; 18 19 import android.filterfw.core.Filter; 20 import android.filterfw.core.FilterContext; 21 import android.filterfw.core.GenerateFieldPort; 22 import android.filterfw.core.GenerateFinalPort; 23 import android.filterfw.core.Frame; 24 import android.filterfw.core.GLFrame; 25 import android.filterfw.core.FrameFormat; 26 import android.filterfw.core.MutableFrameFormat; 27 import android.filterfw.core.Program; 28 import android.filterfw.core.ShaderProgram; 29 import android.filterfw.format.ImageFormat; 30 import android.opengl.GLES20; 31 import android.os.SystemClock; 32 import android.util.Log; 33 34 import java.lang.ArrayIndexOutOfBoundsException; 35 import java.lang.Math; 36 import java.util.Arrays; 37 import java.nio.ByteBuffer; 38 39 /** 40 * @hide 41 */ 42 public class BackDropperFilter extends Filter { 43 /** User-visible parameters */ 44 45 private final int BACKGROUND_STRETCH = 0; 46 private final int BACKGROUND_FIT = 1; 47 private final int BACKGROUND_FILL_CROP = 2; 48 49 @GenerateFieldPort(name = "backgroundFitMode", hasDefault = true) 50 private int mBackgroundFitMode = BACKGROUND_FILL_CROP; 51 @GenerateFieldPort(name = "learningDuration", hasDefault = true) 52 private int mLearningDuration = DEFAULT_LEARNING_DURATION; 53 @GenerateFieldPort(name = "learningVerifyDuration", hasDefault = true) 54 private int mLearningVerifyDuration = DEFAULT_LEARNING_VERIFY_DURATION; 55 @GenerateFieldPort(name = "acceptStddev", hasDefault = true) 56 private float mAcceptStddev = DEFAULT_ACCEPT_STDDEV; 57 @GenerateFieldPort(name = "hierLrgScale", hasDefault = true) 58 private float mHierarchyLrgScale = DEFAULT_HIER_LRG_SCALE; 59 @GenerateFieldPort(name = "hierMidScale", hasDefault = true) 60 private float mHierarchyMidScale = DEFAULT_HIER_MID_SCALE; 61 @GenerateFieldPort(name = "hierSmlScale", hasDefault = true) 62 private float mHierarchySmlScale = DEFAULT_HIER_SML_SCALE; 63 64 // Dimensions of foreground / background mask. Optimum value should take into account only 65 // image contents, NOT dimensions of input video stream. 66 @GenerateFieldPort(name = "maskWidthExp", hasDefault = true) 67 private int mMaskWidthExp = DEFAULT_MASK_WIDTH_EXPONENT; 68 @GenerateFieldPort(name = "maskHeightExp", hasDefault = true) 69 private int mMaskHeightExp = DEFAULT_MASK_HEIGHT_EXPONENT; 70 71 // Levels at which to compute foreground / background decision. Think of them as are deltas 72 // SUBTRACTED from maskWidthExp and maskHeightExp. 73 @GenerateFieldPort(name = "hierLrgExp", hasDefault = true) 74 private int mHierarchyLrgExp = DEFAULT_HIER_LRG_EXPONENT; 75 @GenerateFieldPort(name = "hierMidExp", hasDefault = true) 76 private int mHierarchyMidExp = DEFAULT_HIER_MID_EXPONENT; 77 @GenerateFieldPort(name = "hierSmlExp", hasDefault = true) 78 private int mHierarchySmlExp = DEFAULT_HIER_SML_EXPONENT; 79 80 @GenerateFieldPort(name = "lumScale", hasDefault = true) 81 private float mLumScale = DEFAULT_Y_SCALE_FACTOR; 82 @GenerateFieldPort(name = "chromaScale", hasDefault = true) 83 private float mChromaScale = DEFAULT_UV_SCALE_FACTOR; 84 @GenerateFieldPort(name = "maskBg", hasDefault = true) 85 private float mMaskBg = DEFAULT_MASK_BLEND_BG; 86 @GenerateFieldPort(name = "maskFg", hasDefault = true) 87 private float mMaskFg = DEFAULT_MASK_BLEND_FG; 88 @GenerateFieldPort(name = "exposureChange", hasDefault = true) 89 private float mExposureChange = DEFAULT_EXPOSURE_CHANGE; 90 @GenerateFieldPort(name = "whitebalanceredChange", hasDefault = true) 91 private float mWhiteBalanceRedChange = DEFAULT_WHITE_BALANCE_RED_CHANGE; 92 @GenerateFieldPort(name = "whitebalanceblueChange", hasDefault = true) 93 private float mWhiteBalanceBlueChange = DEFAULT_WHITE_BALANCE_BLUE_CHANGE; 94 @GenerateFieldPort(name = "autowbToggle", hasDefault = true) 95 private int mAutoWBToggle = DEFAULT_WHITE_BALANCE_TOGGLE; 96 97 // TODO: These are not updatable: 98 @GenerateFieldPort(name = "learningAdaptRate", hasDefault = true) 99 private float mAdaptRateLearning = DEFAULT_LEARNING_ADAPT_RATE; 100 @GenerateFieldPort(name = "adaptRateBg", hasDefault = true) 101 private float mAdaptRateBg = DEFAULT_ADAPT_RATE_BG; 102 @GenerateFieldPort(name = "adaptRateFg", hasDefault = true) 103 private float mAdaptRateFg = DEFAULT_ADAPT_RATE_FG; 104 @GenerateFieldPort(name = "maskVerifyRate", hasDefault = true) 105 private float mVerifyRate = DEFAULT_MASK_VERIFY_RATE; 106 @GenerateFieldPort(name = "learningDoneListener", hasDefault = true) 107 private LearningDoneListener mLearningDoneListener = null; 108 109 @GenerateFieldPort(name = "useTheForce", hasDefault = true) 110 private boolean mUseTheForce = false; 111 112 @GenerateFinalPort(name = "provideDebugOutputs", hasDefault = true) 113 private boolean mProvideDebugOutputs = false; 114 115 // Whether to mirror the background or not. For ex, the Camera app 116 // would mirror the preview for the front camera 117 @GenerateFieldPort(name = "mirrorBg", hasDefault = true) 118 private boolean mMirrorBg = false; 119 120 // The orientation of the display. This will change the flipping 121 // coordinates, if we were to mirror the background 122 @GenerateFieldPort(name = "orientation", hasDefault = true) 123 private int mOrientation = 0; 124 125 /** Default algorithm parameter values, for non-shader use */ 126 127 // Frame count for learning bg model 128 private static final int DEFAULT_LEARNING_DURATION = 40; 129 // Frame count for learning verification 130 private static final int DEFAULT_LEARNING_VERIFY_DURATION = 10; 131 // Maximum distance (in standard deviations) for considering a pixel as background 132 private static final float DEFAULT_ACCEPT_STDDEV = 0.85f; 133 // Variance threshold scale factor for large scale of hierarchy 134 private static final float DEFAULT_HIER_LRG_SCALE = 0.7f; 135 // Variance threshold scale factor for medium scale of hierarchy 136 private static final float DEFAULT_HIER_MID_SCALE = 0.6f; 137 // Variance threshold scale factor for small scale of hierarchy 138 private static final float DEFAULT_HIER_SML_SCALE = 0.5f; 139 // Width of foreground / background mask. 140 private static final int DEFAULT_MASK_WIDTH_EXPONENT = 8; 141 // Height of foreground / background mask. 142 private static final int DEFAULT_MASK_HEIGHT_EXPONENT = 8; 143 // Area over which to average for large scale (length in pixels = 2^HIERARCHY_*_EXPONENT) 144 private static final int DEFAULT_HIER_LRG_EXPONENT = 3; 145 // Area over which to average for medium scale 146 private static final int DEFAULT_HIER_MID_EXPONENT = 2; 147 // Area over which to average for small scale 148 private static final int DEFAULT_HIER_SML_EXPONENT = 0; 149 // Scale factor for luminance channel in distance calculations (larger = more significant) 150 private static final float DEFAULT_Y_SCALE_FACTOR = 0.40f; 151 // Scale factor for chroma channels in distance calculations 152 private static final float DEFAULT_UV_SCALE_FACTOR = 1.35f; 153 // Mask value to start blending away from background 154 private static final float DEFAULT_MASK_BLEND_BG = 0.65f; 155 // Mask value to start blending away from foreground 156 private static final float DEFAULT_MASK_BLEND_FG = 0.95f; 157 // Exposure stop number to change the brightness of foreground 158 private static final float DEFAULT_EXPOSURE_CHANGE = 1.0f; 159 // White balance change in Red channel for foreground 160 private static final float DEFAULT_WHITE_BALANCE_RED_CHANGE = 0.0f; 161 // White balance change in Blue channel for foreground 162 private static final float DEFAULT_WHITE_BALANCE_BLUE_CHANGE = 0.0f; 163 // Variable to control automatic white balance effect 164 // 0.f -> Auto WB is off; 1.f-> Auto WB is on 165 private static final int DEFAULT_WHITE_BALANCE_TOGGLE = 0; 166 167 // Default rate at which to learn bg model during learning period 168 private static final float DEFAULT_LEARNING_ADAPT_RATE = 0.2f; 169 // Default rate at which to learn bg model from new background pixels 170 private static final float DEFAULT_ADAPT_RATE_BG = 0.0f; 171 // Default rate at which to learn bg model from new foreground pixels 172 private static final float DEFAULT_ADAPT_RATE_FG = 0.0f; 173 // Default rate at which to verify whether background is stable 174 private static final float DEFAULT_MASK_VERIFY_RATE = 0.25f; 175 // Default rate at which to verify whether background is stable 176 private static final int DEFAULT_LEARNING_DONE_THRESHOLD = 20; 177 178 // Default 3x3 matrix, column major, for fitting background 1:1 179 private static final float[] DEFAULT_BG_FIT_TRANSFORM = new float[] { 180 1.0f, 0.0f, 0.0f, 181 0.0f, 1.0f, 0.0f, 182 0.0f, 0.0f, 1.0f 183 }; 184 185 /** Default algorithm parameter values, for shader use */ 186 187 // Area over which to blur binary mask values (length in pixels = 2^MASK_SMOOTH_EXPONENT) 188 private static final String MASK_SMOOTH_EXPONENT = "2.0"; 189 // Scale value for mapping variance distance to fit nicely to 0-1, 8-bit 190 private static final String DISTANCE_STORAGE_SCALE = "0.6"; 191 // Scale value for mapping variance to fit nicely to 0-1, 8-bit 192 private static final String VARIANCE_STORAGE_SCALE = "5.0"; 193 // Default scale of auto white balance parameters 194 private static final String DEFAULT_AUTO_WB_SCALE = "0.25"; 195 // Minimum variance (0-255 scale) 196 private static final String MIN_VARIANCE = "3.0"; 197 // Column-major array for 4x4 matrix converting RGB to YCbCr, JPEG definition (no pedestal) 198 private static final String RGB_TO_YUV_MATRIX = "0.299, -0.168736, 0.5, 0.000, " + 199 "0.587, -0.331264, -0.418688, 0.000, " + 200 "0.114, 0.5, -0.081312, 0.000, " + 201 "0.000, 0.5, 0.5, 1.000 "; 202 /** Stream names */ 203 204 private static final String[] mInputNames = {"video", 205 "background"}; 206 207 private static final String[] mOutputNames = {"video"}; 208 209 private static final String[] mDebugOutputNames = {"debug1", 210 "debug2"}; 211 212 /** Other private variables */ 213 214 private FrameFormat mOutputFormat; 215 private MutableFrameFormat mMemoryFormat; 216 private MutableFrameFormat mMaskFormat; 217 private MutableFrameFormat mAverageFormat; 218 219 private final boolean mLogVerbose; 220 private static final String TAG = "BackDropperFilter"; 221 222 /** Shader source code */ 223 224 // Shared uniforms and utility functions 225 private static String mSharedUtilShader = 226 "precision mediump float;\n" + 227 "uniform float fg_adapt_rate;\n" + 228 "uniform float bg_adapt_rate;\n" + 229 "const mat4 coeff_yuv = mat4(" + RGB_TO_YUV_MATRIX + ");\n" + 230 "const float dist_scale = " + DISTANCE_STORAGE_SCALE + ";\n" + 231 "const float inv_dist_scale = 1. / dist_scale;\n" + 232 "const float var_scale=" + VARIANCE_STORAGE_SCALE + ";\n" + 233 "const float inv_var_scale = 1. / var_scale;\n" + 234 "const float min_variance = inv_var_scale *" + MIN_VARIANCE + "/ 256.;\n" + 235 "const float auto_wb_scale = " + DEFAULT_AUTO_WB_SCALE + ";\n" + 236 "\n" + 237 // Variance distance in luminance between current pixel and background model 238 "float gauss_dist_y(float y, float mean, float variance) {\n" + 239 " float dist = (y - mean) * (y - mean) / variance;\n" + 240 " return dist;\n" + 241 "}\n" + 242 // Sum of variance distances in chroma between current pixel and background 243 // model 244 "float gauss_dist_uv(vec2 uv, vec2 mean, vec2 variance) {\n" + 245 " vec2 dist = (uv - mean) * (uv - mean) / variance;\n" + 246 " return dist.r + dist.g;\n" + 247 "}\n" + 248 // Select learning rate for pixel based on smoothed decision mask alpha 249 "float local_adapt_rate(float alpha) {\n" + 250 " return mix(bg_adapt_rate, fg_adapt_rate, alpha);\n" + 251 "}\n" + 252 "\n"; 253 254 // Distance calculation shader. Calculates a distance metric between the foreground and the 255 // current background model, in both luminance and in chroma (yuv space). Distance is 256 // measured in variances from the mean background value. For chroma, the distance is the sum 257 // of the two individual color channel distances. The distances are output on the b and alpha 258 // channels, r and g are for debug information. 259 // Inputs: 260 // tex_sampler_0: Mip-map for foreground (live) video frame. 261 // tex_sampler_1: Background mean mask. 262 // tex_sampler_2: Background variance mask. 263 // subsample_level: Level on foreground frame's mip-map. 264 private static final String mBgDistanceShader = 265 "uniform sampler2D tex_sampler_0;\n" + 266 "uniform sampler2D tex_sampler_1;\n" + 267 "uniform sampler2D tex_sampler_2;\n" + 268 "uniform float subsample_level;\n" + 269 "varying vec2 v_texcoord;\n" + 270 "void main() {\n" + 271 " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 272 " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 273 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 274 " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 275 "\n" + 276 " float dist_y = gauss_dist_y(fg.r, mean.r, variance.r);\n" + 277 " float dist_uv = gauss_dist_uv(fg.gb, mean.gb, variance.gb);\n" + 278 " gl_FragColor = vec4(0.5*fg.rg, dist_scale*dist_y, dist_scale*dist_uv);\n" + 279 "}\n"; 280 281 // Foreground/background mask decision shader. Decides whether a frame is in the foreground or 282 // the background using a hierarchical threshold on the distance. Binary foreground/background 283 // mask is placed in the alpha channel. The RGB channels contain debug information. 284 private static final String mBgMaskShader = 285 "uniform sampler2D tex_sampler_0;\n" + 286 "uniform float accept_variance;\n" + 287 "uniform vec2 yuv_weights;\n" + 288 "uniform float scale_lrg;\n" + 289 "uniform float scale_mid;\n" + 290 "uniform float scale_sml;\n" + 291 "uniform float exp_lrg;\n" + 292 "uniform float exp_mid;\n" + 293 "uniform float exp_sml;\n" + 294 "varying vec2 v_texcoord;\n" + 295 // Decide whether pixel is foreground or background based on Y and UV 296 // distance and maximum acceptable variance. 297 // yuv_weights.x is smaller than yuv_weights.y to discount the influence of shadow 298 "bool is_fg(vec2 dist_yc, float accept_variance) {\n" + 299 " return ( dot(yuv_weights, dist_yc) >= accept_variance );\n" + 300 "}\n" + 301 "void main() {\n" + 302 " vec4 dist_lrg_sc = texture2D(tex_sampler_0, v_texcoord, exp_lrg);\n" + 303 " vec4 dist_mid_sc = texture2D(tex_sampler_0, v_texcoord, exp_mid);\n" + 304 " vec4 dist_sml_sc = texture2D(tex_sampler_0, v_texcoord, exp_sml);\n" + 305 " vec2 dist_lrg = inv_dist_scale * dist_lrg_sc.ba;\n" + 306 " vec2 dist_mid = inv_dist_scale * dist_mid_sc.ba;\n" + 307 " vec2 dist_sml = inv_dist_scale * dist_sml_sc.ba;\n" + 308 " vec2 norm_dist = 0.75 * dist_sml / accept_variance;\n" + // For debug viz 309 " bool is_fg_lrg = is_fg(dist_lrg, accept_variance * scale_lrg);\n" + 310 " bool is_fg_mid = is_fg_lrg || is_fg(dist_mid, accept_variance * scale_mid);\n" + 311 " float is_fg_sml =\n" + 312 " float(is_fg_mid || is_fg(dist_sml, accept_variance * scale_sml));\n" + 313 " float alpha = 0.5 * is_fg_sml + 0.3 * float(is_fg_mid) + 0.2 * float(is_fg_lrg);\n" + 314 " gl_FragColor = vec4(alpha, norm_dist, is_fg_sml);\n" + 315 "}\n"; 316 317 // Automatic White Balance parameter decision shader 318 // Use the Gray World assumption that in a white balance corrected image, the average of R, G, B 319 // channel will be a common gray value. 320 // To match the white balance of foreground and background, the average of R, G, B channel of 321 // two videos should match. 322 // Inputs: 323 // tex_sampler_0: Mip-map for foreground (live) video frame. 324 // tex_sampler_1: Mip-map for background (playback) video frame. 325 // pyramid_depth: Depth of input frames' mip-maps. 326 private static final String mAutomaticWhiteBalance = 327 "uniform sampler2D tex_sampler_0;\n" + 328 "uniform sampler2D tex_sampler_1;\n" + 329 "uniform float pyramid_depth;\n" + 330 "uniform bool autowb_toggle;\n" + 331 "varying vec2 v_texcoord;\n" + 332 "void main() {\n" + 333 " vec4 mean_video = texture2D(tex_sampler_0, v_texcoord, pyramid_depth);\n"+ 334 " vec4 mean_bg = texture2D(tex_sampler_1, v_texcoord, pyramid_depth);\n" + 335 // If Auto WB is toggled off, the return texture will be a unicolor texture of value 1 336 // If Auto WB is toggled on, the return texture will be a unicolor texture with 337 // adjustment parameters for R and B channels stored in the corresponding channel 338 " float green_normalizer = mean_video.g / mean_bg.g;\n"+ 339 " vec4 adjusted_value = vec4(mean_bg.r / mean_video.r * green_normalizer, 1., \n" + 340 " mean_bg.b / mean_video.b * green_normalizer, 1.) * auto_wb_scale; \n" + 341 " gl_FragColor = autowb_toggle ? adjusted_value : vec4(auto_wb_scale);\n" + 342 "}\n"; 343 344 345 // Background subtraction shader. Uses a mipmap of the binary mask map to blend smoothly between 346 // foreground and background 347 // Inputs: 348 // tex_sampler_0: Foreground (live) video frame. 349 // tex_sampler_1: Background (playback) video frame. 350 // tex_sampler_2: Foreground/background mask. 351 // tex_sampler_3: Auto white-balance factors. 352 private static final String mBgSubtractShader = 353 "uniform mat3 bg_fit_transform;\n" + 354 "uniform float mask_blend_bg;\n" + 355 "uniform float mask_blend_fg;\n" + 356 "uniform float exposure_change;\n" + 357 "uniform float whitebalancered_change;\n" + 358 "uniform float whitebalanceblue_change;\n" + 359 "uniform sampler2D tex_sampler_0;\n" + 360 "uniform sampler2D tex_sampler_1;\n" + 361 "uniform sampler2D tex_sampler_2;\n" + 362 "uniform sampler2D tex_sampler_3;\n" + 363 "varying vec2 v_texcoord;\n" + 364 "void main() {\n" + 365 " vec2 bg_texcoord = (bg_fit_transform * vec3(v_texcoord, 1.)).xy;\n" + 366 " vec4 bg_rgb = texture2D(tex_sampler_1, bg_texcoord);\n" + 367 // The foreground texture is modified by multiplying both manual and auto white balance changes in R and B 368 // channel and multiplying exposure change in all R, G, B channels. 369 " vec4 wb_auto_scale = texture2D(tex_sampler_3, v_texcoord) * exposure_change / auto_wb_scale;\n" + 370 " vec4 wb_manual_scale = vec4(1. + whitebalancered_change, 1., 1. + whitebalanceblue_change, 1.);\n" + 371 " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord);\n" + 372 " vec4 fg_adjusted = fg_rgb * wb_manual_scale * wb_auto_scale;\n"+ 373 " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 374 " " + MASK_SMOOTH_EXPONENT + ");\n" + 375 " float alpha = smoothstep(mask_blend_bg, mask_blend_fg, mask.a);\n" + 376 " gl_FragColor = mix(bg_rgb, fg_adjusted, alpha);\n"; 377 378 // May the Force... Makes the foreground object translucent blue, with a bright 379 // blue-white outline 380 private static final String mBgSubtractForceShader = 381 " vec4 ghost_rgb = (fg_adjusted * 0.7 + vec4(0.3,0.3,0.4,0.))*0.65 + \n" + 382 " 0.35*bg_rgb;\n" + 383 " float glow_start = 0.75 * mask_blend_bg; \n"+ 384 " float glow_max = mask_blend_bg; \n"+ 385 " gl_FragColor = mask.a < glow_start ? bg_rgb : \n" + 386 " mask.a < glow_max ? mix(bg_rgb, vec4(0.9,0.9,1.0,1.0), \n" + 387 " (mask.a - glow_start) / (glow_max - glow_start) ) : \n" + 388 " mask.a < mask_blend_fg ? mix(vec4(0.9,0.9,1.0,1.0), ghost_rgb, \n" + 389 " (mask.a - glow_max) / (mask_blend_fg - glow_max) ) : \n" + 390 " ghost_rgb;\n" + 391 "}\n"; 392 393 // Background model mean update shader. Skews the current model mean toward the most recent pixel 394 // value for a pixel, weighted by the learning rate and by whether the pixel is classified as 395 // foreground or background. 396 // Inputs: 397 // tex_sampler_0: Mip-map for foreground (live) video frame. 398 // tex_sampler_1: Background mean mask. 399 // tex_sampler_2: Foreground/background mask. 400 // subsample_level: Level on foreground frame's mip-map. 401 private static final String mUpdateBgModelMeanShader = 402 "uniform sampler2D tex_sampler_0;\n" + 403 "uniform sampler2D tex_sampler_1;\n" + 404 "uniform sampler2D tex_sampler_2;\n" + 405 "uniform float subsample_level;\n" + 406 "varying vec2 v_texcoord;\n" + 407 "void main() {\n" + 408 " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 409 " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 410 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 411 " vec4 mask = texture2D(tex_sampler_2, v_texcoord, \n" + 412 " " + MASK_SMOOTH_EXPONENT + ");\n" + 413 "\n" + 414 " float alpha = local_adapt_rate(mask.a);\n" + 415 " vec4 new_mean = mix(mean, fg, alpha);\n" + 416 " gl_FragColor = new_mean;\n" + 417 "}\n"; 418 419 // Background model variance update shader. Skews the current model variance toward the most 420 // recent variance for the pixel, weighted by the learning rate and by whether the pixel is 421 // classified as foreground or background. 422 // Inputs: 423 // tex_sampler_0: Mip-map for foreground (live) video frame. 424 // tex_sampler_1: Background mean mask. 425 // tex_sampler_2: Background variance mask. 426 // tex_sampler_3: Foreground/background mask. 427 // subsample_level: Level on foreground frame's mip-map. 428 // TODO: to improve efficiency, use single mark for mean + variance, then merge this into 429 // mUpdateBgModelMeanShader. 430 private static final String mUpdateBgModelVarianceShader = 431 "uniform sampler2D tex_sampler_0;\n" + 432 "uniform sampler2D tex_sampler_1;\n" + 433 "uniform sampler2D tex_sampler_2;\n" + 434 "uniform sampler2D tex_sampler_3;\n" + 435 "uniform float subsample_level;\n" + 436 "varying vec2 v_texcoord;\n" + 437 "void main() {\n" + 438 " vec4 fg_rgb = texture2D(tex_sampler_0, v_texcoord, subsample_level);\n" + 439 " vec4 fg = coeff_yuv * vec4(fg_rgb.rgb, 1.);\n" + 440 " vec4 mean = texture2D(tex_sampler_1, v_texcoord);\n" + 441 " vec4 variance = inv_var_scale * texture2D(tex_sampler_2, v_texcoord);\n" + 442 " vec4 mask = texture2D(tex_sampler_3, v_texcoord, \n" + 443 " " + MASK_SMOOTH_EXPONENT + ");\n" + 444 "\n" + 445 " float alpha = local_adapt_rate(mask.a);\n" + 446 " vec4 cur_variance = (fg-mean)*(fg-mean);\n" + 447 " vec4 new_variance = mix(variance, cur_variance, alpha);\n" + 448 " new_variance = max(new_variance, vec4(min_variance));\n" + 449 " gl_FragColor = var_scale * new_variance;\n" + 450 "}\n"; 451 452 // Background verification shader. Skews the current background verification mask towards the 453 // most recent frame, weighted by the learning rate. 454 private static final String mMaskVerifyShader = 455 "uniform sampler2D tex_sampler_0;\n" + 456 "uniform sampler2D tex_sampler_1;\n" + 457 "uniform float verify_rate;\n" + 458 "varying vec2 v_texcoord;\n" + 459 "void main() {\n" + 460 " vec4 lastmask = texture2D(tex_sampler_0, v_texcoord);\n" + 461 " vec4 mask = texture2D(tex_sampler_1, v_texcoord);\n" + 462 " float newmask = mix(lastmask.a, mask.a, verify_rate);\n" + 463 " gl_FragColor = vec4(0., 0., 0., newmask);\n" + 464 "}\n"; 465 466 /** Shader program objects */ 467 468 private ShaderProgram mBgDistProgram; 469 private ShaderProgram mBgMaskProgram; 470 private ShaderProgram mBgSubtractProgram; 471 private ShaderProgram mBgUpdateMeanProgram; 472 private ShaderProgram mBgUpdateVarianceProgram; 473 private ShaderProgram mCopyOutProgram; 474 private ShaderProgram mAutomaticWhiteBalanceProgram; 475 private ShaderProgram mMaskVerifyProgram; 476 private ShaderProgram copyShaderProgram; 477 478 /** Background model storage */ 479 480 private boolean mPingPong; 481 private GLFrame mBgMean[]; 482 private GLFrame mBgVariance[]; 483 private GLFrame mMaskVerify[]; 484 private GLFrame mDistance; 485 private GLFrame mAutoWB; 486 private GLFrame mMask; 487 private GLFrame mVideoInput; 488 private GLFrame mBgInput; 489 private GLFrame mMaskAverage; 490 491 /** Overall filter state */ 492 493 private boolean isOpen; 494 private int mFrameCount; 495 private boolean mStartLearning; 496 private boolean mBackgroundFitModeChanged; 497 private float mRelativeAspect; 498 private int mPyramidDepth; 499 private int mSubsampleLevel; 500 501 /** Learning listener object */ 502 503 public interface LearningDoneListener { 504 public void onLearningDone(BackDropperFilter filter); 505 } 506 507 /** Public Filter methods */ 508 509 public BackDropperFilter(String name) { 510 super(name); 511 512 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 513 } 514 515 @Override 516 public void setupPorts() { 517 // Inputs. 518 // TODO: Target should be GPU, but relaxed for now. 519 FrameFormat imageFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 520 FrameFormat.TARGET_UNSPECIFIED); 521 for (String inputName : mInputNames) { 522 addMaskedInputPort(inputName, imageFormat); 523 } 524 // Normal outputs 525 for (String outputName : mOutputNames) { 526 addOutputBasedOnInput(outputName, "video"); 527 } 528 529 // Debug outputs 530 if (mProvideDebugOutputs) { 531 for (String outputName : mDebugOutputNames) { 532 addOutputBasedOnInput(outputName, "video"); 533 } 534 } 535 } 536 537 @Override 538 public FrameFormat getOutputFormat(String portName, FrameFormat inputFormat) { 539 // Create memory format based on video input. 540 MutableFrameFormat format = inputFormat.mutableCopy(); 541 // Is this a debug output port? If so, leave dimensions unspecified. 542 if (!Arrays.asList(mOutputNames).contains(portName)) { 543 format.setDimensions(FrameFormat.SIZE_UNSPECIFIED, FrameFormat.SIZE_UNSPECIFIED); 544 } 545 return format; 546 } 547 548 private boolean createMemoryFormat(FrameFormat inputFormat) { 549 // We can't resize because that would require re-learning. 550 if (mMemoryFormat != null) { 551 return false; 552 } 553 554 if (inputFormat.getWidth() == FrameFormat.SIZE_UNSPECIFIED || 555 inputFormat.getHeight() == FrameFormat.SIZE_UNSPECIFIED) { 556 throw new RuntimeException("Attempting to process input frame with unknown size"); 557 } 558 559 mMaskFormat = inputFormat.mutableCopy(); 560 int maskWidth = (int)Math.pow(2, mMaskWidthExp); 561 int maskHeight = (int)Math.pow(2, mMaskHeightExp); 562 mMaskFormat.setDimensions(maskWidth, maskHeight); 563 564 mPyramidDepth = Math.max(mMaskWidthExp, mMaskHeightExp); 565 mMemoryFormat = mMaskFormat.mutableCopy(); 566 int widthExp = Math.max(mMaskWidthExp, pyramidLevel(inputFormat.getWidth())); 567 int heightExp = Math.max(mMaskHeightExp, pyramidLevel(inputFormat.getHeight())); 568 mPyramidDepth = Math.max(widthExp, heightExp); 569 int memWidth = Math.max(maskWidth, (int)Math.pow(2, widthExp)); 570 int memHeight = Math.max(maskHeight, (int)Math.pow(2, heightExp)); 571 mMemoryFormat.setDimensions(memWidth, memHeight); 572 mSubsampleLevel = mPyramidDepth - Math.max(mMaskWidthExp, mMaskHeightExp); 573 574 if (mLogVerbose) { 575 Log.v(TAG, "Mask frames size " + maskWidth + " x " + maskHeight); 576 Log.v(TAG, "Pyramid levels " + widthExp + " x " + heightExp); 577 Log.v(TAG, "Memory frames size " + memWidth + " x " + memHeight); 578 } 579 580 mAverageFormat = inputFormat.mutableCopy(); 581 mAverageFormat.setDimensions(1,1); 582 return true; 583 } 584 585 public void prepare(FilterContext context){ 586 if (mLogVerbose) Log.v(TAG, "Preparing BackDropperFilter!"); 587 588 mBgMean = new GLFrame[2]; 589 mBgVariance = new GLFrame[2]; 590 mMaskVerify = new GLFrame[2]; 591 copyShaderProgram = ShaderProgram.createIdentity(context); 592 } 593 594 private void allocateFrames(FrameFormat inputFormat, FilterContext context) { 595 if (!createMemoryFormat(inputFormat)) { 596 return; // All set. 597 } 598 if (mLogVerbose) Log.v(TAG, "Allocating BackDropperFilter frames"); 599 600 // Create initial background model values 601 int numBytes = mMaskFormat.getSize(); 602 byte[] initialBgMean = new byte[numBytes]; 603 byte[] initialBgVariance = new byte[numBytes]; 604 byte[] initialMaskVerify = new byte[numBytes]; 605 for (int i = 0; i < numBytes; i++) { 606 initialBgMean[i] = (byte)128; 607 initialBgVariance[i] = (byte)10; 608 initialMaskVerify[i] = (byte)0; 609 } 610 611 // Get frames to store background model in 612 for (int i = 0; i < 2; i++) { 613 mBgMean[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 614 mBgMean[i].setData(initialBgMean, 0, numBytes); 615 616 mBgVariance[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 617 mBgVariance[i].setData(initialBgVariance, 0, numBytes); 618 619 mMaskVerify[i] = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 620 mMaskVerify[i].setData(initialMaskVerify, 0, numBytes); 621 } 622 623 // Get frames to store other textures in 624 if (mLogVerbose) Log.v(TAG, "Done allocating texture for Mean and Variance objects!"); 625 626 mDistance = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 627 mMask = (GLFrame)context.getFrameManager().newFrame(mMaskFormat); 628 mAutoWB = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 629 mVideoInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 630 mBgInput = (GLFrame)context.getFrameManager().newFrame(mMemoryFormat); 631 mMaskAverage = (GLFrame)context.getFrameManager().newFrame(mAverageFormat); 632 633 // Create shader programs 634 mBgDistProgram = new ShaderProgram(context, mSharedUtilShader + mBgDistanceShader); 635 mBgDistProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 636 637 mBgMaskProgram = new ShaderProgram(context, mSharedUtilShader + mBgMaskShader); 638 mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 639 float[] yuvWeights = { mLumScale, mChromaScale }; 640 mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 641 mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 642 mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 643 mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 644 mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 645 mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 646 mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 647 648 if (mUseTheForce) { 649 mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + mBgSubtractForceShader); 650 } else { 651 mBgSubtractProgram = new ShaderProgram(context, mSharedUtilShader + mBgSubtractShader + "}\n"); 652 } 653 mBgSubtractProgram.setHostValue("bg_fit_transform", DEFAULT_BG_FIT_TRANSFORM); 654 mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 655 mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 656 mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 657 mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 658 mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 659 660 661 mBgUpdateMeanProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelMeanShader); 662 mBgUpdateMeanProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 663 664 mBgUpdateVarianceProgram = new ShaderProgram(context, mSharedUtilShader + mUpdateBgModelVarianceShader); 665 mBgUpdateVarianceProgram.setHostValue("subsample_level", (float)mSubsampleLevel); 666 667 mCopyOutProgram = ShaderProgram.createIdentity(context); 668 669 mAutomaticWhiteBalanceProgram = new ShaderProgram(context, mSharedUtilShader + mAutomaticWhiteBalance); 670 mAutomaticWhiteBalanceProgram.setHostValue("pyramid_depth", (float)mPyramidDepth); 671 mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 672 673 mMaskVerifyProgram = new ShaderProgram(context, mSharedUtilShader + mMaskVerifyShader); 674 mMaskVerifyProgram.setHostValue("verify_rate", mVerifyRate); 675 676 if (mLogVerbose) Log.v(TAG, "Shader width set to " + mMemoryFormat.getWidth()); 677 678 mRelativeAspect = 1.f; 679 680 mFrameCount = 0; 681 mStartLearning = true; 682 } 683 684 public void process(FilterContext context) { 685 // Grab inputs and ready intermediate frames and outputs. 686 Frame video = pullInput("video"); 687 Frame background = pullInput("background"); 688 allocateFrames(video.getFormat(), context); 689 690 // Update learning rate after initial learning period 691 if (mStartLearning) { 692 if (mLogVerbose) Log.v(TAG, "Starting learning"); 693 mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 694 mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 695 mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateLearning); 696 mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateLearning); 697 mFrameCount = 0; 698 mStartLearning = false; 699 } 700 701 // Select correct pingpong buffers 702 int inputIndex = mPingPong ? 0 : 1; 703 int outputIndex = mPingPong ? 1 : 0; 704 mPingPong = !mPingPong; 705 706 // Check relative aspect ratios 707 updateBgScaling(video, background, mBackgroundFitModeChanged); 708 mBackgroundFitModeChanged = false; 709 710 // Make copies for input frames to GLFrames 711 712 copyShaderProgram.process(video, mVideoInput); 713 copyShaderProgram.process(background, mBgInput); 714 715 mVideoInput.generateMipMap(); 716 mVideoInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 717 GLES20.GL_LINEAR_MIPMAP_NEAREST); 718 719 mBgInput.generateMipMap(); 720 mBgInput.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 721 GLES20.GL_LINEAR_MIPMAP_NEAREST); 722 723 // Process shaders 724 Frame[] distInputs = { mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex] }; 725 mBgDistProgram.process(distInputs, mDistance); 726 mDistance.generateMipMap(); 727 mDistance.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 728 GLES20.GL_LINEAR_MIPMAP_NEAREST); 729 730 mBgMaskProgram.process(mDistance, mMask); 731 mMask.generateMipMap(); 732 mMask.setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 733 GLES20.GL_LINEAR_MIPMAP_NEAREST); 734 735 Frame[] autoWBInputs = { mVideoInput, mBgInput }; 736 mAutomaticWhiteBalanceProgram.process(autoWBInputs, mAutoWB); 737 738 if (mFrameCount <= mLearningDuration) { 739 // During learning 740 pushOutput("video", video); 741 742 if (mFrameCount == mLearningDuration - mLearningVerifyDuration) { 743 copyShaderProgram.process(mMask, mMaskVerify[outputIndex]); 744 745 mBgUpdateMeanProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 746 mBgUpdateMeanProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 747 mBgUpdateVarianceProgram.setHostValue("bg_adapt_rate", mAdaptRateBg); 748 mBgUpdateVarianceProgram.setHostValue("fg_adapt_rate", mAdaptRateFg); 749 750 751 } else if (mFrameCount > mLearningDuration - mLearningVerifyDuration) { 752 // In the learning verification stage, compute background masks and a weighted average 753 // with weights grow exponentially with time 754 Frame[] maskVerifyInputs = {mMaskVerify[inputIndex], mMask}; 755 mMaskVerifyProgram.process(maskVerifyInputs, mMaskVerify[outputIndex]); 756 mMaskVerify[outputIndex].generateMipMap(); 757 mMaskVerify[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 758 GLES20.GL_LINEAR_MIPMAP_NEAREST); 759 } 760 761 if (mFrameCount == mLearningDuration) { 762 // In the last verification frame, verify if the verification mask is almost blank 763 // If not, restart learning 764 copyShaderProgram.process(mMaskVerify[outputIndex], mMaskAverage); 765 ByteBuffer mMaskAverageByteBuffer = mMaskAverage.getData(); 766 byte[] mask_average = mMaskAverageByteBuffer.array(); 767 int bi = (int)(mask_average[3] & 0xFF); 768 if (mLogVerbose) Log.v(TAG, String.format("Mask_average is %d", bi)); 769 770 if (bi >= DEFAULT_LEARNING_DONE_THRESHOLD) { 771 mStartLearning = true; // Restart learning 772 } else { 773 if (mLogVerbose) Log.v(TAG, "Learning done"); 774 if (mLearningDoneListener != null) { 775 mLearningDoneListener.onLearningDone(this); 776 } 777 } 778 } 779 } else { 780 Frame output = context.getFrameManager().newFrame(video.getFormat()); 781 Frame[] subtractInputs = { video, background, mMask, mAutoWB }; 782 mBgSubtractProgram.process(subtractInputs, output); 783 pushOutput("video", output); 784 output.release(); 785 } 786 787 // Compute mean and variance of the background 788 if (mFrameCount < mLearningDuration - mLearningVerifyDuration || 789 mAdaptRateBg > 0.0 || mAdaptRateFg > 0.0) { 790 Frame[] meanUpdateInputs = { mVideoInput, mBgMean[inputIndex], mMask }; 791 mBgUpdateMeanProgram.process(meanUpdateInputs, mBgMean[outputIndex]); 792 mBgMean[outputIndex].generateMipMap(); 793 mBgMean[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 794 GLES20.GL_LINEAR_MIPMAP_NEAREST); 795 796 Frame[] varianceUpdateInputs = { 797 mVideoInput, mBgMean[inputIndex], mBgVariance[inputIndex], mMask 798 }; 799 mBgUpdateVarianceProgram.process(varianceUpdateInputs, mBgVariance[outputIndex]); 800 mBgVariance[outputIndex].generateMipMap(); 801 mBgVariance[outputIndex].setTextureParameter(GLES20.GL_TEXTURE_MIN_FILTER, 802 GLES20.GL_LINEAR_MIPMAP_NEAREST); 803 } 804 805 // Provide debug output to two smaller viewers 806 if (mProvideDebugOutputs) { 807 Frame dbg1 = context.getFrameManager().newFrame(video.getFormat()); 808 mCopyOutProgram.process(video, dbg1); 809 pushOutput("debug1", dbg1); 810 dbg1.release(); 811 812 Frame dbg2 = context.getFrameManager().newFrame(mMemoryFormat); 813 mCopyOutProgram.process(mMask, dbg2); 814 pushOutput("debug2", dbg2); 815 dbg2.release(); 816 } 817 818 mFrameCount++; 819 820 if (mLogVerbose) { 821 if (mFrameCount % 30 == 0) { 822 if (startTime == -1) { 823 context.getGLEnvironment().activate(); 824 GLES20.glFinish(); 825 startTime = SystemClock.elapsedRealtime(); 826 } else { 827 context.getGLEnvironment().activate(); 828 GLES20.glFinish(); 829 long endTime = SystemClock.elapsedRealtime(); 830 Log.v(TAG, "Avg. frame duration: " + String.format("%.2f",(endTime-startTime)/30.) + 831 " ms. Avg. fps: " + String.format("%.2f", 1000./((endTime-startTime)/30.)) ); 832 startTime = endTime; 833 } 834 } 835 } 836 } 837 838 private long startTime = -1; 839 840 public void close(FilterContext context) { 841 if (mMemoryFormat == null) { 842 return; 843 } 844 845 if (mLogVerbose) Log.v(TAG, "Filter Closing!"); 846 for (int i = 0; i < 2; i++) { 847 mBgMean[i].release(); 848 mBgVariance[i].release(); 849 mMaskVerify[i].release(); 850 } 851 mDistance.release(); 852 mMask.release(); 853 mAutoWB.release(); 854 mVideoInput.release(); 855 mBgInput.release(); 856 mMaskAverage.release(); 857 858 mMemoryFormat = null; 859 } 860 861 // Relearn background model 862 synchronized public void relearn() { 863 // Let the processing thread know about learning restart 864 mStartLearning = true; 865 } 866 867 @Override 868 public void fieldPortValueUpdated(String name, FilterContext context) { 869 // TODO: Many of these can be made ProgramPorts! 870 if (name.equals("backgroundFitMode")) { 871 mBackgroundFitModeChanged = true; 872 } else if (name.equals("acceptStddev")) { 873 mBgMaskProgram.setHostValue("accept_variance", mAcceptStddev * mAcceptStddev); 874 } else if (name.equals("hierLrgScale")) { 875 mBgMaskProgram.setHostValue("scale_lrg", mHierarchyLrgScale); 876 } else if (name.equals("hierMidScale")) { 877 mBgMaskProgram.setHostValue("scale_mid", mHierarchyMidScale); 878 } else if (name.equals("hierSmlScale")) { 879 mBgMaskProgram.setHostValue("scale_sml", mHierarchySmlScale); 880 } else if (name.equals("hierLrgExp")) { 881 mBgMaskProgram.setHostValue("exp_lrg", (float)(mSubsampleLevel + mHierarchyLrgExp)); 882 } else if (name.equals("hierMidExp")) { 883 mBgMaskProgram.setHostValue("exp_mid", (float)(mSubsampleLevel + mHierarchyMidExp)); 884 } else if (name.equals("hierSmlExp")) { 885 mBgMaskProgram.setHostValue("exp_sml", (float)(mSubsampleLevel + mHierarchySmlExp)); 886 } else if (name.equals("lumScale") || name.equals("chromaScale")) { 887 float[] yuvWeights = { mLumScale, mChromaScale }; 888 mBgMaskProgram.setHostValue("yuv_weights", yuvWeights ); 889 } else if (name.equals("maskBg")) { 890 mBgSubtractProgram.setHostValue("mask_blend_bg", mMaskBg); 891 } else if (name.equals("maskFg")) { 892 mBgSubtractProgram.setHostValue("mask_blend_fg", mMaskFg); 893 } else if (name.equals("exposureChange")) { 894 mBgSubtractProgram.setHostValue("exposure_change", mExposureChange); 895 } else if (name.equals("whitebalanceredChange")) { 896 mBgSubtractProgram.setHostValue("whitebalancered_change", mWhiteBalanceRedChange); 897 } else if (name.equals("whitebalanceblueChange")) { 898 mBgSubtractProgram.setHostValue("whitebalanceblue_change", mWhiteBalanceBlueChange); 899 } else if (name.equals("autowbToggle")){ 900 mAutomaticWhiteBalanceProgram.setHostValue("autowb_toggle", mAutoWBToggle); 901 } 902 } 903 904 private void updateBgScaling(Frame video, Frame background, boolean fitModeChanged) { 905 float foregroundAspect = (float)video.getFormat().getWidth() / video.getFormat().getHeight(); 906 float backgroundAspect = (float)background.getFormat().getWidth() / background.getFormat().getHeight(); 907 float currentRelativeAspect = foregroundAspect/backgroundAspect; 908 if (currentRelativeAspect != mRelativeAspect || fitModeChanged) { 909 mRelativeAspect = currentRelativeAspect; 910 float xMin = 0.f, xWidth = 1.f, yMin = 0.f, yWidth = 1.f; 911 switch (mBackgroundFitMode) { 912 case BACKGROUND_STRETCH: 913 // Just map 1:1 914 break; 915 case BACKGROUND_FIT: 916 if (mRelativeAspect > 1.0f) { 917 // Foreground is wider than background, scale down 918 // background in X 919 xMin = 0.5f - 0.5f * mRelativeAspect; 920 xWidth = 1.f * mRelativeAspect; 921 } else { 922 // Foreground is taller than background, scale down 923 // background in Y 924 yMin = 0.5f - 0.5f / mRelativeAspect; 925 yWidth = 1 / mRelativeAspect; 926 } 927 break; 928 case BACKGROUND_FILL_CROP: 929 if (mRelativeAspect > 1.0f) { 930 // Foreground is wider than background, crop 931 // background in Y 932 yMin = 0.5f - 0.5f / mRelativeAspect; 933 yWidth = 1.f / mRelativeAspect; 934 } else { 935 // Foreground is taller than background, crop 936 // background in X 937 xMin = 0.5f - 0.5f * mRelativeAspect; 938 xWidth = mRelativeAspect; 939 } 940 break; 941 } 942 // If mirroring is required (for ex. the camera mirrors the preview 943 // in the front camera) 944 // TODO: Backdropper does not attempt to apply any other transformation 945 // than just flipping. However, in the current state, it's "x-axis" is always aligned 946 // with the Camera's width. Hence, we need to define the mirroring based on the camera 947 // orientation. In the future, a cleaner design would be to cast away all the rotation 948 // in a separate place. 949 if (mMirrorBg) { 950 if (mLogVerbose) Log.v(TAG, "Mirroring the background!"); 951 // Mirroring in portrait 952 if (mOrientation == 0 || mOrientation == 180) { 953 xWidth = -xWidth; 954 xMin = 1.0f - xMin; 955 } else { 956 // Mirroring in landscape 957 yWidth = -yWidth; 958 yMin = 1.0f - yMin; 959 } 960 } 961 if (mLogVerbose) Log.v(TAG, "bgTransform: xMin, yMin, xWidth, yWidth : " + 962 xMin + ", " + yMin + ", " + xWidth + ", " + yWidth + 963 ", mRelAspRatio = " + mRelativeAspect); 964 // The following matrix is the transpose of the actual matrix 965 float[] bgTransform = {xWidth, 0.f, 0.f, 966 0.f, yWidth, 0.f, 967 xMin, yMin, 1.f}; 968 mBgSubtractProgram.setHostValue("bg_fit_transform", bgTransform); 969 } 970 } 971 972 private int pyramidLevel(int size) { 973 return (int)Math.floor(Math.log10(size) / Math.log10(2)) - 1; 974 } 975 976 } 977
