1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18 package android.filterpacks.videosrc; 19 20 import android.content.Context; 21 import android.filterfw.core.Filter; 22 import android.filterfw.core.FilterContext; 23 import android.filterfw.core.Frame; 24 import android.filterfw.core.FrameFormat; 25 import android.filterfw.core.FrameManager; 26 import android.filterfw.core.GenerateFieldPort; 27 import android.filterfw.core.GenerateFinalPort; 28 import android.filterfw.core.GLFrame; 29 import android.filterfw.core.KeyValueMap; 30 import android.filterfw.core.MutableFrameFormat; 31 import android.filterfw.core.NativeFrame; 32 import android.filterfw.core.Program; 33 import android.filterfw.core.ShaderProgram; 34 import android.filterfw.format.ImageFormat; 35 import android.graphics.SurfaceTexture; 36 import android.hardware.Camera; 37 import android.os.ConditionVariable; 38 import android.opengl.Matrix; 39 40 import java.io.IOException; 41 import java.util.List; 42 import java.util.Set; 43 44 import android.util.Log; 45 46 /** 47 * @hide 48 */ 49 public class CameraSource extends Filter { 50 51 /** User-visible parameters */ 52 53 /** Camera ID to use for input. Defaults to 0. */ 54 @GenerateFieldPort(name = "id", hasDefault = true) 55 private int mCameraId = 0; 56 57 /** Frame width to request from camera. Actual size may not match requested. */ 58 @GenerateFieldPort(name = "width", hasDefault = true) 59 private int mWidth = 320; 60 61 /** Frame height to request from camera. Actual size may not match requested. */ 62 @GenerateFieldPort(name = "height", hasDefault = true) 63 private int mHeight = 240; 64 65 /** Stream framerate to request from camera. Actual frame rate may not match requested. */ 66 @GenerateFieldPort(name = "framerate", hasDefault = true) 67 private int mFps = 30; 68 69 /** Whether the filter should always wait for a new frame from the camera 70 * before providing output. If set to false, the filter will keep 71 * outputting the last frame it received from the camera if multiple process 72 * calls are received before the next update from the Camera. Defaults to true. 73 */ 74 @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true) 75 private boolean mWaitForNewFrame = true; 76 77 private Camera mCamera; 78 private GLFrame mCameraFrame; 79 private SurfaceTexture mSurfaceTexture; 80 private ShaderProgram mFrameExtractor; 81 private MutableFrameFormat mOutputFormat; 82 83 private float[] mCameraTransform; 84 private float[] mMappedCoords; 85 // These default source coordinates perform the necessary flip 86 // for converting from OpenGL origin to MFF/Bitmap origin. 87 private static final float[] mSourceCoords = { 0, 1, 0, 1, 88 1, 1, 0, 1, 89 0, 0, 0, 1, 90 1, 0, 0, 1 }; 91 92 private static final int NEWFRAME_TIMEOUT = 100; //ms 93 private static final int NEWFRAME_TIMEOUT_REPEAT = 10; 94 95 private boolean mNewFrameAvailable; 96 97 private Camera.Parameters mCameraParameters; 98 99 private static final String mFrameShader = 100 "#extension GL_OES_EGL_image_external : require\n" + 101 "precision mediump float;\n" + 102 "uniform samplerExternalOES tex_sampler_0;\n" + 103 "varying vec2 v_texcoord;\n" + 104 "void main() {\n" + 105 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 106 "}\n"; 107 108 private final boolean mLogVerbose; 109 private static final String TAG = "CameraSource"; 110 111 public CameraSource(String name) { 112 super(name); 113 mCameraTransform = new float[16]; 114 mMappedCoords = new float[16]; 115 116 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 117 } 118 119 @Override 120 public void setupPorts() { 121 // Add input port 122 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 123 FrameFormat.TARGET_GPU)); 124 } 125 126 private void createFormats() { 127 mOutputFormat = ImageFormat.create(mWidth, mHeight, 128 ImageFormat.COLORSPACE_RGBA, 129 FrameFormat.TARGET_GPU); 130 } 131 132 @Override 133 public void prepare(FilterContext context) { 134 if (mLogVerbose) Log.v(TAG, "Preparing"); 135 // Compile shader TODO: Move to onGLEnvSomething? 136 mFrameExtractor = new ShaderProgram(context, mFrameShader); 137 } 138 139 @Override 140 public void open(FilterContext context) { 141 if (mLogVerbose) Log.v(TAG, "Opening"); 142 // Open camera 143 mCamera = Camera.open(mCameraId); 144 145 // Set parameters 146 getCameraParameters(); 147 mCamera.setParameters(mCameraParameters); 148 149 // Create frame formats 150 createFormats(); 151 152 // Bind it to our camera frame 153 mCameraFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, 154 GLFrame.EXTERNAL_TEXTURE, 155 0); 156 mSurfaceTexture = new SurfaceTexture(mCameraFrame.getTextureId()); 157 try { 158 mCamera.setPreviewTexture(mSurfaceTexture); 159 } catch (IOException e) { 160 throw new RuntimeException("Could not bind camera surface texture: " + 161 e.getMessage() + "!"); 162 } 163 164 // Connect SurfaceTexture to callback 165 mSurfaceTexture.setOnFrameAvailableListener(onCameraFrameAvailableListener); 166 // Start the preview 167 mNewFrameAvailable = false; 168 mCamera.startPreview(); 169 } 170 171 @Override 172 public void process(FilterContext context) { 173 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 174 175 if (mWaitForNewFrame) { 176 int waitCount = 0; 177 while (!mNewFrameAvailable) { 178 if (waitCount == NEWFRAME_TIMEOUT_REPEAT) { 179 throw new RuntimeException("Timeout waiting for new frame"); 180 } 181 try { 182 this.wait(NEWFRAME_TIMEOUT); 183 } catch (InterruptedException e) { 184 if (mLogVerbose) Log.v(TAG, "Interrupted while waiting for new frame"); 185 } 186 } 187 mNewFrameAvailable = false; 188 if (mLogVerbose) Log.v(TAG, "Got new frame"); 189 } 190 191 mSurfaceTexture.updateTexImage(); 192 193 if (mLogVerbose) Log.v(TAG, "Using frame extractor in thread: " + Thread.currentThread()); 194 mSurfaceTexture.getTransformMatrix(mCameraTransform); 195 Matrix.multiplyMM(mMappedCoords, 0, 196 mCameraTransform, 0, 197 mSourceCoords, 0); 198 mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], 199 mMappedCoords[4], mMappedCoords[5], 200 mMappedCoords[8], mMappedCoords[9], 201 mMappedCoords[12], mMappedCoords[13]); 202 203 Frame output = context.getFrameManager().newFrame(mOutputFormat); 204 mFrameExtractor.process(mCameraFrame, output); 205 206 long timestamp = mSurfaceTexture.getTimestamp(); 207 if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s"); 208 output.setTimestamp(timestamp); 209 210 pushOutput("video", output); 211 212 // Release pushed frame 213 output.release(); 214 215 if (mLogVerbose) Log.v(TAG, "Done processing new frame"); 216 } 217 218 @Override 219 public void close(FilterContext context) { 220 if (mLogVerbose) Log.v(TAG, "Closing"); 221 222 mCamera.release(); 223 mCamera = null; 224 mSurfaceTexture.release(); 225 mSurfaceTexture = null; 226 } 227 228 @Override 229 public void tearDown(FilterContext context) { 230 if (mCameraFrame != null) { 231 mCameraFrame.release(); 232 } 233 } 234 235 @Override 236 public void fieldPortValueUpdated(String name, FilterContext context) { 237 if (name.equals("framerate")) { 238 getCameraParameters(); 239 int closestRange[] = findClosestFpsRange(mFps, mCameraParameters); 240 mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 241 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 242 mCamera.setParameters(mCameraParameters); 243 } 244 } 245 246 synchronized public Camera.Parameters getCameraParameters() { 247 boolean closeCamera = false; 248 if (mCameraParameters == null) { 249 if (mCamera == null) { 250 mCamera = Camera.open(mCameraId); 251 closeCamera = true; 252 } 253 mCameraParameters = mCamera.getParameters(); 254 255 if (closeCamera) { 256 mCamera.release(); 257 mCamera = null; 258 } 259 } 260 261 int closestSize[] = findClosestSize(mWidth, mHeight, mCameraParameters); 262 mWidth = closestSize[0]; 263 mHeight = closestSize[1]; 264 mCameraParameters.setPreviewSize(mWidth, mHeight); 265 266 int closestRange[] = findClosestFpsRange(mFps, mCameraParameters); 267 268 mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 269 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 270 271 return mCameraParameters; 272 } 273 274 /** Update camera parameters. Image resolution cannot be changed. */ 275 synchronized public void setCameraParameters(Camera.Parameters params) { 276 params.setPreviewSize(mWidth, mHeight); 277 mCameraParameters = params; 278 if (isOpen()) { 279 mCamera.setParameters(mCameraParameters); 280 } 281 } 282 283 private int[] findClosestSize(int width, int height, Camera.Parameters parameters) { 284 List<Camera.Size> previewSizes = parameters.getSupportedPreviewSizes(); 285 int closestWidth = -1; 286 int closestHeight = -1; 287 int smallestWidth = previewSizes.get(0).width; 288 int smallestHeight = previewSizes.get(0).height; 289 for (Camera.Size size : previewSizes) { 290 // Best match defined as not being larger in either dimension than 291 // the requested size, but as close as possible. The below isn't a 292 // stable selection (reording the size list can give different 293 // results), but since this is a fallback nicety, that's acceptable. 294 if ( size.width <= width && 295 size.height <= height && 296 size.width >= closestWidth && 297 size.height >= closestHeight) { 298 closestWidth = size.width; 299 closestHeight = size.height; 300 } 301 if ( size.width < smallestWidth && 302 size.height < smallestHeight) { 303 smallestWidth = size.width; 304 smallestHeight = size.height; 305 } 306 } 307 if (closestWidth == -1) { 308 // Requested size is smaller than any listed size; match with smallest possible 309 closestWidth = smallestWidth; 310 closestHeight = smallestHeight; 311 } 312 313 if (mLogVerbose) { 314 Log.v(TAG, 315 "Requested resolution: (" + width + ", " + height 316 + "). Closest match: (" + closestWidth + ", " 317 + closestHeight + ")."); 318 } 319 int[] closestSize = {closestWidth, closestHeight}; 320 return closestSize; 321 } 322 323 private int[] findClosestFpsRange(int fps, Camera.Parameters params) { 324 List<int[]> supportedFpsRanges = params.getSupportedPreviewFpsRange(); 325 int[] closestRange = supportedFpsRanges.get(0); 326 for (int[] range : supportedFpsRanges) { 327 if (range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] < fps*1000 && 328 range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] > fps*1000 && 329 range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] > 330 closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] && 331 range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] < 332 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]) { 333 closestRange = range; 334 } 335 } 336 if (mLogVerbose) Log.v(TAG, "Requested fps: " + fps 337 + ".Closest frame rate range: [" 338 + closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] / 1000. 339 + "," 340 + closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] / 1000. 341 + "]"); 342 343 return closestRange; 344 } 345 346 private SurfaceTexture.OnFrameAvailableListener onCameraFrameAvailableListener = 347 new SurfaceTexture.OnFrameAvailableListener() { 348 @Override 349 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 350 if (mLogVerbose) Log.v(TAG, "New frame from camera"); 351 synchronized(CameraSource.this) { 352 mNewFrameAvailable = true; 353 CameraSource.this.notify(); 354 } 355 } 356 }; 357 358 } 359
