1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 18 package android.filterpacks.videosrc; 19 20 import android.filterfw.core.Filter; 21 import android.filterfw.core.FilterContext; 22 import android.filterfw.core.Frame; 23 import android.filterfw.core.FrameFormat; 24 import android.filterfw.core.GenerateFieldPort; 25 import android.filterfw.core.GenerateFinalPort; 26 import android.filterfw.core.GLFrame; 27 import android.filterfw.core.MutableFrameFormat; 28 import android.filterfw.core.ShaderProgram; 29 import android.filterfw.format.ImageFormat; 30 import android.graphics.SurfaceTexture; 31 import android.hardware.Camera; 32 import android.opengl.Matrix; 33 34 import java.io.IOException; 35 import java.util.List; 36 37 import android.util.Log; 38 39 /** 40 * @hide 41 */ 42 public class CameraSource extends Filter { 43 44 /** User-visible parameters */ 45 46 /** Camera ID to use for input. Defaults to 0. */ 47 @GenerateFieldPort(name = "id", hasDefault = true) 48 private int mCameraId = 0; 49 50 /** Frame width to request from camera. Actual size may not match requested. */ 51 @GenerateFieldPort(name = "width", hasDefault = true) 52 private int mWidth = 320; 53 54 /** Frame height to request from camera. Actual size may not match requested. */ 55 @GenerateFieldPort(name = "height", hasDefault = true) 56 private int mHeight = 240; 57 58 /** Stream framerate to request from camera. Actual frame rate may not match requested. */ 59 @GenerateFieldPort(name = "framerate", hasDefault = true) 60 private int mFps = 30; 61 62 /** Whether the filter should always wait for a new frame from the camera 63 * before providing output. If set to false, the filter will keep 64 * outputting the last frame it received from the camera if multiple process 65 * calls are received before the next update from the Camera. Defaults to true. 66 */ 67 @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true) 68 private boolean mWaitForNewFrame = true; 69 70 private Camera mCamera; 71 private GLFrame mCameraFrame; 72 private SurfaceTexture mSurfaceTexture; 73 private ShaderProgram mFrameExtractor; 74 private MutableFrameFormat mOutputFormat; 75 76 private float[] mCameraTransform; 77 private float[] mMappedCoords; 78 // These default source coordinates perform the necessary flip 79 // for converting from OpenGL origin to MFF/Bitmap origin. 80 private static final float[] mSourceCoords = { 0, 1, 0, 1, 81 1, 1, 0, 1, 82 0, 0, 0, 1, 83 1, 0, 0, 1 }; 84 85 private static final int NEWFRAME_TIMEOUT = 100; //ms 86 private static final int NEWFRAME_TIMEOUT_REPEAT = 10; 87 88 private boolean mNewFrameAvailable; 89 90 private Camera.Parameters mCameraParameters; 91 92 private static final String mFrameShader = 93 "#extension GL_OES_EGL_image_external : require\n" + 94 "precision mediump float;\n" + 95 "uniform samplerExternalOES tex_sampler_0;\n" + 96 "varying vec2 v_texcoord;\n" + 97 "void main() {\n" + 98 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 99 "}\n"; 100 101 private final boolean mLogVerbose; 102 private static final String TAG = "CameraSource"; 103 104 public CameraSource(String name) { 105 super(name); 106 mCameraTransform = new float[16]; 107 mMappedCoords = new float[16]; 108 109 mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 110 } 111 112 @Override 113 public void setupPorts() { 114 // Add input port 115 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 116 FrameFormat.TARGET_GPU)); 117 } 118 119 private void createFormats() { 120 mOutputFormat = ImageFormat.create(mWidth, mHeight, 121 ImageFormat.COLORSPACE_RGBA, 122 FrameFormat.TARGET_GPU); 123 } 124 125 @Override 126 public void prepare(FilterContext context) { 127 if (mLogVerbose) Log.v(TAG, "Preparing"); 128 // Compile shader TODO: Move to onGLEnvSomething? 129 mFrameExtractor = new ShaderProgram(context, mFrameShader); 130 } 131 132 @Override 133 public void open(FilterContext context) { 134 if (mLogVerbose) Log.v(TAG, "Opening"); 135 // Open camera 136 mCamera = Camera.open(mCameraId); 137 138 // Set parameters 139 getCameraParameters(); 140 mCamera.setParameters(mCameraParameters); 141 142 // Create frame formats 143 createFormats(); 144 145 // Bind it to our camera frame 146 mCameraFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, 147 GLFrame.EXTERNAL_TEXTURE, 148 0); 149 mSurfaceTexture = new SurfaceTexture(mCameraFrame.getTextureId()); 150 try { 151 mCamera.setPreviewTexture(mSurfaceTexture); 152 } catch (IOException e) { 153 throw new RuntimeException("Could not bind camera surface texture: " + 154 e.getMessage() + "!"); 155 } 156 157 // Connect SurfaceTexture to callback 158 mSurfaceTexture.setOnFrameAvailableListener(onCameraFrameAvailableListener); 159 // Start the preview 160 mNewFrameAvailable = false; 161 mCamera.startPreview(); 162 } 163 164 @Override 165 public void process(FilterContext context) { 166 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 167 168 if (mWaitForNewFrame) { 169 int waitCount = 0; 170 while (!mNewFrameAvailable) { 171 if (waitCount == NEWFRAME_TIMEOUT_REPEAT) { 172 throw new RuntimeException("Timeout waiting for new frame"); 173 } 174 try { 175 this.wait(NEWFRAME_TIMEOUT); 176 } catch (InterruptedException e) { 177 if (mLogVerbose) Log.v(TAG, "Interrupted while waiting for new frame"); 178 } 179 } 180 mNewFrameAvailable = false; 181 if (mLogVerbose) Log.v(TAG, "Got new frame"); 182 } 183 184 mSurfaceTexture.updateTexImage(); 185 186 if (mLogVerbose) Log.v(TAG, "Using frame extractor in thread: " + Thread.currentThread()); 187 mSurfaceTexture.getTransformMatrix(mCameraTransform); 188 Matrix.multiplyMM(mMappedCoords, 0, 189 mCameraTransform, 0, 190 mSourceCoords, 0); 191 mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], 192 mMappedCoords[4], mMappedCoords[5], 193 mMappedCoords[8], mMappedCoords[9], 194 mMappedCoords[12], mMappedCoords[13]); 195 196 Frame output = context.getFrameManager().newFrame(mOutputFormat); 197 mFrameExtractor.process(mCameraFrame, output); 198 199 long timestamp = mSurfaceTexture.getTimestamp(); 200 if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s"); 201 output.setTimestamp(timestamp); 202 203 pushOutput("video", output); 204 205 // Release pushed frame 206 output.release(); 207 208 if (mLogVerbose) Log.v(TAG, "Done processing new frame"); 209 } 210 211 @Override 212 public void close(FilterContext context) { 213 if (mLogVerbose) Log.v(TAG, "Closing"); 214 215 mCamera.release(); 216 mCamera = null; 217 mSurfaceTexture.release(); 218 mSurfaceTexture = null; 219 } 220 221 @Override 222 public void tearDown(FilterContext context) { 223 if (mCameraFrame != null) { 224 mCameraFrame.release(); 225 } 226 } 227 228 @Override 229 public void fieldPortValueUpdated(String name, FilterContext context) { 230 if (name.equals("framerate")) { 231 getCameraParameters(); 232 int closestRange[] = findClosestFpsRange(mFps, mCameraParameters); 233 mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 234 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 235 mCamera.setParameters(mCameraParameters); 236 } 237 } 238 239 synchronized public Camera.Parameters getCameraParameters() { 240 boolean closeCamera = false; 241 if (mCameraParameters == null) { 242 if (mCamera == null) { 243 mCamera = Camera.open(mCameraId); 244 closeCamera = true; 245 } 246 mCameraParameters = mCamera.getParameters(); 247 248 if (closeCamera) { 249 mCamera.release(); 250 mCamera = null; 251 } 252 } 253 254 int closestSize[] = findClosestSize(mWidth, mHeight, mCameraParameters); 255 mWidth = closestSize[0]; 256 mHeight = closestSize[1]; 257 mCameraParameters.setPreviewSize(mWidth, mHeight); 258 259 int closestRange[] = findClosestFpsRange(mFps, mCameraParameters); 260 261 mCameraParameters.setPreviewFpsRange(closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 262 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 263 264 return mCameraParameters; 265 } 266 267 /** Update camera parameters. Image resolution cannot be changed. */ 268 synchronized public void setCameraParameters(Camera.Parameters params) { 269 params.setPreviewSize(mWidth, mHeight); 270 mCameraParameters = params; 271 if (isOpen()) { 272 mCamera.setParameters(mCameraParameters); 273 } 274 } 275 276 private int[] findClosestSize(int width, int height, Camera.Parameters parameters) { 277 List<Camera.Size> previewSizes = parameters.getSupportedPreviewSizes(); 278 int closestWidth = -1; 279 int closestHeight = -1; 280 int smallestWidth = previewSizes.get(0).width; 281 int smallestHeight = previewSizes.get(0).height; 282 for (Camera.Size size : previewSizes) { 283 // Best match defined as not being larger in either dimension than 284 // the requested size, but as close as possible. The below isn't a 285 // stable selection (reording the size list can give different 286 // results), but since this is a fallback nicety, that's acceptable. 287 if ( size.width <= width && 288 size.height <= height && 289 size.width >= closestWidth && 290 size.height >= closestHeight) { 291 closestWidth = size.width; 292 closestHeight = size.height; 293 } 294 if ( size.width < smallestWidth && 295 size.height < smallestHeight) { 296 smallestWidth = size.width; 297 smallestHeight = size.height; 298 } 299 } 300 if (closestWidth == -1) { 301 // Requested size is smaller than any listed size; match with smallest possible 302 closestWidth = smallestWidth; 303 closestHeight = smallestHeight; 304 } 305 306 if (mLogVerbose) { 307 Log.v(TAG, 308 "Requested resolution: (" + width + ", " + height 309 + "). Closest match: (" + closestWidth + ", " 310 + closestHeight + ")."); 311 } 312 int[] closestSize = {closestWidth, closestHeight}; 313 return closestSize; 314 } 315 316 private int[] findClosestFpsRange(int fps, Camera.Parameters params) { 317 List<int[]> supportedFpsRanges = params.getSupportedPreviewFpsRange(); 318 int[] closestRange = supportedFpsRanges.get(0); 319 for (int[] range : supportedFpsRanges) { 320 if (range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] < fps*1000 && 321 range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] > fps*1000 && 322 range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] > 323 closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] && 324 range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] < 325 closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]) { 326 closestRange = range; 327 } 328 } 329 if (mLogVerbose) Log.v(TAG, "Requested fps: " + fps 330 + ".Closest frame rate range: [" 331 + closestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] / 1000. 332 + "," 333 + closestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] / 1000. 334 + "]"); 335 336 return closestRange; 337 } 338 339 private SurfaceTexture.OnFrameAvailableListener onCameraFrameAvailableListener = 340 new SurfaceTexture.OnFrameAvailableListener() { 341 @Override 342 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 343 if (mLogVerbose) Log.v(TAG, "New frame from camera"); 344 synchronized(CameraSource.this) { 345 mNewFrameAvailable = true; 346 CameraSource.this.notify(); 347 } 348 } 349 }; 350 351 } 352
