1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.filterpacks.videosrc; 18 19 import android.filterfw.core.Filter; 20 import android.filterfw.core.FilterContext; 21 import android.filterfw.core.Frame; 22 import android.filterfw.core.FrameFormat; 23 import android.filterfw.core.GenerateFieldPort; 24 import android.filterfw.core.GenerateFinalPort; 25 import android.filterfw.core.GLFrame; 26 import android.filterfw.core.MutableFrameFormat; 27 import android.filterfw.core.ShaderProgram; 28 import android.filterfw.format.ImageFormat; 29 import android.graphics.SurfaceTexture; 30 import android.os.ConditionVariable; 31 import android.opengl.Matrix; 32 33 import android.util.Log; 34 35 /** <p>A filter that converts textures from a SurfaceTexture object into frames for 36 * processing in the filter framework.</p> 37 * 38 * <p>To use, connect up the sourceListener callback, and then when executing 39 * the graph, use the SurfaceTexture object passed to the callback to feed 40 * frames into the filter graph. For example, pass the SurfaceTexture into 41 * {#link 42 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}. 43 * This filter is intended for applications that need for flexibility than the 44 * CameraSource and MediaSource provide. Note that the application needs to 45 * provide width and height information for the SurfaceTextureSource, which it 46 * should obtain from wherever the SurfaceTexture data is coming from to avoid 47 * unnecessary resampling.</p> 48 * 49 * @hide 50 */ 51 public class SurfaceTextureSource extends Filter { 52 53 /** User-visible parameters */ 54 55 /** The callback interface for the sourceListener parameter */ 56 public interface SurfaceTextureSourceListener { 57 public void onSurfaceTextureSourceReady(SurfaceTexture source); 58 } 59 /** A callback to send the internal SurfaceTexture object to, once it is 60 * created. This callback will be called when the the filter graph is 61 * preparing to execute, but before any processing has actually taken 62 * place. The SurfaceTexture object passed to this callback is the only way 63 * to feed this filter. When the filter graph is shutting down, this 64 * callback will be called again with null as the source. 65 * 66 * This callback may be called from an arbitrary thread, so it should not 67 * assume it is running in the UI thread in particular. 68 */ 69 @GenerateFinalPort(name = "sourceListener") 70 private SurfaceTextureSourceListener mSourceListener; 71 72 /** The width of the output image frame. If the texture width for the 73 * SurfaceTexture source is known, use it here to minimize resampling. */ 74 @GenerateFieldPort(name = "width") 75 private int mWidth; 76 77 /** The height of the output image frame. If the texture height for the 78 * SurfaceTexture source is known, use it here to minimize resampling. */ 79 @GenerateFieldPort(name = "height") 80 private int mHeight; 81 82 /** Whether the filter will always wait for a new frame from its 83 * SurfaceTexture, or whether it will output an old frame again if a new 84 * frame isn't available. The filter will always wait for the first frame, 85 * to avoid outputting a blank frame. Defaults to true. 86 */ 87 @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true) 88 private boolean mWaitForNewFrame = true; 89 90 /** Maximum timeout before signaling error when waiting for a new frame. Set 91 * this to zero to disable the timeout and wait indefinitely. In milliseconds. 92 */ 93 @GenerateFieldPort(name = "waitTimeout", hasDefault = true) 94 private int mWaitTimeout = 1000; 95 96 /** Whether a timeout is an exception-causing failure, or just causes the 97 * filter to close. 98 */ 99 @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true) 100 private boolean mCloseOnTimeout = false; 101 102 // Variables for input->output conversion 103 private GLFrame mMediaFrame; 104 private ShaderProgram mFrameExtractor; 105 private SurfaceTexture mSurfaceTexture; 106 private MutableFrameFormat mOutputFormat; 107 private ConditionVariable mNewFrameAvailable; 108 private boolean mFirstFrame; 109 110 private float[] mFrameTransform; 111 private float[] mMappedCoords; 112 // These default source coordinates perform the necessary flip 113 // for converting from MFF/Bitmap origin to OpenGL origin. 114 private static final float[] mSourceCoords = { 0, 1, 0, 1, 115 1, 1, 0, 1, 116 0, 0, 0, 1, 117 1, 0, 0, 1 }; 118 // Shader for output 119 private final String mRenderShader = 120 "#extension GL_OES_EGL_image_external : require\n" + 121 "precision mediump float;\n" + 122 "uniform samplerExternalOES tex_sampler_0;\n" + 123 "varying vec2 v_texcoord;\n" + 124 "void main() {\n" + 125 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 126 "}\n"; 127 128 // Variables for logging 129 130 private static final String TAG = "SurfaceTextureSource"; 131 private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 132 133 public SurfaceTextureSource(String name) { 134 super(name); 135 mNewFrameAvailable = new ConditionVariable(); 136 mFrameTransform = new float[16]; 137 mMappedCoords = new float[16]; 138 } 139 140 @Override 141 public void setupPorts() { 142 // Add input port 143 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 144 FrameFormat.TARGET_GPU)); 145 } 146 147 private void createFormats() { 148 mOutputFormat = ImageFormat.create(mWidth, mHeight, 149 ImageFormat.COLORSPACE_RGBA, 150 FrameFormat.TARGET_GPU); 151 } 152 153 @Override 154 protected void prepare(FilterContext context) { 155 if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource"); 156 157 createFormats(); 158 159 // Prepare input 160 mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, 161 GLFrame.EXTERNAL_TEXTURE, 162 0); 163 164 // Prepare output 165 mFrameExtractor = new ShaderProgram(context, mRenderShader); 166 } 167 168 @Override 169 public void open(FilterContext context) { 170 if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource"); 171 // Create SurfaceTexture anew each time - it can use substantial memory. 172 mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); 173 // Connect SurfaceTexture to callback 174 mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener); 175 // Connect SurfaceTexture to source 176 mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture); 177 mFirstFrame = true; 178 } 179 180 @Override 181 public void process(FilterContext context) { 182 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 183 184 // First, get new frame if available 185 if (mWaitForNewFrame || mFirstFrame) { 186 boolean gotNewFrame; 187 if (mWaitTimeout != 0) { 188 gotNewFrame = mNewFrameAvailable.block(mWaitTimeout); 189 if (!gotNewFrame) { 190 if (!mCloseOnTimeout) { 191 throw new RuntimeException("Timeout waiting for new frame"); 192 } else { 193 if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing."); 194 closeOutputPort("video"); 195 return; 196 } 197 } 198 } else { 199 mNewFrameAvailable.block(); 200 } 201 mNewFrameAvailable.close(); 202 mFirstFrame = false; 203 } 204 205 mSurfaceTexture.updateTexImage(); 206 207 mSurfaceTexture.getTransformMatrix(mFrameTransform); 208 Matrix.multiplyMM(mMappedCoords, 0, 209 mFrameTransform, 0, 210 mSourceCoords, 0); 211 mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], 212 mMappedCoords[4], mMappedCoords[5], 213 mMappedCoords[8], mMappedCoords[9], 214 mMappedCoords[12], mMappedCoords[13]); 215 // Next, render to output 216 Frame output = context.getFrameManager().newFrame(mOutputFormat); 217 mFrameExtractor.process(mMediaFrame, output); 218 219 output.setTimestamp(mSurfaceTexture.getTimestamp()); 220 221 pushOutput("video", output); 222 output.release(); 223 } 224 225 @Override 226 public void close(FilterContext context) { 227 if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed"); 228 mSourceListener.onSurfaceTextureSourceReady(null); 229 mSurfaceTexture.release(); 230 mSurfaceTexture = null; 231 } 232 233 @Override 234 public void tearDown(FilterContext context) { 235 if (mMediaFrame != null) { 236 mMediaFrame.release(); 237 } 238 } 239 240 @Override 241 public void fieldPortValueUpdated(String name, FilterContext context) { 242 if (name.equals("width") || name.equals("height") ) { 243 mOutputFormat.setDimensions(mWidth, mHeight); 244 } 245 } 246 247 private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener = 248 new SurfaceTexture.OnFrameAvailableListener() { 249 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 250 if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture"); 251 mNewFrameAvailable.open(); 252 } 253 }; 254 } 255
