1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.filterpacks.videosrc; 18 19 import android.content.Context; 20 import android.content.res.AssetFileDescriptor; 21 import android.filterfw.core.Filter; 22 import android.filterfw.core.FilterContext; 23 import android.filterfw.core.Frame; 24 import android.filterfw.core.FrameFormat; 25 import android.filterfw.core.FrameManager; 26 import android.filterfw.core.GenerateFieldPort; 27 import android.filterfw.core.GenerateFinalPort; 28 import android.filterfw.core.GLFrame; 29 import android.filterfw.core.KeyValueMap; 30 import android.filterfw.core.MutableFrameFormat; 31 import android.filterfw.core.ShaderProgram; 32 import android.filterfw.format.ImageFormat; 33 import android.graphics.SurfaceTexture; 34 import android.media.MediaPlayer; 35 import android.os.ConditionVariable; 36 import android.opengl.Matrix; 37 38 import java.io.IOException; 39 import java.io.FileDescriptor; 40 import java.lang.IllegalArgumentException; 41 import java.util.List; 42 import java.util.Set; 43 44 import android.util.Log; 45 46 /** <p>A filter that converts textures from a SurfaceTexture object into frames for 47 * processing in the filter framework.</p> 48 * 49 * <p>To use, connect up the sourceListener callback, and then when executing 50 * the graph, use the SurfaceTexture object passed to the callback to feed 51 * frames into the filter graph. For example, pass the SurfaceTexture into 52 * {#link 53 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}. 54 * This filter is intended for applications that need for flexibility than the 55 * CameraSource and MediaSource provide. Note that the application needs to 56 * provide width and height information for the SurfaceTextureSource, which it 57 * should obtain from wherever the SurfaceTexture data is coming from to avoid 58 * unnecessary resampling.</p> 59 * 60 * @hide 61 */ 62 public class SurfaceTextureSource extends Filter { 63 64 /** User-visible parameters */ 65 66 /** The callback interface for the sourceListener parameter */ 67 public interface SurfaceTextureSourceListener { 68 public void onSurfaceTextureSourceReady(SurfaceTexture source); 69 } 70 /** A callback to send the internal SurfaceTexture object to, once it is 71 * created. This callback will be called when the the filter graph is 72 * preparing to execute, but before any processing has actually taken 73 * place. The SurfaceTexture object passed to this callback is the only way 74 * to feed this filter. When the filter graph is shutting down, this 75 * callback will be called again with null as the source. 76 * 77 * This callback may be called from an arbitrary thread, so it should not 78 * assume it is running in the UI thread in particular. 79 */ 80 @GenerateFinalPort(name = "sourceListener") 81 private SurfaceTextureSourceListener mSourceListener; 82 83 /** The width of the output image frame. If the texture width for the 84 * SurfaceTexture source is known, use it here to minimize resampling. */ 85 @GenerateFieldPort(name = "width") 86 private int mWidth; 87 88 /** The height of the output image frame. If the texture height for the 89 * SurfaceTexture source is known, use it here to minimize resampling. */ 90 @GenerateFieldPort(name = "height") 91 private int mHeight; 92 93 /** Whether the filter will always wait for a new frame from its 94 * SurfaceTexture, or whether it will output an old frame again if a new 95 * frame isn't available. The filter will always wait for the first frame, 96 * to avoid outputting a blank frame. Defaults to true. 97 */ 98 @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true) 99 private boolean mWaitForNewFrame = true; 100 101 /** Maximum timeout before signaling error when waiting for a new frame. Set 102 * this to zero to disable the timeout and wait indefinitely. In milliseconds. 103 */ 104 @GenerateFieldPort(name = "waitTimeout", hasDefault = true) 105 private int mWaitTimeout = 1000; 106 107 /** Whether a timeout is an exception-causing failure, or just causes the 108 * filter to close. 109 */ 110 @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true) 111 private boolean mCloseOnTimeout = false; 112 113 // Variables for input->output conversion 114 private GLFrame mMediaFrame; 115 private ShaderProgram mFrameExtractor; 116 private SurfaceTexture mSurfaceTexture; 117 private MutableFrameFormat mOutputFormat; 118 private ConditionVariable mNewFrameAvailable; 119 private boolean mFirstFrame; 120 121 private float[] mFrameTransform; 122 private float[] mMappedCoords; 123 // These default source coordinates perform the necessary flip 124 // for converting from MFF/Bitmap origin to OpenGL origin. 125 private static final float[] mSourceCoords = { 0, 1, 0, 1, 126 1, 1, 0, 1, 127 0, 0, 0, 1, 128 1, 0, 0, 1 }; 129 // Shader for output 130 private final String mRenderShader = 131 "#extension GL_OES_EGL_image_external : require\n" + 132 "precision mediump float;\n" + 133 "uniform samplerExternalOES tex_sampler_0;\n" + 134 "varying vec2 v_texcoord;\n" + 135 "void main() {\n" + 136 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + 137 "}\n"; 138 139 // Variables for logging 140 141 private static final String TAG = "SurfaceTextureSource"; 142 private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 143 144 public SurfaceTextureSource(String name) { 145 super(name); 146 mNewFrameAvailable = new ConditionVariable(); 147 mFrameTransform = new float[16]; 148 mMappedCoords = new float[16]; 149 } 150 151 @Override 152 public void setupPorts() { 153 // Add input port 154 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA, 155 FrameFormat.TARGET_GPU)); 156 } 157 158 private void createFormats() { 159 mOutputFormat = ImageFormat.create(mWidth, mHeight, 160 ImageFormat.COLORSPACE_RGBA, 161 FrameFormat.TARGET_GPU); 162 } 163 164 @Override 165 protected void prepare(FilterContext context) { 166 if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource"); 167 168 createFormats(); 169 170 // Prepare input 171 mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat, 172 GLFrame.EXTERNAL_TEXTURE, 173 0); 174 175 // Prepare output 176 mFrameExtractor = new ShaderProgram(context, mRenderShader); 177 } 178 179 @Override 180 public void open(FilterContext context) { 181 if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource"); 182 // Create SurfaceTexture anew each time - it can use substantial memory. 183 mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId()); 184 // Connect SurfaceTexture to callback 185 mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener); 186 // Connect SurfaceTexture to source 187 mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture); 188 mFirstFrame = true; 189 } 190 191 @Override 192 public void process(FilterContext context) { 193 if (mLogVerbose) Log.v(TAG, "Processing new frame"); 194 195 // First, get new frame if available 196 if (mWaitForNewFrame || mFirstFrame) { 197 boolean gotNewFrame; 198 if (mWaitTimeout != 0) { 199 gotNewFrame = mNewFrameAvailable.block(mWaitTimeout); 200 if (!gotNewFrame) { 201 if (!mCloseOnTimeout) { 202 throw new RuntimeException("Timeout waiting for new frame"); 203 } else { 204 if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing."); 205 closeOutputPort("video"); 206 return; 207 } 208 } 209 } else { 210 mNewFrameAvailable.block(); 211 } 212 mNewFrameAvailable.close(); 213 mFirstFrame = false; 214 } 215 216 mSurfaceTexture.updateTexImage(); 217 218 mSurfaceTexture.getTransformMatrix(mFrameTransform); 219 Matrix.multiplyMM(mMappedCoords, 0, 220 mFrameTransform, 0, 221 mSourceCoords, 0); 222 mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], 223 mMappedCoords[4], mMappedCoords[5], 224 mMappedCoords[8], mMappedCoords[9], 225 mMappedCoords[12], mMappedCoords[13]); 226 // Next, render to output 227 Frame output = context.getFrameManager().newFrame(mOutputFormat); 228 mFrameExtractor.process(mMediaFrame, output); 229 230 output.setTimestamp(mSurfaceTexture.getTimestamp()); 231 232 pushOutput("video", output); 233 output.release(); 234 } 235 236 @Override 237 public void close(FilterContext context) { 238 if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed"); 239 mSourceListener.onSurfaceTextureSourceReady(null); 240 mSurfaceTexture.release(); 241 mSurfaceTexture = null; 242 } 243 244 @Override 245 public void tearDown(FilterContext context) { 246 if (mMediaFrame != null) { 247 mMediaFrame.release(); 248 } 249 } 250 251 @Override 252 public void fieldPortValueUpdated(String name, FilterContext context) { 253 if (name.equals("width") || name.equals("height") ) { 254 mOutputFormat.setDimensions(mWidth, mHeight); 255 } 256 } 257 258 private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener = 259 new SurfaceTexture.OnFrameAvailableListener() { 260 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 261 if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTexture"); 262 mNewFrameAvailable.open(); 263 } 264 }; 265 } 266
