1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.graphics; 18 19 import android.annotation.Nullable; 20 import android.os.Handler; 21 import android.os.Looper; 22 import android.os.Message; 23 import android.view.Surface; 24 25 import java.lang.ref.WeakReference; 26 27 /** 28 * Captures frames from an image stream as an OpenGL ES texture. 29 * 30 * <p>The image stream may come from either camera preview or video decode. A 31 * {@link android.view.Surface} created from a SurfaceTexture can be used as an output 32 * destination for the {@link android.hardware.camera2}, {@link android.media.MediaCodec}, 33 * {@link android.media.MediaPlayer}, and {@link android.renderscript.Allocation} APIs. 34 * When {@link #updateTexImage} is called, the contents of the texture object specified 35 * when the SurfaceTexture was created are updated to contain the most recent image from the image 36 * stream. This may cause some frames of the stream to be skipped. 37 * 38 * <p>A SurfaceTexture may also be used in place of a SurfaceHolder when specifying the output 39 * destination of the older {@link android.hardware.Camera} API. Doing so will cause all the 40 * frames from the image stream to be sent to the SurfaceTexture object rather than to the device's 41 * display. 42 * 43 * <p>When sampling from the texture one should first transform the texture coordinates using the 44 * matrix queried via {@link #getTransformMatrix(float[])}. The transform matrix may change each 45 * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image 46 * is updated. 47 * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s, 48 * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in 49 * the streamed texture. This transform compensates for any properties of the image stream source 50 * that cause it to appear different from a traditional OpenGL ES texture. For example, sampling 51 * from the bottom left corner of the image can be accomplished by transforming the column vector 52 * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can 53 * be done by transforming (1, 1, 0, 1). 54 * 55 * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the 56 * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt"> 57 * GL_OES_EGL_image_external</a> OpenGL ES extension. This limits how the texture may be used. 58 * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than 59 * the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture 60 * must declare its use of this extension using, for example, an "#extension 61 * GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using 62 * the samplerExternalOES GLSL sampler type. 63 * 64 * <p>SurfaceTexture objects may be created on any thread. {@link #updateTexImage} may only be 65 * called on the thread with the OpenGL ES context that contains the texture object. The 66 * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link 67 * #updateTexImage} should not be called directly from the callback. 68 */ 69 public class SurfaceTexture { 70 private final Looper mCreatorLooper; 71 private Handler mOnFrameAvailableHandler; 72 73 /** 74 * These fields are used by native code, do not access or modify. 75 */ 76 private long mSurfaceTexture; 77 private long mProducer; 78 private long mFrameAvailableListener; 79 80 private boolean mIsSingleBuffered; 81 82 /** 83 * Callback interface for being notified that a new stream frame is available. 84 */ 85 public interface OnFrameAvailableListener { 86 void onFrameAvailable(SurfaceTexture surfaceTexture); 87 } 88 89 /** 90 * Exception thrown when a SurfaceTexture couldn't be created or resized. 91 * 92 * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException} 93 * is used instead. 94 */ 95 @SuppressWarnings("serial") 96 @Deprecated 97 public static class OutOfResourcesException extends Exception { 98 public OutOfResourcesException() { 99 } 100 public OutOfResourcesException(String name) { 101 super(name); 102 } 103 } 104 105 /** 106 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 107 * 108 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 109 * 110 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 111 */ 112 public SurfaceTexture(int texName) { 113 this(texName, false); 114 } 115 116 /** 117 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 118 * 119 * In single buffered mode the application is responsible for serializing access to the image 120 * content buffer. Each time the image content is to be updated, the 121 * {@link #releaseTexImage()} method must be called before the image content producer takes 122 * ownership of the buffer. For example, when producing image content with the NDK 123 * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()} 124 * must be called before each ANativeWindow_lock, or that call will fail. When producing 125 * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first 126 * OpenGL ES function call each frame. 127 * 128 * @param texName the OpenGL texture object name (e.g. generated via glGenTextures) 129 * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode. 130 * 131 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 132 */ 133 public SurfaceTexture(int texName, boolean singleBufferMode) { 134 mCreatorLooper = Looper.myLooper(); 135 mIsSingleBuffered = singleBufferMode; 136 nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this)); 137 } 138 139 /** 140 * Construct a new SurfaceTexture to stream images to a given OpenGL texture. 141 * 142 * In single buffered mode the application is responsible for serializing access to the image 143 * content buffer. Each time the image content is to be updated, the 144 * {@link #releaseTexImage()} method must be called before the image content producer takes 145 * ownership of the buffer. For example, when producing image content with the NDK 146 * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()} 147 * must be called before each ANativeWindow_lock, or that call will fail. When producing 148 * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first 149 * OpenGL ES function call each frame. 150 * 151 * Unlike {@link #SurfaceTexture(int, boolean)}, which takes an OpenGL texture object name, 152 * this constructor creates the SurfaceTexture in detached mode. A texture name must be passed 153 * in using {@link #attachToGLContext} before calling {@link #releaseTexImage()} and producing 154 * image content using OpenGL ES. 155 * 156 * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode. 157 * 158 * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created. 159 */ 160 public SurfaceTexture(boolean singleBufferMode) { 161 mCreatorLooper = Looper.myLooper(); 162 mIsSingleBuffered = singleBufferMode; 163 nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this)); 164 } 165 166 /** 167 * Register a callback to be invoked when a new image frame becomes available to the 168 * SurfaceTexture. 169 * <p> 170 * The callback may be called on an arbitrary thread, so it is not 171 * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the 172 * thread invoking the callback. 173 * </p> 174 * 175 * @param listener The listener to use, or null to remove the listener. 176 */ 177 public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) { 178 setOnFrameAvailableListener(listener, null); 179 } 180 181 /** 182 * Register a callback to be invoked when a new image frame becomes available to the 183 * SurfaceTexture. 184 * <p> 185 * If a handler is specified, the callback will be invoked on that handler's thread. 186 * If no handler is specified, then the callback may be called on an arbitrary thread, 187 * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES 188 * context to the thread invoking the callback. 189 * </p> 190 * 191 * @param listener The listener to use, or null to remove the listener. 192 * @param handler The handler on which the listener should be invoked, or null 193 * to use an arbitrary thread. 194 */ 195 public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener, 196 @Nullable Handler handler) { 197 if (listener != null) { 198 // Although we claim the thread is arbitrary, earlier implementation would 199 // prefer to send the callback on the creating looper or the main looper 200 // so we preserve this behavior here. 201 Looper looper = handler != null ? handler.getLooper() : 202 mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper(); 203 mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) { 204 @Override 205 public void handleMessage(Message msg) { 206 listener.onFrameAvailable(SurfaceTexture.this); 207 } 208 }; 209 } else { 210 mOnFrameAvailableHandler = null; 211 } 212 } 213 214 /** 215 * Set the default size of the image buffers. The image producer may override the buffer size, 216 * in which case the producer-set buffer size will be used, not the default size set by this 217 * method. Both video and camera based image producers do override the size. This method may 218 * be used to set the image size when producing images with {@link android.graphics.Canvas} (via 219 * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface). 220 * 221 * The new default buffer size will take effect the next time the image producer requests a 222 * buffer to fill. For {@link android.graphics.Canvas} this will be the next time {@link 223 * android.view.Surface#lockCanvas} is called. For OpenGL ES, the EGLSurface should be 224 * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated 225 * (via eglCreateWindowSurface) to ensure that the new default size has taken effect. 226 * 227 * The width and height parameters must be no greater than the minimum of 228 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see 229 * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}). 230 * An error due to invalid dimensions might not be reported until 231 * updateTexImage() is called. 232 */ 233 public void setDefaultBufferSize(int width, int height) { 234 nativeSetDefaultBufferSize(width, height); 235 } 236 237 /** 238 * Update the texture image to the most recent frame from the image stream. This may only be 239 * called while the OpenGL ES context that owns the texture is current on the calling thread. 240 * It will implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target. 241 */ 242 public void updateTexImage() { 243 nativeUpdateTexImage(); 244 } 245 246 /** 247 * Releases the the texture content. This is needed in single buffered mode to allow the image 248 * content producer to take ownership of the image buffer. 249 * For more information see {@link #SurfaceTexture(int, boolean)}. 250 */ 251 public void releaseTexImage() { 252 nativeReleaseTexImage(); 253 } 254 255 /** 256 * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object. 257 * This call must be made with the OpenGL ES context current on the calling thread. The OpenGL 258 * ES texture object will be deleted as a result of this call. After calling this method all 259 * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until 260 * a successful call to {@link #attachToGLContext} is made. 261 * 262 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 263 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 264 * context at a time. 265 */ 266 public void detachFromGLContext() { 267 int err = nativeDetachFromGLContext(); 268 if (err != 0) { 269 throw new RuntimeException("Error during detachFromGLContext (see logcat for details)"); 270 } 271 } 272 273 /** 274 * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread. A 275 * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame 276 * that was current at the time of the last call to {@link #detachFromGLContext}. This new 277 * texture is bound to the GL_TEXTURE_EXTERNAL_OES texture target. 278 * 279 * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES 280 * contexts. Note, however, that the image contents are only accessible from one OpenGL ES 281 * context at a time. 282 * 283 * @param texName The name of the OpenGL ES texture that will be created. This texture name 284 * must be unusued in the OpenGL ES context that is current on the calling thread. 285 */ 286 public void attachToGLContext(int texName) { 287 int err = nativeAttachToGLContext(texName); 288 if (err != 0) { 289 throw new RuntimeException("Error during attachToGLContext (see logcat for details)"); 290 } 291 } 292 293 /** 294 * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by 295 * the most recent call to updateTexImage. 296 * 297 * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s 298 * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample 299 * that location from the texture. Sampling the texture outside of the range of this transform 300 * is undefined. 301 * 302 * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via 303 * the glLoadMatrixf or glUniformMatrix4fv functions. 304 * 305 * @param mtx the array into which the 4x4 matrix will be stored. The array must have exactly 306 * 16 elements. 307 */ 308 public void getTransformMatrix(float[] mtx) { 309 // Note we intentionally don't check mtx for null, so this will result in a 310 // NullPointerException. But it's safe because it happens before the call to native. 311 if (mtx.length != 16) { 312 throw new IllegalArgumentException(); 313 } 314 nativeGetTransformMatrix(mtx); 315 } 316 317 /** 318 * Retrieve the timestamp associated with the texture image set by the most recent call to 319 * updateTexImage. 320 * 321 * <p>This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp 322 * should be unaffected by time-of-day adjustments. The specific meaning and zero point of the 323 * timestamp depends on the source providing images to the SurfaceTexture. Unless otherwise 324 * specified by the image source, timestamps cannot generally be compared across SurfaceTexture 325 * instances, or across multiple program invocations. It is mostly useful for determining time 326 * offsets between subsequent frames.</p> 327 * 328 * <p>For camera sources, timestamps should be strictly monotonic. Timestamps from MediaPlayer 329 * sources may be reset when the playback position is set. For EGL and Vulkan producers, the 330 * timestamp is the desired present time set with the EGL_ANDROID_presentation_time or 331 * VK_GOOGLE_display_timing extensions.</p> 332 */ 333 334 public long getTimestamp() { 335 return nativeGetTimestamp(); 336 } 337 338 /** 339 * release() frees all the buffers and puts the SurfaceTexture into the 340 * 'abandoned' state. Once put in this state the SurfaceTexture can never 341 * leave it. When in the 'abandoned' state, all methods of the 342 * IGraphicBufferProducer interface will fail with the NO_INIT error. 343 * 344 * Note that while calling this method causes all the buffers to be freed 345 * from the perspective of the the SurfaceTexture, if there are additional 346 * references on the buffers (e.g. if a buffer is referenced by a client or 347 * by OpenGL ES as a texture) then those buffer will remain allocated. 348 * 349 * Always call this method when you are done with SurfaceTexture. Failing 350 * to do so may delay resource deallocation for a significant amount of 351 * time. 352 * 353 * @see #isReleased() 354 */ 355 public void release() { 356 nativeRelease(); 357 } 358 359 /** 360 * Returns true if the SurfaceTexture was released. 361 * 362 * @see #release() 363 */ 364 public boolean isReleased() { 365 return nativeIsReleased(); 366 } 367 368 @Override 369 protected void finalize() throws Throwable { 370 try { 371 nativeFinalize(); 372 } finally { 373 super.finalize(); 374 } 375 } 376 377 /** 378 * This method is invoked from native code only. 379 */ 380 @SuppressWarnings({"UnusedDeclaration"}) 381 private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) { 382 SurfaceTexture st = weakSelf.get(); 383 if (st != null) { 384 Handler handler = st.mOnFrameAvailableHandler; 385 if (handler != null) { 386 handler.sendEmptyMessage(0); 387 } 388 } 389 } 390 391 /** 392 * Returns true if the SurfaceTexture is single-buffered 393 * @hide 394 */ 395 public boolean isSingleBuffered() { 396 return mIsSingleBuffered; 397 } 398 399 private native void nativeInit(boolean isDetached, int texName, 400 boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf) 401 throws Surface.OutOfResourcesException; 402 private native void nativeFinalize(); 403 private native void nativeGetTransformMatrix(float[] mtx); 404 private native long nativeGetTimestamp(); 405 private native void nativeSetDefaultBufferSize(int width, int height); 406 private native void nativeUpdateTexImage(); 407 private native void nativeReleaseTexImage(); 408 private native int nativeDetachFromGLContext(); 409 private native int nativeAttachToGLContext(int texName); 410 private native void nativeRelease(); 411 private native boolean nativeIsReleased(); 412 } 413
