Home | History | Annotate | Download | only in legacy 
      1 /*
      2  * Copyright (C) 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 package android.hardware.camera2.legacy;
     17 
     18 import android.graphics.ImageFormat;
     19 import android.graphics.RectF;
     20 import android.graphics.SurfaceTexture;
     21 import android.hardware.camera2.CameraCharacteristics;
     22 import android.os.Environment;
     23 import android.opengl.EGL14;
     24 import android.opengl.EGLConfig;
     25 import android.opengl.EGLContext;
     26 import android.opengl.EGLDisplay;
     27 import android.opengl.EGLSurface;
     28 import android.opengl.GLES11Ext;
     29 import android.opengl.GLES20;
     30 import android.opengl.Matrix;
     31 import android.text.format.Time;
     32 import android.util.Log;
     33 import android.util.Pair;
     34 import android.util.Size;
     35 import android.view.Surface;
     36 import android.os.SystemProperties;
     37 
     38 import java.io.File;
     39 import java.nio.ByteBuffer;
     40 import java.nio.ByteOrder;
     41 import java.nio.FloatBuffer;
     42 import java.util.ArrayList;
     43 import java.util.Collection;
     44 import java.util.List;
     45 
     46 /**
     47  * A renderer class that manages the GL state, and can draw a frame into a set of output
     48  * {@link Surface}s.
     49  */
     50 public class SurfaceTextureRenderer {
     51     private static final String TAG = SurfaceTextureRenderer.class.getSimpleName();
     52     private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG);
     53     private static final int EGL_RECORDABLE_ANDROID = 0x3142; // from EGL/eglext.h
     54     private static final int GL_MATRIX_SIZE = 16;
     55     private static final int VERTEX_POS_SIZE = 3;
     56     private static final int VERTEX_UV_SIZE = 2;
     57     private static final int EGL_COLOR_BITLENGTH = 8;
     58     private static final int GLES_VERSION = 2;
     59     private static final int PBUFFER_PIXEL_BYTES = 4;
     60 
     61     private static final int FLIP_TYPE_NONE = 0;
     62     private static final int FLIP_TYPE_HORIZONTAL = 1;
     63     private static final int FLIP_TYPE_VERTICAL = 2;
     64     private static final int FLIP_TYPE_BOTH = FLIP_TYPE_HORIZONTAL | FLIP_TYPE_VERTICAL;
     65 
     66     private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
     67     private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
     68     private EGLConfig mConfigs;
     69 
     70     private class EGLSurfaceHolder {
     71         Surface surface;
     72         EGLSurface eglSurface;
     73         int width;
     74         int height;
     75     }
     76 
     77     private List<EGLSurfaceHolder> mSurfaces = new ArrayList<EGLSurfaceHolder>();
     78     private List<EGLSurfaceHolder> mConversionSurfaces = new ArrayList<EGLSurfaceHolder>();
     79 
     80     private ByteBuffer mPBufferPixels;
     81 
     82     // Hold this to avoid GC
     83     private volatile SurfaceTexture mSurfaceTexture;
     84 
     85     private static final int FLOAT_SIZE_BYTES = 4;
     86     private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
     87     private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
     88     private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
     89 
     90     // Sampling is mirrored across the horizontal axis
     91     private static final float[] sHorizontalFlipTriangleVertices = {
     92             // X, Y, Z, U, V
     93             -1.0f, -1.0f, 0, 1.f, 0.f,
     94             1.0f, -1.0f, 0, 0.f, 0.f,
     95             -1.0f,  1.0f, 0, 1.f, 1.f,
     96             1.0f,  1.0f, 0, 0.f, 1.f,
     97     };
     98 
     99     // Sampling is mirrored across the vertical axis
    100     private static final float[] sVerticalFlipTriangleVertices = {
    101             // X, Y, Z, U, V
    102             -1.0f, -1.0f, 0, 0.f, 1.f,
    103             1.0f, -1.0f, 0, 1.f, 1.f,
    104             -1.0f,  1.0f, 0, 0.f, 0.f,
    105             1.0f,  1.0f, 0, 1.f, 0.f,
    106     };
    107 
    108     // Sampling is mirrored across the both axes
    109     private static final float[] sBothFlipTriangleVertices = {
    110             // X, Y, Z, U, V
    111             -1.0f, -1.0f, 0, 1.f, 1.f,
    112             1.0f, -1.0f, 0, 0.f, 1.f,
    113             -1.0f,  1.0f, 0, 1.f, 0.f,
    114             1.0f,  1.0f, 0, 0.f, 0.f,
    115     };
    116 
    117     // Sampling is 1:1 for a straight copy for the back camera
    118     private static final float[] sRegularTriangleVertices = {
    119             // X, Y, Z, U, V
    120             -1.0f, -1.0f, 0, 0.f, 0.f,
    121             1.0f, -1.0f, 0, 1.f, 0.f,
    122             -1.0f,  1.0f, 0, 0.f, 1.f,
    123             1.0f,  1.0f, 0, 1.f, 1.f,
    124     };
    125 
    126     private FloatBuffer mRegularTriangleVertices;
    127     private FloatBuffer mHorizontalFlipTriangleVertices;
    128     private FloatBuffer mVerticalFlipTriangleVertices;
    129     private FloatBuffer mBothFlipTriangleVertices;
    130     private final int mFacing;
    131 
    132     /**
    133      * As used in this file, this vertex shader maps a unit square to the view, and
    134      * tells the fragment shader to interpolate over it.  Each surface pixel position
    135      * is mapped to a 2D homogeneous texture coordinate of the form (s, t, 0, 1) with
    136      * s and t in the inclusive range [0, 1], and the matrix from
    137      * {@link SurfaceTexture#getTransformMatrix(float[])} is used to map this
    138      * coordinate to a texture location.
    139      */
    140     private static final String VERTEX_SHADER =
    141             "uniform mat4 uMVPMatrix;\n" +
    142             "uniform mat4 uSTMatrix;\n" +
    143             "attribute vec4 aPosition;\n" +
    144             "attribute vec4 aTextureCoord;\n" +
    145             "varying vec2 vTextureCoord;\n" +
    146             "void main() {\n" +
    147             "  gl_Position = uMVPMatrix * aPosition;\n" +
    148             "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
    149             "}\n";
    150 
    151     /**
    152      * This fragment shader simply draws the color in the 2D texture at
    153      * the location from the {@code VERTEX_SHADER}.
    154      */
    155     private static final String FRAGMENT_SHADER =
    156             "#extension GL_OES_EGL_image_external : require\n" +
    157             "precision mediump float;\n" +
    158             "varying vec2 vTextureCoord;\n" +
    159             "uniform samplerExternalOES sTexture;\n" +
    160             "void main() {\n" +
    161             "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
    162             "}\n";
    163 
    164     private float[] mMVPMatrix = new float[GL_MATRIX_SIZE];
    165     private float[] mSTMatrix = new float[GL_MATRIX_SIZE];
    166 
    167     private int mProgram;
    168     private int mTextureID = 0;
    169     private int muMVPMatrixHandle;
    170     private int muSTMatrixHandle;
    171     private int maPositionHandle;
    172     private int maTextureHandle;
    173 
    174     private PerfMeasurement mPerfMeasurer = null;
    175     private static final String LEGACY_PERF_PROPERTY = "persist.camera.legacy_perf";
    176 
    177     public SurfaceTextureRenderer(int facing) {
    178         mFacing = facing;
    179 
    180         mRegularTriangleVertices = ByteBuffer.allocateDirect(sRegularTriangleVertices.length *
    181                 FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
    182         mRegularTriangleVertices.put(sRegularTriangleVertices).position(0);
    183 
    184         mHorizontalFlipTriangleVertices = ByteBuffer.allocateDirect(
    185                 sHorizontalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    186                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    187         mHorizontalFlipTriangleVertices.put(sHorizontalFlipTriangleVertices).position(0);
    188 
    189         mVerticalFlipTriangleVertices = ByteBuffer.allocateDirect(
    190                 sVerticalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    191                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    192         mVerticalFlipTriangleVertices.put(sVerticalFlipTriangleVertices).position(0);
    193 
    194         mBothFlipTriangleVertices = ByteBuffer.allocateDirect(
    195                 sBothFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    196                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    197         mBothFlipTriangleVertices.put(sBothFlipTriangleVertices).position(0);
    198 
    199         Matrix.setIdentityM(mSTMatrix, 0);
    200     }
    201 
    202     private int loadShader(int shaderType, String source) {
    203         int shader = GLES20.glCreateShader(shaderType);
    204         checkGlError("glCreateShader type=" + shaderType);
    205         GLES20.glShaderSource(shader, source);
    206         GLES20.glCompileShader(shader);
    207         int[] compiled = new int[1];
    208         GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    209         if (compiled[0] == 0) {
    210             Log.e(TAG, "Could not compile shader " + shaderType + ":");
    211             Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
    212             GLES20.glDeleteShader(shader);
    213             // TODO: handle this more gracefully
    214             throw new IllegalStateException("Could not compile shader " + shaderType);
    215         }
    216         return shader;
    217     }
    218 
    219     private int createProgram(String vertexSource, String fragmentSource) {
    220         int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
    221         if (vertexShader == 0) {
    222             return 0;
    223         }
    224         int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
    225         if (pixelShader == 0) {
    226             return 0;
    227         }
    228 
    229         int program = GLES20.glCreateProgram();
    230         checkGlError("glCreateProgram");
    231         if (program == 0) {
    232             Log.e(TAG, "Could not create program");
    233         }
    234         GLES20.glAttachShader(program, vertexShader);
    235         checkGlError("glAttachShader");
    236         GLES20.glAttachShader(program, pixelShader);
    237         checkGlError("glAttachShader");
    238         GLES20.glLinkProgram(program);
    239         int[] linkStatus = new int[1];
    240         GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
    241         if (linkStatus[0] != GLES20.GL_TRUE) {
    242             Log.e(TAG, "Could not link program: ");
    243             Log.e(TAG, GLES20.glGetProgramInfoLog(program));
    244             GLES20.glDeleteProgram(program);
    245             // TODO: handle this more gracefully
    246             throw new IllegalStateException("Could not link program");
    247         }
    248         return program;
    249     }
    250 
    251     private void drawFrame(SurfaceTexture st, int width, int height, int flipType) {
    252         checkGlError("onDrawFrame start");
    253         st.getTransformMatrix(mSTMatrix);
    254 
    255         Matrix.setIdentityM(mMVPMatrix, /*smOffset*/0);
    256 
    257         // Find intermediate buffer dimensions
    258         Size dimens;
    259         try {
    260             dimens = LegacyCameraDevice.getTextureSize(st);
    261         } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    262             // Should never hit this.
    263             throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e);
    264         }
    265         float texWidth = dimens.getWidth();
    266         float texHeight = dimens.getHeight();
    267 
    268         if (texWidth <= 0 || texHeight <= 0) {
    269             throw new IllegalStateException("Illegal intermediate texture with dimension of 0");
    270         }
    271 
    272         // Letterbox or pillar-box output dimensions into intermediate dimensions.
    273         RectF intermediate = new RectF(/*left*/0, /*top*/0, /*right*/texWidth, /*bottom*/texHeight);
    274         RectF output = new RectF(/*left*/0, /*top*/0, /*right*/width, /*bottom*/height);
    275         android.graphics.Matrix boxingXform = new android.graphics.Matrix();
    276         boxingXform.setRectToRect(output, intermediate, android.graphics.Matrix.ScaleToFit.CENTER);
    277         boxingXform.mapRect(output);
    278 
    279         // Find scaling factor from pillar-boxed/letter-boxed output dimensions to intermediate
    280         // buffer dimensions.
    281         float scaleX = intermediate.width() / output.width();
    282         float scaleY = intermediate.height() / output.height();
    283 
    284         // Intermediate texture is implicitly scaled to 'fill' the output dimensions in clip space
    285         // coordinates in the shader.  To avoid stretching, we need to scale the larger dimension
    286         // of the intermediate buffer so that the output buffer is actually letter-boxed
    287         // or pillar-boxed into the intermediate buffer after clipping.
    288         Matrix.scaleM(mMVPMatrix, /*offset*/0, /*x*/scaleX, /*y*/scaleY, /*z*/1);
    289 
    290         if (DEBUG) {
    291             Log.d(TAG, "Scaling factors (S_x = " + scaleX + ",S_y = " + scaleY + ") used for " +
    292                     width + "x" + height + " surface, intermediate buffer size is " + texWidth +
    293                     "x" + texHeight);
    294         }
    295 
    296         // Set viewport to be output buffer dimensions
    297         GLES20.glViewport(0, 0, width, height);
    298 
    299         if (DEBUG) {
    300             GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
    301             GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
    302         }
    303 
    304         GLES20.glUseProgram(mProgram);
    305         checkGlError("glUseProgram");
    306 
    307         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    308         GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    309 
    310         FloatBuffer triangleVertices;
    311         switch(flipType) {
    312             case FLIP_TYPE_HORIZONTAL:
    313                 triangleVertices = mHorizontalFlipTriangleVertices;
    314                 break;
    315             case FLIP_TYPE_VERTICAL:
    316                 triangleVertices = mVerticalFlipTriangleVertices;
    317                 break;
    318             case FLIP_TYPE_BOTH:
    319                 triangleVertices = mBothFlipTriangleVertices;
    320                 break;
    321             default:
    322                 triangleVertices = mRegularTriangleVertices;
    323                 break;
    324         }
    325 
    326         triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
    327         GLES20.glVertexAttribPointer(maPositionHandle, VERTEX_POS_SIZE, GLES20.GL_FLOAT,
    328                 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
    329         checkGlError("glVertexAttribPointer maPosition");
    330         GLES20.glEnableVertexAttribArray(maPositionHandle);
    331         checkGlError("glEnableVertexAttribArray maPositionHandle");
    332 
    333         triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
    334         GLES20.glVertexAttribPointer(maTextureHandle, VERTEX_UV_SIZE, GLES20.GL_FLOAT,
    335                 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
    336         checkGlError("glVertexAttribPointer maTextureHandle");
    337         GLES20.glEnableVertexAttribArray(maTextureHandle);
    338         checkGlError("glEnableVertexAttribArray maTextureHandle");
    339 
    340         GLES20.glUniformMatrix4fv(muMVPMatrixHandle, /*count*/ 1, /*transpose*/ false, mMVPMatrix,
    341                 /*offset*/ 0);
    342         GLES20.glUniformMatrix4fv(muSTMatrixHandle, /*count*/ 1, /*transpose*/ false, mSTMatrix,
    343                 /*offset*/ 0);
    344 
    345         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*offset*/ 0, /*count*/ 4);
    346         checkGlError("glDrawArrays");
    347     }
    348 
    349     /**
    350      * Initializes GL state.  Call this after the EGL surface has been created and made current.
    351      */
    352     private void initializeGLState() {
    353         mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
    354         if (mProgram == 0) {
    355             throw new IllegalStateException("failed creating program");
    356         }
    357         maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
    358         checkGlError("glGetAttribLocation aPosition");
    359         if (maPositionHandle == -1) {
    360             throw new IllegalStateException("Could not get attrib location for aPosition");
    361         }
    362         maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
    363         checkGlError("glGetAttribLocation aTextureCoord");
    364         if (maTextureHandle == -1) {
    365             throw new IllegalStateException("Could not get attrib location for aTextureCoord");
    366         }
    367 
    368         muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
    369         checkGlError("glGetUniformLocation uMVPMatrix");
    370         if (muMVPMatrixHandle == -1) {
    371             throw new IllegalStateException("Could not get attrib location for uMVPMatrix");
    372         }
    373 
    374         muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
    375         checkGlError("glGetUniformLocation uSTMatrix");
    376         if (muSTMatrixHandle == -1) {
    377             throw new IllegalStateException("Could not get attrib location for uSTMatrix");
    378         }
    379 
    380         int[] textures = new int[1];
    381         GLES20.glGenTextures(/*n*/ 1, textures, /*offset*/ 0);
    382 
    383         mTextureID = textures[0];
    384         GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    385         checkGlError("glBindTexture mTextureID");
    386 
    387         GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
    388                 GLES20.GL_NEAREST);
    389         GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
    390                 GLES20.GL_LINEAR);
    391         GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
    392                 GLES20.GL_CLAMP_TO_EDGE);
    393         GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
    394                 GLES20.GL_CLAMP_TO_EDGE);
    395         checkGlError("glTexParameter");
    396     }
    397 
    398     private int getTextureId() {
    399         return mTextureID;
    400     }
    401 
    402     private void clearState() {
    403         mSurfaces.clear();
    404         mConversionSurfaces.clear();
    405         mPBufferPixels = null;
    406         if (mSurfaceTexture != null) {
    407             mSurfaceTexture.release();
    408         }
    409         mSurfaceTexture = null;
    410     }
    411 
    412     private void configureEGLContext() {
    413         mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
    414         if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
    415             throw new IllegalStateException("No EGL14 display");
    416         }
    417         int[] version = new int[2];
    418         if (!EGL14.eglInitialize(mEGLDisplay, version, /*offset*/ 0, version, /*offset*/ 1)) {
    419             throw new IllegalStateException("Cannot initialize EGL14");
    420         }
    421 
    422         int[] attribList = {
    423                 EGL14.EGL_RED_SIZE, EGL_COLOR_BITLENGTH,
    424                 EGL14.EGL_GREEN_SIZE, EGL_COLOR_BITLENGTH,
    425                 EGL14.EGL_BLUE_SIZE, EGL_COLOR_BITLENGTH,
    426                 EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
    427                 EGL_RECORDABLE_ANDROID, 1,
    428                 EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT | EGL14.EGL_WINDOW_BIT,
    429                 EGL14.EGL_NONE
    430         };
    431         EGLConfig[] configs = new EGLConfig[1];
    432         int[] numConfigs = new int[1];
    433         EGL14.eglChooseConfig(mEGLDisplay, attribList, /*offset*/ 0, configs, /*offset*/ 0,
    434                 configs.length, numConfigs, /*offset*/ 0);
    435         checkEglError("eglCreateContext RGB888+recordable ES2");
    436         mConfigs = configs[0];
    437         int[] attrib_list = {
    438                 EGL14.EGL_CONTEXT_CLIENT_VERSION, GLES_VERSION,
    439                 EGL14.EGL_NONE
    440         };
    441         mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
    442                 attrib_list, /*offset*/ 0);
    443         checkEglError("eglCreateContext");
    444         if(mEGLContext == EGL14.EGL_NO_CONTEXT) {
    445             throw new IllegalStateException("No EGLContext could be made");
    446         }
    447     }
    448 
    449     private void configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces) {
    450         if (surfaces == null || surfaces.size() == 0) {
    451             throw new IllegalStateException("No Surfaces were provided to draw to");
    452         }
    453         int[] surfaceAttribs = {
    454                 EGL14.EGL_NONE
    455         };
    456         for (EGLSurfaceHolder holder : surfaces) {
    457             holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
    458                     holder.surface, surfaceAttribs, /*offset*/ 0);
    459             checkEglError("eglCreateWindowSurface");
    460         }
    461     }
    462 
    463     private void configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces) {
    464         if (surfaces == null || surfaces.size() == 0) {
    465             throw new IllegalStateException("No Surfaces were provided to draw to");
    466         }
    467 
    468         int maxLength = 0;
    469         for (EGLSurfaceHolder holder : surfaces) {
    470             int length = holder.width * holder.height;
    471             // Find max surface size, ensure PBuffer can hold this many pixels
    472             maxLength = (length > maxLength) ? length : maxLength;
    473             int[] surfaceAttribs = {
    474                     EGL14.EGL_WIDTH, holder.width,
    475                     EGL14.EGL_HEIGHT, holder.height,
    476                     EGL14.EGL_NONE
    477             };
    478             holder.eglSurface =
    479                     EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
    480             checkEglError("eglCreatePbufferSurface");
    481         }
    482         mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES)
    483                 .order(ByteOrder.nativeOrder());
    484     }
    485 
    486     private void releaseEGLContext() {
    487         if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
    488             EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
    489                     EGL14.EGL_NO_CONTEXT);
    490             dumpGlTiming();
    491             if (mSurfaces != null) {
    492                 for (EGLSurfaceHolder holder : mSurfaces) {
    493                     if (holder.eglSurface != null) {
    494                         EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
    495                     }
    496                 }
    497             }
    498             if (mConversionSurfaces != null) {
    499                 for (EGLSurfaceHolder holder : mConversionSurfaces) {
    500                     if (holder.eglSurface != null) {
    501                         EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
    502                     }
    503                 }
    504             }
    505             EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
    506             EGL14.eglReleaseThread();
    507             EGL14.eglTerminate(mEGLDisplay);
    508         }
    509 
    510         mConfigs = null;
    511         mEGLDisplay = EGL14.EGL_NO_DISPLAY;
    512         mEGLContext = EGL14.EGL_NO_CONTEXT;
    513         clearState();
    514     }
    515 
    516     private void makeCurrent(EGLSurface surface) {
    517         EGL14.eglMakeCurrent(mEGLDisplay, surface, surface, mEGLContext);
    518         checkEglError("makeCurrent");
    519     }
    520 
    521     private boolean swapBuffers(EGLSurface surface) {
    522         boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface);
    523         checkEglError("swapBuffers");
    524         return result;
    525     }
    526 
    527     private void checkEglError(String msg) {
    528         int error;
    529         if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
    530             throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error));
    531         }
    532     }
    533 
    534     private void checkGlError(String msg) {
    535         int error;
    536         while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
    537             throw new IllegalStateException(msg + ": GLES20 error: 0x" + Integer.toHexString(error));
    538         }
    539     }
    540 
    541     /**
    542      * Save a measurement dump to disk, in
    543      * {@code /sdcard/CameraLegacy/durations_<time>_<width1>x<height1>_...txt}
    544      */
    545     private void dumpGlTiming() {
    546         if (mPerfMeasurer == null) return;
    547 
    548         File legacyStorageDir = new File(Environment.getExternalStorageDirectory(), "CameraLegacy");
    549         if (!legacyStorageDir.exists()){
    550             if (!legacyStorageDir.mkdirs()){
    551                 Log.e(TAG, "Failed to create directory for data dump");
    552                 return;
    553             }
    554         }
    555 
    556         StringBuilder path = new StringBuilder(legacyStorageDir.getPath());
    557         path.append(File.separator);
    558         path.append("durations_");
    559 
    560         Time now = new Time();
    561         now.setToNow();
    562         path.append(now.format2445());
    563         path.append("_S");
    564         for (EGLSurfaceHolder surface : mSurfaces) {
    565             path.append(String.format("_%d_%d", surface.width, surface.height));
    566         }
    567         path.append("_C");
    568         for (EGLSurfaceHolder surface : mConversionSurfaces) {
    569             path.append(String.format("_%d_%d", surface.width, surface.height));
    570         }
    571         path.append(".txt");
    572         mPerfMeasurer.dumpPerformanceData(path.toString());
    573     }
    574 
    575     private void setupGlTiming() {
    576         if (PerfMeasurement.isGlTimingSupported()) {
    577             Log.d(TAG, "Enabling GL performance measurement");
    578             mPerfMeasurer = new PerfMeasurement();
    579         } else {
    580             Log.d(TAG, "GL performance measurement not supported on this device");
    581             mPerfMeasurer = null;
    582         }
    583     }
    584 
    585     private void beginGlTiming() {
    586         if (mPerfMeasurer == null) return;
    587         mPerfMeasurer.startTimer();
    588     }
    589 
    590     private void addGlTimestamp(long timestamp) {
    591         if (mPerfMeasurer == null) return;
    592         mPerfMeasurer.addTimestamp(timestamp);
    593     }
    594 
    595     private void endGlTiming() {
    596         if (mPerfMeasurer == null) return;
    597         mPerfMeasurer.stopTimer();
    598     }
    599 
    600     /**
    601      * Return the surface texture to draw to - this is the texture use to when producing output
    602      * surface buffers.
    603      *
    604      * @return a {@link SurfaceTexture}.
    605      */
    606     public SurfaceTexture getSurfaceTexture() {
    607         return mSurfaceTexture;
    608     }
    609 
    610     /**
    611      * Set a collection of output {@link Surface}s that can be drawn to.
    612      *
    613      * @param surfaces a {@link Collection} of surfaces.
    614      */
    615     public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) {
    616         releaseEGLContext();
    617 
    618         if (surfaces == null || surfaces.size() == 0) {
    619             Log.w(TAG, "No output surfaces configured for GL drawing.");
    620             return;
    621         }
    622 
    623         for (Pair<Surface, Size> p : surfaces) {
    624             Surface s = p.first;
    625             Size surfaceSize = p.second;
    626             // If pixel conversions aren't handled by egl, use a pbuffer
    627             try {
    628                 EGLSurfaceHolder holder = new EGLSurfaceHolder();
    629                 holder.surface = s;
    630                 holder.width = surfaceSize.getWidth();
    631                 holder.height = surfaceSize.getHeight();
    632                 if (LegacyCameraDevice.needsConversion(s)) {
    633                     // Always override to YV12 output for YUV surface formats.
    634                     LegacyCameraDevice.setSurfaceFormat(s, ImageFormat.YV12);
    635                     mConversionSurfaces.add(holder);
    636                 } else {
    637                     mSurfaces.add(holder);
    638                 }
    639             } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    640                 Log.w(TAG, "Surface abandoned, skipping configuration... ", e);
    641             }
    642         }
    643 
    644         // Set up egl display
    645         configureEGLContext();
    646 
    647         // Set up regular egl surfaces if needed
    648         if (mSurfaces.size() > 0) {
    649             configureEGLOutputSurfaces(mSurfaces);
    650         }
    651 
    652         // Set up pbuffer surface if needed
    653         if (mConversionSurfaces.size() > 0) {
    654             configureEGLPbufferSurfaces(mConversionSurfaces);
    655         }
    656         makeCurrent((mSurfaces.size() > 0) ? mSurfaces.get(0).eglSurface :
    657                 mConversionSurfaces.get(0).eglSurface);
    658         initializeGLState();
    659         mSurfaceTexture = new SurfaceTexture(getTextureId());
    660 
    661         // Set up performance tracking if enabled
    662         if (SystemProperties.getBoolean(LEGACY_PERF_PROPERTY, false)) {
    663             setupGlTiming();
    664         }
    665     }
    666 
    667     /**
    668      * Draw the current buffer in the {@link SurfaceTexture} returned from
    669      * {@link #getSurfaceTexture()} into the set of target {@link Surface}s
    670      * in the next request from the given {@link CaptureCollector}, or drop
    671      * the frame if none is available.
    672      *
    673      * <p>
    674      * Any {@link Surface}s targeted must be a subset of the {@link Surface}s
    675      * set in the last {@link #configureSurfaces(java.util.Collection)} call.
    676      * </p>
    677      *
    678      * @param targetCollector the surfaces to draw to.
    679      */
    680     public void drawIntoSurfaces(CaptureCollector targetCollector) {
    681         if ((mSurfaces == null || mSurfaces.size() == 0)
    682                 && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) {
    683             return;
    684         }
    685 
    686         boolean doTiming = targetCollector.hasPendingPreviewCaptures();
    687         checkGlError("before updateTexImage");
    688 
    689         if (doTiming) {
    690             beginGlTiming();
    691         }
    692 
    693         mSurfaceTexture.updateTexImage();
    694 
    695         long timestamp = mSurfaceTexture.getTimestamp();
    696 
    697         Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp);
    698 
    699         // No preview request queued, drop frame.
    700         if (captureHolder == null) {
    701             if (DEBUG) {
    702                 Log.d(TAG, "Dropping preview frame.");
    703             }
    704             if (doTiming) {
    705                 endGlTiming();
    706             }
    707             return;
    708         }
    709 
    710         RequestHolder request = captureHolder.first;
    711 
    712         Collection<Surface> targetSurfaces = request.getHolderTargets();
    713         if (doTiming) {
    714             addGlTimestamp(timestamp);
    715         }
    716 
    717         List<Long> targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
    718         for (EGLSurfaceHolder holder : mSurfaces) {
    719             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
    720                 try{
    721                     LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
    722                             holder.height);
    723                     makeCurrent(holder.eglSurface);
    724 
    725                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
    726                     drawFrame(mSurfaceTexture, holder.width, holder.height,
    727                             (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
    728                                     FLIP_TYPE_HORIZONTAL : FLIP_TYPE_NONE);
    729                     swapBuffers(holder.eglSurface);
    730                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    731                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
    732                 }
    733             }
    734         }
    735         for (EGLSurfaceHolder holder : mConversionSurfaces) {
    736             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
    737                 makeCurrent(holder.eglSurface);
    738                 // glReadPixels reads from the bottom of the buffer, so add an extra vertical flip
    739                 drawFrame(mSurfaceTexture, holder.width, holder.height,
    740                         (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
    741                                 FLIP_TYPE_BOTH : FLIP_TYPE_VERTICAL);
    742                 mPBufferPixels.clear();
    743                 GLES20.glReadPixels(/*x*/ 0, /*y*/ 0, holder.width, holder.height,
    744                         GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPBufferPixels);
    745                 checkGlError("glReadPixels");
    746 
    747                 try {
    748                     int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
    749                     LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
    750                             holder.height);
    751                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
    752                     LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
    753                             holder.width, holder.height, format);
    754                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    755                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
    756                 }
    757             }
    758         }
    759         targetCollector.previewProduced();
    760 
    761         if (doTiming) {
    762             endGlTiming();
    763         }
    764     }
    765 
    766     /**
    767      * Clean up the current GL context.
    768      */
    769     public void cleanupEGLContext() {
    770         releaseEGLContext();
    771     }
    772 
    773     /**
    774      * Drop all current GL operations on the floor.
    775      */
    776     public void flush() {
    777         // TODO: implement flush
    778         Log.e(TAG, "Flush not yet implemented.");
    779     }
    780 }
    781