Home | History | Annotate | Download | only in legacy 
      1 /*
      2  * Copyright (C) 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 package android.hardware.camera2.legacy;
     17 
     18 import android.graphics.ImageFormat;
     19 import android.graphics.RectF;
     20 import android.graphics.SurfaceTexture;
     21 import android.hardware.camera2.CameraCharacteristics;
     22 import android.os.Environment;
     23 import android.opengl.EGL14;
     24 import android.opengl.EGLConfig;
     25 import android.opengl.EGLContext;
     26 import android.opengl.EGLDisplay;
     27 import android.opengl.EGLSurface;
     28 import android.opengl.GLES11Ext;
     29 import android.opengl.GLES20;
     30 import android.opengl.Matrix;
     31 import android.text.format.Time;
     32 import android.util.Log;
     33 import android.util.Pair;
     34 import android.util.Size;
     35 import android.view.Surface;
     36 import android.os.SystemProperties;
     37 
     38 import java.io.File;
     39 import java.nio.ByteBuffer;
     40 import java.nio.ByteOrder;
     41 import java.nio.FloatBuffer;
     42 import java.util.ArrayList;
     43 import java.util.Collection;
     44 import java.util.List;
     45 
     46 /**
     47  * A renderer class that manages the GL state, and can draw a frame into a set of output
     48  * {@link Surface}s.
     49  */
     50 public class SurfaceTextureRenderer {
     51     private static final String TAG = SurfaceTextureRenderer.class.getSimpleName();
     52     private static final boolean DEBUG = false;
     53     private static final int EGL_RECORDABLE_ANDROID = 0x3142; // from EGL/eglext.h
     54     private static final int GL_MATRIX_SIZE = 16;
     55     private static final int VERTEX_POS_SIZE = 3;
     56     private static final int VERTEX_UV_SIZE = 2;
     57     private static final int EGL_COLOR_BITLENGTH = 8;
     58     private static final int GLES_VERSION = 2;
     59     private static final int PBUFFER_PIXEL_BYTES = 4;
     60 
     61     private static final int FLIP_TYPE_NONE = 0;
     62     private static final int FLIP_TYPE_HORIZONTAL = 1;
     63     private static final int FLIP_TYPE_VERTICAL = 2;
     64     private static final int FLIP_TYPE_BOTH = FLIP_TYPE_HORIZONTAL | FLIP_TYPE_VERTICAL;
     65 
     66     private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY;
     67     private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT;
     68     private EGLConfig mConfigs;
     69 
     70     private class EGLSurfaceHolder {
     71         Surface surface;
     72         EGLSurface eglSurface;
     73         int width;
     74         int height;
     75     }
     76 
     77     private List<EGLSurfaceHolder> mSurfaces = new ArrayList<EGLSurfaceHolder>();
     78     private List<EGLSurfaceHolder> mConversionSurfaces = new ArrayList<EGLSurfaceHolder>();
     79 
     80     private ByteBuffer mPBufferPixels;
     81 
     82     // Hold this to avoid GC
     83     private volatile SurfaceTexture mSurfaceTexture;
     84 
     85     private static final int FLOAT_SIZE_BYTES = 4;
     86     private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
     87     private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
     88     private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
     89 
     90     // Sampling is mirrored across the horizontal axis
     91     private static final float[] sHorizontalFlipTriangleVertices = {
     92             // X, Y, Z, U, V
     93             -1.0f, -1.0f, 0, 1.f, 0.f,
     94             1.0f, -1.0f, 0, 0.f, 0.f,
     95             -1.0f,  1.0f, 0, 1.f, 1.f,
     96             1.0f,  1.0f, 0, 0.f, 1.f,
     97     };
     98 
     99     // Sampling is mirrored across the vertical axis
    100     private static final float[] sVerticalFlipTriangleVertices = {
    101             // X, Y, Z, U, V
    102             -1.0f, -1.0f, 0, 0.f, 1.f,
    103             1.0f, -1.0f, 0, 1.f, 1.f,
    104             -1.0f,  1.0f, 0, 0.f, 0.f,
    105             1.0f,  1.0f, 0, 1.f, 0.f,
    106     };
    107 
    108     // Sampling is mirrored across the both axes
    109     private static final float[] sBothFlipTriangleVertices = {
    110             // X, Y, Z, U, V
    111             -1.0f, -1.0f, 0, 1.f, 1.f,
    112             1.0f, -1.0f, 0, 0.f, 1.f,
    113             -1.0f,  1.0f, 0, 1.f, 0.f,
    114             1.0f,  1.0f, 0, 0.f, 0.f,
    115     };
    116 
    117     // Sampling is 1:1 for a straight copy for the back camera
    118     private static final float[] sRegularTriangleVertices = {
    119             // X, Y, Z, U, V
    120             -1.0f, -1.0f, 0, 0.f, 0.f,
    121             1.0f, -1.0f, 0, 1.f, 0.f,
    122             -1.0f,  1.0f, 0, 0.f, 1.f,
    123             1.0f,  1.0f, 0, 1.f, 1.f,
    124     };
    125 
    126     private FloatBuffer mRegularTriangleVertices;
    127     private FloatBuffer mHorizontalFlipTriangleVertices;
    128     private FloatBuffer mVerticalFlipTriangleVertices;
    129     private FloatBuffer mBothFlipTriangleVertices;
    130     private final int mFacing;
    131 
    132     /**
    133      * As used in this file, this vertex shader maps a unit square to the view, and
    134      * tells the fragment shader to interpolate over it.  Each surface pixel position
    135      * is mapped to a 2D homogeneous texture coordinate of the form (s, t, 0, 1) with
    136      * s and t in the inclusive range [0, 1], and the matrix from
    137      * {@link SurfaceTexture#getTransformMatrix(float[])} is used to map this
    138      * coordinate to a texture location.
    139      */
    140     private static final String VERTEX_SHADER =
    141             "uniform mat4 uMVPMatrix;\n" +
    142             "uniform mat4 uSTMatrix;\n" +
    143             "attribute vec4 aPosition;\n" +
    144             "attribute vec4 aTextureCoord;\n" +
    145             "varying vec2 vTextureCoord;\n" +
    146             "void main() {\n" +
    147             "  gl_Position = uMVPMatrix * aPosition;\n" +
    148             "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
    149             "}\n";
    150 
    151     /**
    152      * This fragment shader simply draws the color in the 2D texture at
    153      * the location from the {@code VERTEX_SHADER}.
    154      */
    155     private static final String FRAGMENT_SHADER =
    156             "#extension GL_OES_EGL_image_external : require\n" +
    157             "precision mediump float;\n" +
    158             "varying vec2 vTextureCoord;\n" +
    159             "uniform samplerExternalOES sTexture;\n" +
    160             "void main() {\n" +
    161             "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
    162             "}\n";
    163 
    164     private float[] mMVPMatrix = new float[GL_MATRIX_SIZE];
    165     private float[] mSTMatrix = new float[GL_MATRIX_SIZE];
    166 
    167     private int mProgram;
    168     private int mTextureID = 0;
    169     private int muMVPMatrixHandle;
    170     private int muSTMatrixHandle;
    171     private int maPositionHandle;
    172     private int maTextureHandle;
    173 
    174     private PerfMeasurement mPerfMeasurer = null;
    175     private static final String LEGACY_PERF_PROPERTY = "persist.camera.legacy_perf";
    176 
    177     public SurfaceTextureRenderer(int facing) {
    178         mFacing = facing;
    179 
    180         mRegularTriangleVertices = ByteBuffer.allocateDirect(sRegularTriangleVertices.length *
    181                 FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
    182         mRegularTriangleVertices.put(sRegularTriangleVertices).position(0);
    183 
    184         mHorizontalFlipTriangleVertices = ByteBuffer.allocateDirect(
    185                 sHorizontalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    186                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    187         mHorizontalFlipTriangleVertices.put(sHorizontalFlipTriangleVertices).position(0);
    188 
    189         mVerticalFlipTriangleVertices = ByteBuffer.allocateDirect(
    190                 sVerticalFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    191                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    192         mVerticalFlipTriangleVertices.put(sVerticalFlipTriangleVertices).position(0);
    193 
    194         mBothFlipTriangleVertices = ByteBuffer.allocateDirect(
    195                 sBothFlipTriangleVertices.length * FLOAT_SIZE_BYTES).
    196                 order(ByteOrder.nativeOrder()).asFloatBuffer();
    197         mBothFlipTriangleVertices.put(sBothFlipTriangleVertices).position(0);
    198 
    199         Matrix.setIdentityM(mSTMatrix, 0);
    200     }
    201 
    202     private int loadShader(int shaderType, String source) {
    203         int shader = GLES20.glCreateShader(shaderType);
    204         checkGlError("glCreateShader type=" + shaderType);
    205         GLES20.glShaderSource(shader, source);
    206         GLES20.glCompileShader(shader);
    207         int[] compiled = new int[1];
    208         GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
    209         if (compiled[0] == 0) {
    210             Log.e(TAG, "Could not compile shader " + shaderType + ":");
    211             Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
    212             GLES20.glDeleteShader(shader);
    213             // TODO: handle this more gracefully
    214             throw new IllegalStateException("Could not compile shader " + shaderType);
    215         }
    216         return shader;
    217     }
    218 
    219     private int createProgram(String vertexSource, String fragmentSource) {
    220         int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
    221         if (vertexShader == 0) {
    222             return 0;
    223         }
    224         int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
    225         if (pixelShader == 0) {
    226             return 0;
    227         }
    228 
    229         int program = GLES20.glCreateProgram();
    230         checkGlError("glCreateProgram");
    231         if (program == 0) {
    232             Log.e(TAG, "Could not create program");
    233         }
    234         GLES20.glAttachShader(program, vertexShader);
    235         checkGlError("glAttachShader");
    236         GLES20.glAttachShader(program, pixelShader);
    237         checkGlError("glAttachShader");
    238         GLES20.glLinkProgram(program);
    239         int[] linkStatus = new int[1];
    240         GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
    241         if (linkStatus[0] != GLES20.GL_TRUE) {
    242             Log.e(TAG, "Could not link program: ");
    243             Log.e(TAG, GLES20.glGetProgramInfoLog(program));
    244             GLES20.glDeleteProgram(program);
    245             // TODO: handle this more gracefully
    246             throw new IllegalStateException("Could not link program");
    247         }
    248         return program;
    249     }
    250 
    251     private void drawFrame(SurfaceTexture st, int width, int height, int flipType) {
    252         checkGlError("onDrawFrame start");
    253         st.getTransformMatrix(mSTMatrix);
    254 
    255         Matrix.setIdentityM(mMVPMatrix, /*smOffset*/0);
    256 
    257         // Find intermediate buffer dimensions
    258         Size dimens;
    259         try {
    260             dimens = LegacyCameraDevice.getTextureSize(st);
    261         } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    262             // Should never hit this.
    263             throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e);
    264         }
    265         float texWidth = dimens.getWidth();
    266         float texHeight = dimens.getHeight();
    267 
    268         if (texWidth <= 0 || texHeight <= 0) {
    269             throw new IllegalStateException("Illegal intermediate texture with dimension of 0");
    270         }
    271 
    272         // Letterbox or pillar-box output dimensions into intermediate dimensions.
    273         RectF intermediate = new RectF(/*left*/0, /*top*/0, /*right*/texWidth, /*bottom*/texHeight);
    274         RectF output = new RectF(/*left*/0, /*top*/0, /*right*/width, /*bottom*/height);
    275         android.graphics.Matrix boxingXform = new android.graphics.Matrix();
    276         boxingXform.setRectToRect(output, intermediate, android.graphics.Matrix.ScaleToFit.CENTER);
    277         boxingXform.mapRect(output);
    278 
    279         // Find scaling factor from pillar-boxed/letter-boxed output dimensions to intermediate
    280         // buffer dimensions.
    281         float scaleX = intermediate.width() / output.width();
    282         float scaleY = intermediate.height() / output.height();
    283 
    284         // Intermediate texture is implicitly scaled to 'fill' the output dimensions in clip space
    285         // coordinates in the shader.  To avoid stretching, we need to scale the larger dimension
    286         // of the intermediate buffer so that the output buffer is actually letter-boxed
    287         // or pillar-boxed into the intermediate buffer after clipping.
    288         Matrix.scaleM(mMVPMatrix, /*offset*/0, /*x*/scaleX, /*y*/scaleY, /*z*/1);
    289 
    290         if (DEBUG) {
    291             Log.d(TAG, "Scaling factors (S_x = " + scaleX + ",S_y = " + scaleY + ") used for " +
    292                     width + "x" + height + " surface, intermediate buffer size is " + texWidth +
    293                     "x" + texHeight);
    294         }
    295 
    296         // Set viewport to be output buffer dimensions
    297         GLES20.glViewport(0, 0, width, height);
    298 
    299         if (DEBUG) {
    300             GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
    301             GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
    302         }
    303 
    304         GLES20.glUseProgram(mProgram);
    305         checkGlError("glUseProgram");
    306 
    307         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    308         GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    309 
    310         FloatBuffer triangleVertices;
    311         switch(flipType) {
    312             case FLIP_TYPE_HORIZONTAL:
    313                 triangleVertices = mHorizontalFlipTriangleVertices;
    314                 break;
    315             case FLIP_TYPE_VERTICAL:
    316                 triangleVertices = mVerticalFlipTriangleVertices;
    317                 break;
    318             case FLIP_TYPE_BOTH:
    319                 triangleVertices = mBothFlipTriangleVertices;
    320                 break;
    321             default:
    322                 triangleVertices = mRegularTriangleVertices;
    323                 break;
    324         }
    325 
    326         triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
    327         GLES20.glVertexAttribPointer(maPositionHandle, VERTEX_POS_SIZE, GLES20.GL_FLOAT,
    328                 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
    329         checkGlError("glVertexAttribPointer maPosition");
    330         GLES20.glEnableVertexAttribArray(maPositionHandle);
    331         checkGlError("glEnableVertexAttribArray maPositionHandle");
    332 
    333         triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
    334         GLES20.glVertexAttribPointer(maTextureHandle, VERTEX_UV_SIZE, GLES20.GL_FLOAT,
    335                 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices);
    336         checkGlError("glVertexAttribPointer maTextureHandle");
    337         GLES20.glEnableVertexAttribArray(maTextureHandle);
    338         checkGlError("glEnableVertexAttribArray maTextureHandle");
    339 
    340         GLES20.glUniformMatrix4fv(muMVPMatrixHandle, /*count*/ 1, /*transpose*/ false, mMVPMatrix,
    341                 /*offset*/ 0);
    342         GLES20.glUniformMatrix4fv(muSTMatrixHandle, /*count*/ 1, /*transpose*/ false, mSTMatrix,
    343                 /*offset*/ 0);
    344 
    345         GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*offset*/ 0, /*count*/ 4);
    346         checkGlError("glDrawArrays");
    347     }
    348 
    349     /**
    350      * Initializes GL state.  Call this after the EGL surface has been created and made current.
    351      */
    352     private void initializeGLState() {
    353         mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
    354         if (mProgram == 0) {
    355             throw new IllegalStateException("failed creating program");
    356         }
    357         maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
    358         checkGlError("glGetAttribLocation aPosition");
    359         if (maPositionHandle == -1) {
    360             throw new IllegalStateException("Could not get attrib location for aPosition");
    361         }
    362         maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
    363         checkGlError("glGetAttribLocation aTextureCoord");
    364         if (maTextureHandle == -1) {
    365             throw new IllegalStateException("Could not get attrib location for aTextureCoord");
    366         }
    367 
    368         muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
    369         checkGlError("glGetUniformLocation uMVPMatrix");
    370         if (muMVPMatrixHandle == -1) {
    371             throw new IllegalStateException("Could not get attrib location for uMVPMatrix");
    372         }
    373 
    374         muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
    375         checkGlError("glGetUniformLocation uSTMatrix");
    376         if (muSTMatrixHandle == -1) {
    377             throw new IllegalStateException("Could not get attrib location for uSTMatrix");
    378         }
    379 
    380         int[] textures = new int[1];
    381         GLES20.glGenTextures(/*n*/ 1, textures, /*offset*/ 0);
    382 
    383         mTextureID = textures[0];
    384         GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
    385         checkGlError("glBindTexture mTextureID");
    386 
    387         GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
    388                 GLES20.GL_NEAREST);
    389         GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
    390                 GLES20.GL_LINEAR);
    391         GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
    392                 GLES20.GL_CLAMP_TO_EDGE);
    393         GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
    394                 GLES20.GL_CLAMP_TO_EDGE);
    395         checkGlError("glTexParameter");
    396     }
    397 
    398     private int getTextureId() {
    399         return mTextureID;
    400     }
    401 
    402     private void clearState() {
    403         mSurfaces.clear();
    404         for (EGLSurfaceHolder holder : mConversionSurfaces) {
    405             try {
    406                 LegacyCameraDevice.disconnectSurface(holder.surface);
    407             } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    408                 Log.w(TAG, "Surface abandoned, skipping...", e);
    409             }
    410         }
    411         mConversionSurfaces.clear();
    412         mPBufferPixels = null;
    413         if (mSurfaceTexture != null) {
    414             mSurfaceTexture.release();
    415         }
    416         mSurfaceTexture = null;
    417     }
    418 
    419     private void configureEGLContext() {
    420         mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
    421         if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
    422             throw new IllegalStateException("No EGL14 display");
    423         }
    424         int[] version = new int[2];
    425         if (!EGL14.eglInitialize(mEGLDisplay, version, /*offset*/ 0, version, /*offset*/ 1)) {
    426             throw new IllegalStateException("Cannot initialize EGL14");
    427         }
    428 
    429         int[] attribList = {
    430                 EGL14.EGL_RED_SIZE, EGL_COLOR_BITLENGTH,
    431                 EGL14.EGL_GREEN_SIZE, EGL_COLOR_BITLENGTH,
    432                 EGL14.EGL_BLUE_SIZE, EGL_COLOR_BITLENGTH,
    433                 EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
    434                 EGL_RECORDABLE_ANDROID, 1,
    435                 EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT | EGL14.EGL_WINDOW_BIT,
    436                 EGL14.EGL_NONE
    437         };
    438         EGLConfig[] configs = new EGLConfig[1];
    439         int[] numConfigs = new int[1];
    440         EGL14.eglChooseConfig(mEGLDisplay, attribList, /*offset*/ 0, configs, /*offset*/ 0,
    441                 configs.length, numConfigs, /*offset*/ 0);
    442         checkEglError("eglCreateContext RGB888+recordable ES2");
    443         mConfigs = configs[0];
    444         int[] attrib_list = {
    445                 EGL14.EGL_CONTEXT_CLIENT_VERSION, GLES_VERSION,
    446                 EGL14.EGL_NONE
    447         };
    448         mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
    449                 attrib_list, /*offset*/ 0);
    450         checkEglError("eglCreateContext");
    451         if(mEGLContext == EGL14.EGL_NO_CONTEXT) {
    452             throw new IllegalStateException("No EGLContext could be made");
    453         }
    454     }
    455 
    456     private void configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces) {
    457         if (surfaces == null || surfaces.size() == 0) {
    458             throw new IllegalStateException("No Surfaces were provided to draw to");
    459         }
    460         int[] surfaceAttribs = {
    461                 EGL14.EGL_NONE
    462         };
    463         for (EGLSurfaceHolder holder : surfaces) {
    464             holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
    465                     holder.surface, surfaceAttribs, /*offset*/ 0);
    466             checkEglError("eglCreateWindowSurface");
    467         }
    468     }
    469 
    470     private void configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces) {
    471         if (surfaces == null || surfaces.size() == 0) {
    472             throw new IllegalStateException("No Surfaces were provided to draw to");
    473         }
    474 
    475         int maxLength = 0;
    476         for (EGLSurfaceHolder holder : surfaces) {
    477             int length = holder.width * holder.height;
    478             // Find max surface size, ensure PBuffer can hold this many pixels
    479             maxLength = (length > maxLength) ? length : maxLength;
    480             int[] surfaceAttribs = {
    481                     EGL14.EGL_WIDTH, holder.width,
    482                     EGL14.EGL_HEIGHT, holder.height,
    483                     EGL14.EGL_NONE
    484             };
    485             holder.eglSurface =
    486                     EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
    487             checkEglError("eglCreatePbufferSurface");
    488         }
    489         mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES)
    490                 .order(ByteOrder.nativeOrder());
    491     }
    492 
    493     private void releaseEGLContext() {
    494         if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) {
    495             EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
    496                     EGL14.EGL_NO_CONTEXT);
    497             dumpGlTiming();
    498             if (mSurfaces != null) {
    499                 for (EGLSurfaceHolder holder : mSurfaces) {
    500                     if (holder.eglSurface != null) {
    501                         EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
    502                     }
    503                 }
    504             }
    505             if (mConversionSurfaces != null) {
    506                 for (EGLSurfaceHolder holder : mConversionSurfaces) {
    507                     if (holder.eglSurface != null) {
    508                         EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface);
    509                     }
    510                 }
    511             }
    512             EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
    513             EGL14.eglReleaseThread();
    514             EGL14.eglTerminate(mEGLDisplay);
    515         }
    516 
    517         mConfigs = null;
    518         mEGLDisplay = EGL14.EGL_NO_DISPLAY;
    519         mEGLContext = EGL14.EGL_NO_CONTEXT;
    520         clearState();
    521     }
    522 
    523     private void makeCurrent(EGLSurface surface) {
    524         EGL14.eglMakeCurrent(mEGLDisplay, surface, surface, mEGLContext);
    525         checkEglError("makeCurrent");
    526     }
    527 
    528     private boolean swapBuffers(EGLSurface surface)
    529             throws LegacyExceptionUtils.BufferQueueAbandonedException {
    530         boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface);
    531         int error = EGL14.eglGetError();
    532         if (error == EGL14.EGL_BAD_SURFACE) {
    533             throw new LegacyExceptionUtils.BufferQueueAbandonedException();
    534         } else if (error != EGL14.EGL_SUCCESS) {
    535             throw new IllegalStateException("swapBuffers: EGL error: 0x" +
    536                     Integer.toHexString(error));
    537         }
    538         return result;
    539     }
    540 
    541     private void checkEglError(String msg) {
    542         int error;
    543         if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
    544             throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error));
    545         }
    546     }
    547 
    548     private void checkGlError(String msg) {
    549         int error;
    550         while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
    551             throw new IllegalStateException(msg + ": GLES20 error: 0x" + Integer.toHexString(error));
    552         }
    553     }
    554 
    555     /**
    556      * Save a measurement dump to disk, in
    557      * {@code /sdcard/CameraLegacy/durations_<time>_<width1>x<height1>_...txt}
    558      */
    559     private void dumpGlTiming() {
    560         if (mPerfMeasurer == null) return;
    561 
    562         File legacyStorageDir = new File(Environment.getExternalStorageDirectory(), "CameraLegacy");
    563         if (!legacyStorageDir.exists()){
    564             if (!legacyStorageDir.mkdirs()){
    565                 Log.e(TAG, "Failed to create directory for data dump");
    566                 return;
    567             }
    568         }
    569 
    570         StringBuilder path = new StringBuilder(legacyStorageDir.getPath());
    571         path.append(File.separator);
    572         path.append("durations_");
    573 
    574         Time now = new Time();
    575         now.setToNow();
    576         path.append(now.format2445());
    577         path.append("_S");
    578         for (EGLSurfaceHolder surface : mSurfaces) {
    579             path.append(String.format("_%d_%d", surface.width, surface.height));
    580         }
    581         path.append("_C");
    582         for (EGLSurfaceHolder surface : mConversionSurfaces) {
    583             path.append(String.format("_%d_%d", surface.width, surface.height));
    584         }
    585         path.append(".txt");
    586         mPerfMeasurer.dumpPerformanceData(path.toString());
    587     }
    588 
    589     private void setupGlTiming() {
    590         if (PerfMeasurement.isGlTimingSupported()) {
    591             Log.d(TAG, "Enabling GL performance measurement");
    592             mPerfMeasurer = new PerfMeasurement();
    593         } else {
    594             Log.d(TAG, "GL performance measurement not supported on this device");
    595             mPerfMeasurer = null;
    596         }
    597     }
    598 
    599     private void beginGlTiming() {
    600         if (mPerfMeasurer == null) return;
    601         mPerfMeasurer.startTimer();
    602     }
    603 
    604     private void addGlTimestamp(long timestamp) {
    605         if (mPerfMeasurer == null) return;
    606         mPerfMeasurer.addTimestamp(timestamp);
    607     }
    608 
    609     private void endGlTiming() {
    610         if (mPerfMeasurer == null) return;
    611         mPerfMeasurer.stopTimer();
    612     }
    613 
    614     /**
    615      * Return the surface texture to draw to - this is the texture use to when producing output
    616      * surface buffers.
    617      *
    618      * @return a {@link SurfaceTexture}.
    619      */
    620     public SurfaceTexture getSurfaceTexture() {
    621         return mSurfaceTexture;
    622     }
    623 
    624     /**
    625      * Set a collection of output {@link Surface}s that can be drawn to.
    626      *
    627      * @param surfaces a {@link Collection} of surfaces.
    628      */
    629     public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) {
    630         releaseEGLContext();
    631 
    632         if (surfaces == null || surfaces.size() == 0) {
    633             Log.w(TAG, "No output surfaces configured for GL drawing.");
    634             return;
    635         }
    636 
    637         for (Pair<Surface, Size> p : surfaces) {
    638             Surface s = p.first;
    639             Size surfaceSize = p.second;
    640             // If pixel conversions aren't handled by egl, use a pbuffer
    641             try {
    642                 EGLSurfaceHolder holder = new EGLSurfaceHolder();
    643                 holder.surface = s;
    644                 holder.width = surfaceSize.getWidth();
    645                 holder.height = surfaceSize.getHeight();
    646                 if (LegacyCameraDevice.needsConversion(s)) {
    647                     mConversionSurfaces.add(holder);
    648                     // LegacyCameraDevice is the producer of surfaces if it's not handled by EGL,
    649                     // so LegacyCameraDevice needs to connect to the surfaces.
    650                     LegacyCameraDevice.connectSurface(s);
    651                 } else {
    652                     mSurfaces.add(holder);
    653                 }
    654             } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    655                 Log.w(TAG, "Surface abandoned, skipping configuration... ", e);
    656             }
    657         }
    658 
    659         // Set up egl display
    660         configureEGLContext();
    661 
    662         // Set up regular egl surfaces if needed
    663         if (mSurfaces.size() > 0) {
    664             configureEGLOutputSurfaces(mSurfaces);
    665         }
    666 
    667         // Set up pbuffer surface if needed
    668         if (mConversionSurfaces.size() > 0) {
    669             configureEGLPbufferSurfaces(mConversionSurfaces);
    670         }
    671         makeCurrent((mSurfaces.size() > 0) ? mSurfaces.get(0).eglSurface :
    672                 mConversionSurfaces.get(0).eglSurface);
    673         initializeGLState();
    674         mSurfaceTexture = new SurfaceTexture(getTextureId());
    675 
    676         // Set up performance tracking if enabled
    677         if (SystemProperties.getBoolean(LEGACY_PERF_PROPERTY, false)) {
    678             setupGlTiming();
    679         }
    680     }
    681 
    682     /**
    683      * Draw the current buffer in the {@link SurfaceTexture} returned from
    684      * {@link #getSurfaceTexture()} into the set of target {@link Surface}s
    685      * in the next request from the given {@link CaptureCollector}, or drop
    686      * the frame if none is available.
    687      *
    688      * <p>
    689      * Any {@link Surface}s targeted must be a subset of the {@link Surface}s
    690      * set in the last {@link #configureSurfaces(java.util.Collection)} call.
    691      * </p>
    692      *
    693      * @param targetCollector the surfaces to draw to.
    694      */
    695     public void drawIntoSurfaces(CaptureCollector targetCollector) {
    696         if ((mSurfaces == null || mSurfaces.size() == 0)
    697                 && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) {
    698             return;
    699         }
    700 
    701         boolean doTiming = targetCollector.hasPendingPreviewCaptures();
    702         checkGlError("before updateTexImage");
    703 
    704         if (doTiming) {
    705             beginGlTiming();
    706         }
    707 
    708         mSurfaceTexture.updateTexImage();
    709 
    710         long timestamp = mSurfaceTexture.getTimestamp();
    711 
    712         Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp);
    713 
    714         // No preview request queued, drop frame.
    715         if (captureHolder == null) {
    716             if (DEBUG) {
    717                 Log.d(TAG, "Dropping preview frame.");
    718             }
    719             if (doTiming) {
    720                 endGlTiming();
    721             }
    722             return;
    723         }
    724 
    725         RequestHolder request = captureHolder.first;
    726 
    727         Collection<Surface> targetSurfaces = request.getHolderTargets();
    728         if (doTiming) {
    729             addGlTimestamp(timestamp);
    730         }
    731 
    732         List<Long> targetSurfaceIds = new ArrayList();
    733         try {
    734             targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
    735         } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    736             Log.w(TAG, "Surface abandoned, dropping frame. ", e);
    737             request.setOutputAbandoned();
    738         }
    739 
    740         for (EGLSurfaceHolder holder : mSurfaces) {
    741             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
    742                 try{
    743                     LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
    744                             holder.height);
    745                     makeCurrent(holder.eglSurface);
    746 
    747                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
    748                     drawFrame(mSurfaceTexture, holder.width, holder.height,
    749                             (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
    750                                     FLIP_TYPE_HORIZONTAL : FLIP_TYPE_NONE);
    751                     swapBuffers(holder.eglSurface);
    752                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    753                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
    754                     request.setOutputAbandoned();
    755                 }
    756             }
    757         }
    758         for (EGLSurfaceHolder holder : mConversionSurfaces) {
    759             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
    760                 makeCurrent(holder.eglSurface);
    761                 // glReadPixels reads from the bottom of the buffer, so add an extra vertical flip
    762                 drawFrame(mSurfaceTexture, holder.width, holder.height,
    763                         (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ?
    764                                 FLIP_TYPE_BOTH : FLIP_TYPE_VERTICAL);
    765                 mPBufferPixels.clear();
    766                 GLES20.glReadPixels(/*x*/ 0, /*y*/ 0, holder.width, holder.height,
    767                         GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPBufferPixels);
    768                 checkGlError("glReadPixels");
    769 
    770                 try {
    771                     int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
    772                     LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
    773                             holder.height);
    774                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
    775                     LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
    776                             holder.width, holder.height, format);
    777                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
    778                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
    779                     request.setOutputAbandoned();
    780                 }
    781             }
    782         }
    783         targetCollector.previewProduced();
    784 
    785         if (doTiming) {
    786             endGlTiming();
    787         }
    788     }
    789 
    790     /**
    791      * Clean up the current GL context.
    792      */
    793     public void cleanupEGLContext() {
    794         releaseEGLContext();
    795     }
    796 
    797     /**
    798      * Drop all current GL operations on the floor.
    799      */
    800     public void flush() {
    801         // TODO: implement flush
    802         Log.e(TAG, "Flush not yet implemented.");
    803     }
    804 }
    805