Home | History | Annotate | Download | only in decoder 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package androidx.media.filterfw.decoder;
     18 
     19 import android.annotation.TargetApi;
     20 import android.graphics.SurfaceTexture;
     21 import android.graphics.SurfaceTexture.OnFrameAvailableListener;
     22 import android.media.MediaCodec;
     23 import android.media.MediaCodec.BufferInfo;
     24 import android.media.MediaFormat;
     25 import android.view.Surface;
     26 
     27 import androidx.media.filterfw.FrameImage2D;
     28 import androidx.media.filterfw.ImageShader;
     29 import androidx.media.filterfw.TextureSource;
     30 
     31 import java.nio.ByteBuffer;
     32 
     33 /**
     34  * {@link TrackDecoder} that decodes a video track and renders the frames onto a
     35  * {@link SurfaceTexture}.
     36  *
     37  * This implementation uses the GPU for image operations such as copying
     38  * and color-space conversion.
     39  */
     40 @TargetApi(16)
     41 public class GpuVideoTrackDecoder extends VideoTrackDecoder {
     42 
     43     /**
     44      * Identity fragment shader for external textures.
     45      */
     46     private static final String COPY_FRAGMENT_SHADER =
     47             "#extension GL_OES_EGL_image_external : require\n" +
     48             "precision mediump float;\n" +
     49             "uniform samplerExternalOES tex_sampler_0;\n" +
     50             "varying vec2 v_texcoord;\n" +
     51             "void main() {\n" +
     52             "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
     53             "}\n";
     54 
     55     private final TextureSource mTextureSource;
     56     private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor.
     57     private final float[] mTransformMatrix;
     58 
     59     private final int mOutputWidth;
     60     private final int mOutputHeight;
     61 
     62     private ImageShader mImageShader;
     63 
     64     private long mCurrentPresentationTimeUs;
     65 
     66     public GpuVideoTrackDecoder(
     67             int trackIndex, MediaFormat format, Listener listener) {
     68         super(trackIndex, format, listener);
     69 
     70         // Create a surface texture to be used by the video track decoder.
     71         mTextureSource = TextureSource.newExternalTexture();
     72         mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId());
     73         mSurfaceTexture.detachFromGLContext();
     74         mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() {
     75             @Override
     76             public void onFrameAvailable(SurfaceTexture surfaceTexture) {
     77                 markFrameAvailable();
     78             }
     79         });
     80 
     81         mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH);
     82         mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
     83 
     84         mTransformMatrix = new float[16];
     85     }
     86 
     87     @Override
     88     protected MediaCodec initMediaCodec(MediaFormat format) {
     89         Surface surface = new Surface(mSurfaceTexture);
     90         MediaCodec mediaCodec = MediaCodec.createDecoderByType(
     91                 format.getString(MediaFormat.KEY_MIME));
     92         mediaCodec.configure(format, surface, null, 0);
     93         surface.release();
     94         return mediaCodec;
     95     }
     96 
     97     @Override
     98     protected boolean onDataAvailable(
     99             MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
    100         boolean textureAvailable = waitForFrameGrab();
    101 
    102         mCurrentPresentationTimeUs = info.presentationTimeUs;
    103 
    104         // Only render the next frame if we weren't interrupted.
    105         codec.releaseOutputBuffer(bufferIndex, textureAvailable);
    106 
    107         if (textureAvailable) {
    108             if (updateTexture()) {
    109                 notifyListener();
    110             }
    111         }
    112 
    113         return false;
    114     }
    115 
    116     /**
    117      * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates
    118      * the internal {@link SurfaceTexture}.
    119      */
    120     private boolean updateTexture() {
    121         // Wait for the frame we just released to appear in the texture.
    122         synchronized (mFrameMonitor) {
    123             try {
    124                 while (!mFrameAvailable) {
    125                     mFrameMonitor.wait();
    126                 }
    127                 mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId());
    128                 mSurfaceTexture.updateTexImage();
    129                 mSurfaceTexture.detachFromGLContext();
    130                 return true;
    131             } catch (InterruptedException e) {
    132                 return false;
    133             }
    134         }
    135     }
    136 
    137     @Override
    138     protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
    139         TextureSource targetTexture = TextureSource.newExternalTexture();
    140         mSurfaceTexture.attachToGLContext(targetTexture.getTextureId());
    141         mSurfaceTexture.getTransformMatrix(mTransformMatrix);
    142 
    143         ImageShader imageShader = getImageShader();
    144         imageShader.setSourceTransform(mTransformMatrix);
    145 
    146         int outputWidth = mOutputWidth;
    147         int outputHeight = mOutputHeight;
    148         if (rotation != 0) {
    149             float[] targetCoords = getRotationCoords(rotation);
    150             imageShader.setTargetCoords(targetCoords);
    151             if (needSwapDimension(rotation)) {
    152                 outputWidth = mOutputHeight;
    153                 outputHeight = mOutputWidth;
    154             }
    155         }
    156         outputVideoFrame.resize(new int[] { outputWidth, outputHeight });
    157         imageShader.process(
    158                 targetTexture,
    159                 outputVideoFrame.lockRenderTarget(),
    160                 outputWidth,
    161                 outputHeight);
    162         outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
    163         outputVideoFrame.unlock();
    164         targetTexture.release();
    165 
    166         mSurfaceTexture.detachFromGLContext();
    167     }
    168 
    169     @Override
    170     public void release() {
    171         super.release();
    172         synchronized (mFrameMonitor) {
    173             mTextureSource.release();
    174             mSurfaceTexture.release();
    175         }
    176     }
    177 
    178     /*
    179      * This method has to be called on the MFF processing thread.
    180      */
    181     private ImageShader getImageShader() {
    182         if (mImageShader == null) {
    183             mImageShader = new ImageShader(COPY_FRAGMENT_SHADER);
    184             mImageShader.setTargetRect(0f, 1f, 1f, -1f);
    185         }
    186         return mImageShader;
    187     }
    188 
    189     /**
    190      * Get the quad coords for rotation.
    191      * @param rotation applied to the frame, value is one of
    192      *   {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}
    193      * @return coords the calculated quad coords for the given rotation
    194      */
    195     private static float[] getRotationCoords(int rotation) {
    196          switch(rotation) {
    197              case MediaDecoder.ROTATE_90_RIGHT:
    198                  return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f };
    199              case MediaDecoder.ROTATE_180:
    200                  return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f };
    201              case MediaDecoder.ROTATE_90_LEFT:
    202                  return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f };
    203              case MediaDecoder.ROTATE_NONE:
    204                  return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f };
    205              default:
    206                  throw new IllegalArgumentException("Unsupported rotation angle.");
    207          }
    208      }
    209 
    210 }
    211