Home | History | Annotate | Download | only in decoder 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package androidx.media.filterfw.decoder;
     18 
     19 import android.annotation.TargetApi;
     20 import android.graphics.SurfaceTexture;
     21 import android.graphics.SurfaceTexture.OnFrameAvailableListener;
     22 import android.media.MediaCodec;
     23 import android.media.MediaCodec.BufferInfo;
     24 import android.media.MediaFormat;
     25 import android.view.Surface;
     26 
     27 import androidx.media.filterfw.FrameImage2D;
     28 import androidx.media.filterfw.ImageShader;
     29 import androidx.media.filterfw.TextureSource;
     30 
     31 import java.io.IOException;
     32 import java.nio.ByteBuffer;
     33 
     34 /**
     35  * {@link TrackDecoder} that decodes a video track and renders the frames onto a
     36  * {@link SurfaceTexture}.
     37  *
     38  * This implementation uses the GPU for image operations such as copying
     39  * and color-space conversion.
     40  */
     41 @TargetApi(16)
     42 public class GpuVideoTrackDecoder extends VideoTrackDecoder {
     43 
     44     /**
     45      * Identity fragment shader for external textures.
     46      */
     47     private static final String COPY_FRAGMENT_SHADER =
     48             "#extension GL_OES_EGL_image_external : require\n" +
     49             "precision mediump float;\n" +
     50             "uniform samplerExternalOES tex_sampler_0;\n" +
     51             "varying vec2 v_texcoord;\n" +
     52             "void main() {\n" +
     53             "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
     54             "}\n";
     55 
     56     private final TextureSource mTextureSource;
     57     private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor.
     58     private final float[] mTransformMatrix;
     59 
     60     private final int mOutputWidth;
     61     private final int mOutputHeight;
     62 
     63     private ImageShader mImageShader;
     64 
     65     private long mCurrentPresentationTimeUs;
     66 
     67     public GpuVideoTrackDecoder(
     68             int trackIndex, MediaFormat format, Listener listener) {
     69         super(trackIndex, format, listener);
     70 
     71         // Create a surface texture to be used by the video track decoder.
     72         mTextureSource = TextureSource.newExternalTexture();
     73         mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId());
     74         mSurfaceTexture.detachFromGLContext();
     75         mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() {
     76             @Override
     77             public void onFrameAvailable(SurfaceTexture surfaceTexture) {
     78                 markFrameAvailable();
     79             }
     80         });
     81 
     82         mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH);
     83         mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
     84 
     85         mTransformMatrix = new float[16];
     86     }
     87 
     88     @Override
     89     protected MediaCodec initMediaCodec(MediaFormat format) {
     90         MediaCodec mediaCodec;
     91         try {
     92             mediaCodec = MediaCodec.createDecoderByType(
     93                     format.getString(MediaFormat.KEY_MIME));
     94         } catch (IOException e) {
     95             throw new RuntimeException(
     96                     "failed to create decoder for "
     97                     + format.getString(MediaFormat.KEY_MIME), e);
     98         }
     99         Surface surface = new Surface(mSurfaceTexture);
    100         mediaCodec.configure(format, surface, null, 0);
    101         surface.release();
    102         return mediaCodec;
    103     }
    104 
    105     @Override
    106     protected boolean onDataAvailable(
    107             MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) {
    108         boolean textureAvailable = waitForFrameGrab();
    109 
    110         mCurrentPresentationTimeUs = info.presentationTimeUs;
    111 
    112         // Only render the next frame if we weren't interrupted.
    113         codec.releaseOutputBuffer(bufferIndex, textureAvailable);
    114 
    115         if (textureAvailable) {
    116             if (updateTexture()) {
    117                 notifyListener();
    118             }
    119         }
    120 
    121         return false;
    122     }
    123 
    124     /**
    125      * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates
    126      * the internal {@link SurfaceTexture}.
    127      */
    128     private boolean updateTexture() {
    129         // Wait for the frame we just released to appear in the texture.
    130         synchronized (mFrameMonitor) {
    131             try {
    132                 while (!mFrameAvailable) {
    133                     mFrameMonitor.wait();
    134                 }
    135                 mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId());
    136                 mSurfaceTexture.updateTexImage();
    137                 mSurfaceTexture.detachFromGLContext();
    138                 return true;
    139             } catch (InterruptedException e) {
    140                 return false;
    141             }
    142         }
    143     }
    144 
    145     @Override
    146     protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) {
    147         TextureSource targetTexture = TextureSource.newExternalTexture();
    148         mSurfaceTexture.attachToGLContext(targetTexture.getTextureId());
    149         mSurfaceTexture.getTransformMatrix(mTransformMatrix);
    150 
    151         ImageShader imageShader = getImageShader();
    152         imageShader.setSourceTransform(mTransformMatrix);
    153 
    154         int outputWidth = mOutputWidth;
    155         int outputHeight = mOutputHeight;
    156         if (rotation != 0) {
    157             float[] targetCoords = getRotationCoords(rotation);
    158             imageShader.setTargetCoords(targetCoords);
    159             if (needSwapDimension(rotation)) {
    160                 outputWidth = mOutputHeight;
    161                 outputHeight = mOutputWidth;
    162             }
    163         }
    164         outputVideoFrame.resize(new int[] { outputWidth, outputHeight });
    165         imageShader.process(
    166                 targetTexture,
    167                 outputVideoFrame.lockRenderTarget(),
    168                 outputWidth,
    169                 outputHeight);
    170         outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000);
    171         outputVideoFrame.unlock();
    172         targetTexture.release();
    173 
    174         mSurfaceTexture.detachFromGLContext();
    175     }
    176 
    177     @Override
    178     public void release() {
    179         super.release();
    180         synchronized (mFrameMonitor) {
    181             mTextureSource.release();
    182             mSurfaceTexture.release();
    183         }
    184     }
    185 
    186     /*
    187      * This method has to be called on the MFF processing thread.
    188      */
    189     private ImageShader getImageShader() {
    190         if (mImageShader == null) {
    191             mImageShader = new ImageShader(COPY_FRAGMENT_SHADER);
    192             mImageShader.setTargetRect(0f, 1f, 1f, -1f);
    193         }
    194         return mImageShader;
    195     }
    196 
    197     /**
    198      * Get the quad coords for rotation.
    199      * @param rotation applied to the frame, value is one of
    200      *   {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}
    201      * @return coords the calculated quad coords for the given rotation
    202      */
    203     private static float[] getRotationCoords(int rotation) {
    204          switch(rotation) {
    205              case MediaDecoder.ROTATE_90_RIGHT:
    206                  return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f };
    207              case MediaDecoder.ROTATE_180:
    208                  return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f };
    209              case MediaDecoder.ROTATE_90_LEFT:
    210                  return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f };
    211              case MediaDecoder.ROTATE_NONE:
    212                  return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f };
    213              default:
    214                  throw new IllegalArgumentException("Unsupported rotation angle.");
    215          }
    216      }
    217 
    218 }
    219