videosrc/java/MediaSource.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package android.filterpacks.videosrc;

import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.filterfw.core.Filter;
import android.filterfw.core.FilterContext;
import android.filterfw.core.Frame;
import android.filterfw.core.FrameFormat;
import android.filterfw.core.FrameManager;
import android.filterfw.core.GenerateFieldPort;
import android.filterfw.core.GenerateFinalPort;
import android.filterfw.core.GLFrame;
import android.filterfw.core.KeyValueMap;
import android.filterfw.core.MutableFrameFormat;
import android.filterfw.core.NativeFrame;
import android.filterfw.core.Program;
import android.filterfw.core.ShaderProgram;
import android.filterfw.format.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.os.ConditionVariable;
import android.opengl.Matrix;
import android.view.Surface;

import java.io.IOException;
import java.io.FileDescriptor;
import java.lang.IllegalArgumentException;
import java.util.List;
import java.util.Set;

import android.util.Log;

/**
 * @hide
 */
public class MediaSource extends Filter {

    /** User-visible parameters */

    /** The source URL for the media source. Can be an http: link to a remote
     * resource, or a file: link to a local media file
     */
    @GenerateFieldPort(name = "sourceUrl", hasDefault = true)
    private String mSourceUrl = "";

    /** An open asset file descriptor to a local media source. Default is null */
    @GenerateFieldPort(name = "sourceAsset", hasDefault = true)
    private AssetFileDescriptor mSourceAsset = null;

    /** Whether the media source is a URL or an asset file descriptor. Defaults
     * to false.
     */
    @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true)
    private boolean mSelectedIsUrl = false;

    /** Whether the filter will always wait for a new video frame, or whether it
     * will output an old frame again if a new frame isn't available. Defaults
     * to true.
     */
    @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
    private boolean mWaitForNewFrame = true;

    /** Whether the media source should loop automatically or not. Defaults to
     * true.
     */
    @GenerateFieldPort(name = "loop", hasDefault = true)
    private boolean mLooping = true;

    /** Volume control. Currently sound is piped directly to the speakers, so
     * this defaults to mute.
     */
    @GenerateFieldPort(name = "volume", hasDefault = true)
    private float mVolume = 0.f;

    /** Orientation. This controls the output orientation of the video. Valid
     * values are 0, 90, 180, 270
     */
    @GenerateFieldPort(name = "orientation", hasDefault = true)
    private int mOrientation = 0;

    private MediaPlayer mMediaPlayer;
    private GLFrame mMediaFrame;
    private SurfaceTexture mSurfaceTexture;
    private ShaderProgram mFrameExtractor;
    private MutableFrameFormat mOutputFormat;
    private int mWidth, mHeight;

    // Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT
    private static final int PREP_TIMEOUT = 100; // ms
    private static final int PREP_TIMEOUT_REPEAT = 100;
    private static final int NEWFRAME_TIMEOUT = 100; //ms
    private static final int NEWFRAME_TIMEOUT_REPEAT = 10;

    // This is an identity shader; not using the default identity
    // shader because reading from a SurfaceTexture requires the
    // GL_OES_EGL_image_external extension.
    private final String mFrameShader =
            "#extension GL_OES_EGL_image_external : require\n" +
            "precision mediump float;\n" +
            "uniform samplerExternalOES tex_sampler_0;\n" +
            "varying vec2 v_texcoord;\n" +
            "void main() {\n" +
            "  gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
            "}\n";

    // The following transforms enable rotation of the decoded source.
    // These are multiplied with the transform obtained from the
    // SurfaceTexture to get the final transform to be set on the media source.
    // Currently, given a device orientation, the MediaSource rotates in such a way
    // that the source is displayed upright. A particular use case
    // is "Background Replacement" feature in the Camera app
    // where the MediaSource rotates the source to align with the camera feed and pass it
    // on to the backdropper filter. The backdropper only does the blending
    // and does not have to do any rotation
    // (except for mirroring in case of front camera).
    // TODO: Currently the rotations are spread over a bunch of stages in the
    // pipeline. A cleaner design
    // could be to cast away all the rotation in a separate filter or attach a transform
    // to the frame so that MediaSource itself need not know about any rotation.
    private static final float[] mSourceCoords_0 = { 1, 1, 0, 1,
                                                     0, 1, 0, 1,
                                                     1, 0, 0, 1,
                                                     0, 0, 0, 1 };
    private static final float[] mSourceCoords_270 = { 0, 1, 0, 1,
                                                      0, 0, 0, 1,
                                                      1, 1, 0, 1,
                                                      1, 0, 0, 1 };
    private static final float[] mSourceCoords_180 = { 0, 0, 0, 1,
                                                       1, 0, 0, 1,
                                                       0, 1, 0, 1,
                                                       1, 1, 0, 1 };
    private static final float[] mSourceCoords_90 = { 1, 0, 0, 1,
                                                       1, 1, 0, 1,
                                                       0, 0, 0, 1,
                                                       0, 1, 0, 1 };

    private boolean mGotSize;
    private boolean mPrepared;
    private boolean mPlaying;
    private boolean mNewFrameAvailable;
    private boolean mOrientationUpdated;
    private boolean mPaused;
    private boolean mCompleted;

    private final boolean mLogVerbose;
    private static final String TAG = "MediaSource";

    public MediaSource(String name) {
        super(name);
        mNewFrameAvailable = false;

        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
    }

    @Override
    public void setupPorts() {
        // Add input port
        addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                                  FrameFormat.TARGET_GPU));
    }

    private void createFormats() {
        mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
                                           FrameFormat.TARGET_GPU);
    }

    @Override
    protected void prepare(FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Preparing MediaSource");

        mFrameExtractor = new ShaderProgram(context, mFrameShader);
        // SurfaceTexture defines (0,0) to be bottom-left. The filter framework
        // defines (0,0) as top-left, so do the flip here.
        mFrameExtractor.setSourceRect(0, 1, 1, -1);

        createFormats();
    }

    @Override
    public void open(FilterContext context) {
        if (mLogVerbose) {
            Log.v(TAG, "Opening MediaSource");
            if (mSelectedIsUrl) {
                Log.v(TAG, "Current URL is " + mSourceUrl);
            } else {
                Log.v(TAG, "Current source is Asset!");
            }
        }

        mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(
                mOutputFormat,
                GLFrame.EXTERNAL_TEXTURE,
                0);

        mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());

        if (!setupMediaPlayer(mSelectedIsUrl)) {
          throw new RuntimeException("Error setting up MediaPlayer!");
        }
    }

    @Override
    public void process(FilterContext context) {
        // Note: process is synchronized by its caller in the Filter base class
        if (mLogVerbose) Log.v(TAG, "Processing new frame");

        if (mMediaPlayer == null) {
            // Something went wrong in initialization or parameter updates
            throw new NullPointerException("Unexpected null media player!");
        }

        if (mCompleted) {
            // Video playback is done, so close us down
            closeOutputPort("video");
            return;
        }

        if (!mPlaying) {
            int waitCount = 0;
            if (mLogVerbose) Log.v(TAG, "Waiting for preparation to complete");
            while (!mGotSize || !mPrepared) {
                try {
                    this.wait(PREP_TIMEOUT);
                } catch (InterruptedException e) {
                    // ignoring
                }
                if (mCompleted) {
                    // Video playback is done, so close us down
                    closeOutputPort("video");
                    return;
                }
                waitCount++;
                if (waitCount == PREP_TIMEOUT_REPEAT) {
                    mMediaPlayer.release();
                    throw new RuntimeException("MediaPlayer timed out while preparing!");
                }
            }
            if (mLogVerbose) Log.v(TAG, "Starting playback");
            mMediaPlayer.start();
        }

        // Use last frame if paused, unless just starting playback, in which case
        // we want at least one valid frame before pausing
        if (!mPaused || !mPlaying) {
            if (mWaitForNewFrame) {
                if (mLogVerbose) Log.v(TAG, "Waiting for new frame");

                int waitCount = 0;
                while (!mNewFrameAvailable) {
                    if (waitCount == NEWFRAME_TIMEOUT_REPEAT) {
                        if (mCompleted) {
                            // Video playback is done, so close us down
                            closeOutputPort("video");
                            return;
                        } else {
                            throw new RuntimeException("Timeout waiting for new frame!");
                        }
                    }
                    try {
                        this.wait(NEWFRAME_TIMEOUT);
                    } catch (InterruptedException e) {
                        if (mLogVerbose) Log.v(TAG, "interrupted");
                        // ignoring
                    }
                    waitCount++;
                }
                mNewFrameAvailable = false;
                if (mLogVerbose) Log.v(TAG, "Got new frame");
            }

            mSurfaceTexture.updateTexImage();
            mOrientationUpdated = true;
        }
        if (mOrientationUpdated) {
            float[] surfaceTransform = new float[16];
            mSurfaceTexture.getTransformMatrix(surfaceTransform);

            float[] sourceCoords = new float[16];
            switch (mOrientation) {
                default:
                case 0:
                    Matrix.multiplyMM(sourceCoords, 0,
                                      surfaceTransform, 0,
                                      mSourceCoords_0, 0);
                    break;
                case 90:
                    Matrix.multiplyMM(sourceCoords, 0,
                                      surfaceTransform, 0,
                                      mSourceCoords_90, 0);
                    break;
                case 180:
                    Matrix.multiplyMM(sourceCoords, 0,
                                      surfaceTransform, 0,
                                      mSourceCoords_180, 0);
                    break;
                case 270:
                    Matrix.multiplyMM(sourceCoords, 0,
                                      surfaceTransform, 0,
                                      mSourceCoords_270, 0);
                    break;
            }
            if (mLogVerbose) {
                Log.v(TAG, "OrientationHint = " + mOrientation);
                String temp = String.format("SetSourceRegion: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f",
                        sourceCoords[4], sourceCoords[5],sourceCoords[0], sourceCoords[1],
                        sourceCoords[12], sourceCoords[13],sourceCoords[8], sourceCoords[9]);
                Log.v(TAG, temp);
            }
            mFrameExtractor.setSourceRegion(sourceCoords[4], sourceCoords[5],
                    sourceCoords[0], sourceCoords[1],
                    sourceCoords[12], sourceCoords[13],
                    sourceCoords[8], sourceCoords[9]);
            mOrientationUpdated = false;
        }

        Frame output = context.getFrameManager().newFrame(mOutputFormat);
        mFrameExtractor.process(mMediaFrame, output);

        long timestamp = mSurfaceTexture.getTimestamp();
        if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
        output.setTimestamp(timestamp);

        pushOutput("video", output);
        output.release();

        mPlaying = true;
    }

    @Override
    public void close(FilterContext context) {
        if (mMediaPlayer.isPlaying()) {
            mMediaPlayer.stop();
        }
        mPrepared = false;
        mGotSize = false;
        mPlaying = false;
        mPaused = false;
        mCompleted = false;
        mNewFrameAvailable = false;

        mMediaPlayer.release();
        mMediaPlayer = null;
        mSurfaceTexture.release();
        mSurfaceTexture = null;
        if (mLogVerbose) Log.v(TAG, "MediaSource closed");
    }

    @Override
    public void tearDown(FilterContext context) {
        if (mMediaFrame != null) {
            mMediaFrame.release();
        }
    }

    // When updating the port values of the filter, users can update sourceIsUrl to switch
    //   between using URL objects or Assets.
    // If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player
    //   uses Url objects/Asset.
    // Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch
    //   sourceIsUrl next time.
    @Override
    public void fieldPortValueUpdated(String name, FilterContext context) {
        if (mLogVerbose) Log.v(TAG, "Parameter update");
        if (name.equals("sourceUrl")) {
           if (isOpen()) {
                if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                if (mSelectedIsUrl) {
                    setupMediaPlayer(mSelectedIsUrl);
                }
            }
        } else if (name.equals("sourceAsset") ) {
            if (isOpen()) {
                if (mLogVerbose) Log.v(TAG, "Opening new source FD");
                if (!mSelectedIsUrl) {
                    setupMediaPlayer(mSelectedIsUrl);
                }
            }
        } else if (name.equals("loop")) {
            if (isOpen()) {
                mMediaPlayer.setLooping(mLooping);
            }
        } else if (name.equals("sourceIsUrl")) {
            if (isOpen()){
                if (mSelectedIsUrl){
                    if (mLogVerbose) Log.v(TAG, "Opening new source URL");
                } else {
                    if (mLogVerbose) Log.v(TAG, "Opening new source Asset");
                }
                setupMediaPlayer(mSelectedIsUrl);
            }
        } else if (name.equals("volume")) {
            if (isOpen()) {
                mMediaPlayer.setVolume(mVolume, mVolume);
            }
        } else if (name.equals("orientation") && mGotSize) {
            if (mOrientation == 0 || mOrientation == 180) {
                mOutputFormat.setDimensions(mWidth, mHeight);
            } else {
                mOutputFormat.setDimensions(mHeight, mWidth);
            }
            mOrientationUpdated = true;
        }
    }

    synchronized public void pauseVideo(boolean pauseState) {
        if (isOpen()) {
            if (pauseState && !mPaused) {
                mMediaPlayer.pause();
            } else if (!pauseState && mPaused) {
                mMediaPlayer.start();
            }
        }
        mPaused = pauseState;
    }

    /** Creates a media player, sets it up, and calls prepare */
    synchronized private boolean setupMediaPlayer(boolean useUrl) {
        mPrepared = false;
        mGotSize = false;
        mPlaying = false;
        mPaused = false;
        mCompleted = false;
        mNewFrameAvailable = false;

        if (mLogVerbose) Log.v(TAG, "Setting up playback.");

        if (mMediaPlayer != null) {
            // Clean up existing media players
            if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer.");
            mMediaPlayer.reset();
        } else {
            // Create new media player
            if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer.");
            mMediaPlayer = new MediaPlayer();
        }

        if (mMediaPlayer == null) {
            throw new RuntimeException("Unable to create a MediaPlayer!");
        }

        // Set up data sources, etc
        try {
            if (useUrl) {
                if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl);
                mMediaPlayer.setDataSource(mSourceUrl);
            } else {
                if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset);
                mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength());
            }
        } catch(IOException e) {
            mMediaPlayer.release();
            mMediaPlayer = null;
            if (useUrl) {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
            } else {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
            }
        } catch(IllegalArgumentException e) {
            mMediaPlayer.release();
            mMediaPlayer = null;
            if (useUrl) {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
            } else {
                throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
            }
        }

        mMediaPlayer.setLooping(mLooping);
        mMediaPlayer.setVolume(mVolume, mVolume);

        // Bind it to our media frame
        Surface surface = new Surface(mSurfaceTexture);
        mMediaPlayer.setSurface(surface);
        surface.release();

        // Connect Media Player to callbacks

        mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener);
        mMediaPlayer.setOnPreparedListener(onPreparedListener);
        mMediaPlayer.setOnCompletionListener(onCompletionListener);

        // Connect SurfaceTexture to callback
        mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener);

        if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer.");
        mMediaPlayer.prepareAsync();

        return true;
    }

    private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener =
            new MediaPlayer.OnVideoSizeChangedListener() {
        public void onVideoSizeChanged(MediaPlayer mp, int width, int height) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height);
            if (!mGotSize) {
                if (mOrientation == 0 || mOrientation == 180) {
                    mOutputFormat.setDimensions(width, height);
                } else {
                    mOutputFormat.setDimensions(height, width);
                }
                mWidth = width;
                mHeight = height;
            } else {
                if (mOutputFormat.getWidth() != width ||
                    mOutputFormat.getHeight() != height) {
                    Log.e(TAG, "Multiple video size change events received!");
                }
            }
            synchronized(MediaSource.this) {
                mGotSize = true;
                MediaSource.this.notify();
            }
        }
    };

    private MediaPlayer.OnPreparedListener onPreparedListener =
            new MediaPlayer.OnPreparedListener() {
        public void onPrepared(MediaPlayer mp) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared");
            synchronized(MediaSource.this) {
                mPrepared = true;
                MediaSource.this.notify();
            }
        }
    };

    private MediaPlayer.OnCompletionListener onCompletionListener =
            new MediaPlayer.OnCompletionListener() {
        public void onCompletion(MediaPlayer mp) {
            if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback");
            synchronized(MediaSource.this) {
                mCompleted = true;
            }
        }
    };

    private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener =
            new SurfaceTexture.OnFrameAvailableListener() {
        public void onFrameAvailable(SurfaceTexture surfaceTexture) {
            if (mLogVerbose) Log.v(TAG, "New frame from media player");
            synchronized(MediaSource.this) {
                if (mLogVerbose) Log.v(TAG, "New frame: notify");
                mNewFrameAvailable = true;
                MediaSource.this.notify();
                if (mLogVerbose) Log.v(TAG, "New frame: notify done");
            }
        }
    };

}