1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.media.filterfw.decoder; 18 19 import android.annotation.TargetApi; 20 import android.content.Context; 21 import android.media.MediaExtractor; 22 import android.media.MediaFormat; 23 import android.media.MediaMetadataRetriever; 24 import android.net.Uri; 25 import android.os.Build; 26 import android.util.Log; 27 import androidx.media.filterfw.FrameImage2D; 28 import androidx.media.filterfw.FrameValue; 29 import androidx.media.filterfw.RenderTarget; 30 31 import java.util.concurrent.LinkedBlockingQueue; 32 33 @TargetApi(16) 34 public class MediaDecoder implements 35 Runnable, 36 TrackDecoder.Listener { 37 38 public interface Listener { 39 /** 40 * Notifies a listener when a decoded video frame is available. The listener should use 41 * {@link MediaDecoder#grabVideoFrame(FrameImage2D, int)} to grab the video data for this 42 * frame. 43 */ 44 void onVideoFrameAvailable(); 45 46 /** 47 * Notifies a listener when one or more audio samples are available. The listener should use 48 * {@link MediaDecoder#grabAudioSamples(FrameValue)} to grab the audio samples. 49 */ 50 void onAudioSamplesAvailable(); 51 52 /** 53 * Notifies a listener that decoding has started. This method is called on the decoder 54 * thread. 55 */ 56 void onDecodingStarted(); 57 58 /** 59 * Notifies a listener that decoding has stopped. This method is called on the decoder 60 * thread. 61 */ 62 void onDecodingStopped(); 63 64 /** 65 * Notifies a listener that an error occurred. If an error occurs, {@link MediaDecoder} is 66 * stopped and no more events are reported to this {@link Listener}'s callbacks. 67 * This method is called on the decoder thread. 68 */ 69 void onError(Exception e); 70 } 71 72 public static final int ROTATE_NONE = 0; 73 public static final int ROTATE_90_RIGHT = 90; 74 public static final int ROTATE_180 = 180; 75 public static final int ROTATE_90_LEFT = 270; 76 77 private static final String LOG_TAG = "MediaDecoder"; 78 private static final boolean DEBUG = false; 79 80 private static final int MAX_EVENTS = 32; 81 private static final int EVENT_START = 0; 82 private static final int EVENT_STOP = 1; 83 private static final int EVENT_EOF = 2; 84 85 private final Listener mListener; 86 private final Uri mUri; 87 private final Context mContext; 88 89 private final LinkedBlockingQueue<Integer> mEventQueue; 90 91 private final Thread mDecoderThread; 92 93 private MediaExtractor mMediaExtractor; 94 95 private RenderTarget mRenderTarget; 96 97 private int mDefaultRotation; 98 private int mVideoTrackIndex; 99 private int mAudioTrackIndex; 100 101 private VideoTrackDecoder mVideoTrackDecoder; 102 private AudioTrackDecoder mAudioTrackDecoder; 103 104 private boolean mStarted; 105 106 private long mStartMicros; 107 108 private boolean mOpenGLEnabled = true; 109 110 private boolean mSignaledEndOfInput; 111 private boolean mSeenEndOfAudioOutput; 112 private boolean mSeenEndOfVideoOutput; 113 114 public MediaDecoder(Context context, Uri uri, Listener listener) { 115 this(context, uri, 0, listener); 116 } 117 118 public MediaDecoder(Context context, Uri uri, long startMicros, Listener listener) { 119 if (context == null) { 120 throw new NullPointerException("context cannot be null"); 121 } 122 mContext = context; 123 124 if (uri == null) { 125 throw new NullPointerException("uri cannot be null"); 126 } 127 mUri = uri; 128 129 if (startMicros < 0) { 130 throw new IllegalArgumentException("startMicros cannot be negative"); 131 } 132 mStartMicros = startMicros; 133 134 if (listener == null) { 135 throw new NullPointerException("listener cannot be null"); 136 } 137 mListener = listener; 138 139 mEventQueue = new LinkedBlockingQueue<Integer>(MAX_EVENTS); 140 mDecoderThread = new Thread(this); 141 } 142 143 /** 144 * Set whether decoder may use OpenGL for decoding. 145 * 146 * This must be called before {@link #start()}. 147 * 148 * @param enabled flag whether to enable OpenGL decoding (default is true). 149 */ 150 public void setOpenGLEnabled(boolean enabled) { 151 // If event-queue already has events, we have started already. 152 if (mEventQueue.isEmpty()) { 153 mOpenGLEnabled = enabled; 154 } else { 155 throw new IllegalStateException( 156 "Must call setOpenGLEnabled() before calling start()!"); 157 } 158 } 159 160 /** 161 * Returns whether OpenGL is enabled for decoding. 162 * 163 * @return whether OpenGL is enabled for decoding. 164 */ 165 public boolean isOpenGLEnabled() { 166 return mOpenGLEnabled; 167 } 168 169 public void start() { 170 mEventQueue.offer(EVENT_START); 171 mDecoderThread.start(); 172 } 173 174 public void stop() { 175 stop(true); 176 } 177 178 private void stop(boolean manual) { 179 if (manual) { 180 mEventQueue.offer(EVENT_STOP); 181 mDecoderThread.interrupt(); 182 } else { 183 mEventQueue.offer(EVENT_EOF); 184 } 185 } 186 187 @Override 188 public void run() { 189 Integer event; 190 try { 191 while (true) { 192 event = mEventQueue.poll(); 193 boolean shouldStop = false; 194 if (event != null) { 195 switch (event) { 196 case EVENT_START: 197 onStart(); 198 break; 199 case EVENT_EOF: 200 if (mVideoTrackDecoder != null) { 201 mVideoTrackDecoder.waitForFrameGrab(); 202 } 203 // once the last frame has been grabbed, fall through and stop 204 case EVENT_STOP: 205 onStop(true); 206 shouldStop = true; 207 break; 208 } 209 } else if (mStarted) { 210 decode(); 211 } 212 if (shouldStop) { 213 break; 214 } 215 216 } 217 } catch (Exception e) { 218 mListener.onError(e); 219 onStop(false); 220 } 221 } 222 223 private void onStart() throws Exception { 224 if (mOpenGLEnabled) { 225 getRenderTarget().focus(); 226 } 227 228 mMediaExtractor = new MediaExtractor(); 229 mMediaExtractor.setDataSource(mContext, mUri, null); 230 231 mVideoTrackIndex = -1; 232 mAudioTrackIndex = -1; 233 234 for (int i = 0; i < mMediaExtractor.getTrackCount(); i++) { 235 MediaFormat format = mMediaExtractor.getTrackFormat(i); 236 if (DEBUG) { 237 Log.i(LOG_TAG, "Uri " + mUri + ", track " + i + ": " + format); 238 } 239 if (DecoderUtil.isVideoFormat(format) && mVideoTrackIndex == -1) { 240 mVideoTrackIndex = i; 241 } else if (DecoderUtil.isAudioFormat(format) && mAudioTrackIndex == -1) { 242 mAudioTrackIndex = i; 243 } 244 } 245 246 if (mVideoTrackIndex == -1 && mAudioTrackIndex == -1) { 247 throw new IllegalArgumentException( 248 "Couldn't find a video or audio track in the provided file"); 249 } 250 251 if (mVideoTrackIndex != -1) { 252 MediaFormat videoFormat = mMediaExtractor.getTrackFormat(mVideoTrackIndex); 253 mVideoTrackDecoder = mOpenGLEnabled 254 ? new GpuVideoTrackDecoder(mVideoTrackIndex, videoFormat, this) 255 : new CpuVideoTrackDecoder(mVideoTrackIndex, videoFormat, this); 256 mVideoTrackDecoder.init(); 257 mMediaExtractor.selectTrack(mVideoTrackIndex); 258 if (Build.VERSION.SDK_INT >= 17) { 259 retrieveDefaultRotation(); 260 } 261 } 262 263 if (mAudioTrackIndex != -1) { 264 MediaFormat audioFormat = mMediaExtractor.getTrackFormat(mAudioTrackIndex); 265 mAudioTrackDecoder = new AudioTrackDecoder(mAudioTrackIndex, audioFormat, this); 266 mAudioTrackDecoder.init(); 267 mMediaExtractor.selectTrack(mAudioTrackIndex); 268 } 269 270 if (mStartMicros > 0) { 271 mMediaExtractor.seekTo(mStartMicros, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); 272 } 273 274 mStarted = true; 275 mListener.onDecodingStarted(); 276 } 277 278 @TargetApi(17) 279 private void retrieveDefaultRotation() { 280 MediaMetadataRetriever metadataRetriever = new MediaMetadataRetriever(); 281 metadataRetriever.setDataSource(mContext, mUri); 282 String rotationString = metadataRetriever.extractMetadata( 283 MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION); 284 mDefaultRotation = rotationString == null ? 0 : Integer.parseInt(rotationString); 285 } 286 287 private void onStop(boolean notifyListener) { 288 mMediaExtractor.release(); 289 mMediaExtractor = null; 290 291 if (mVideoTrackDecoder != null) { 292 mVideoTrackDecoder.release(); 293 mVideoTrackDecoder = null; 294 } 295 296 if (mAudioTrackDecoder != null) { 297 mAudioTrackDecoder.release(); 298 mAudioTrackDecoder = null; 299 } 300 301 if (mOpenGLEnabled) { 302 if (mRenderTarget != null) { 303 getRenderTarget().release(); 304 } 305 RenderTarget.focusNone(); 306 } 307 308 mVideoTrackIndex = -1; 309 mAudioTrackIndex = -1; 310 311 mEventQueue.clear(); 312 mStarted = false; 313 if (notifyListener) { 314 mListener.onDecodingStopped(); 315 } 316 } 317 318 private void decode() { 319 int sampleTrackIndex = mMediaExtractor.getSampleTrackIndex(); 320 if (sampleTrackIndex >= 0) { 321 if (sampleTrackIndex == mVideoTrackIndex) { 322 mVideoTrackDecoder.feedInput(mMediaExtractor); 323 } else if (sampleTrackIndex == mAudioTrackIndex) { 324 mAudioTrackDecoder.feedInput(mMediaExtractor); 325 } 326 } else if (!mSignaledEndOfInput) { 327 if (mVideoTrackDecoder != null) { 328 mVideoTrackDecoder.signalEndOfInput(); 329 } 330 if (mAudioTrackDecoder != null) { 331 mAudioTrackDecoder.signalEndOfInput(); 332 } 333 mSignaledEndOfInput = true; 334 } 335 336 if (mVideoTrackDecoder != null) { 337 mVideoTrackDecoder.drainOutputBuffer(); 338 } 339 if (mAudioTrackDecoder != null) { 340 mAudioTrackDecoder.drainOutputBuffer(); 341 } 342 } 343 344 /** 345 * Fills the argument frame with the video data, using the rotation hint obtained from the 346 * file's metadata, if any. 347 * 348 * @see #grabVideoFrame(FrameImage2D, int) 349 */ 350 public void grabVideoFrame(FrameImage2D outputVideoFrame) { 351 grabVideoFrame(outputVideoFrame, mDefaultRotation); 352 } 353 354 /** 355 * Fills the argument frame with the video data, the frame will be returned with the given 356 * rotation applied. 357 * 358 * @param outputVideoFrame the output video frame. 359 * @param videoRotation the rotation angle that is applied to the raw decoded frame. 360 * Value is one of {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT}. 361 */ 362 public void grabVideoFrame(FrameImage2D outputVideoFrame, int videoRotation) { 363 if (mVideoTrackDecoder != null && outputVideoFrame != null) { 364 mVideoTrackDecoder.grabFrame(outputVideoFrame, videoRotation); 365 } 366 } 367 368 /** 369 * Fills the argument frame with the audio data. 370 * 371 * @param outputAudioFrame the output audio frame. 372 */ 373 public void grabAudioSamples(FrameValue outputAudioFrame) { 374 if (mAudioTrackDecoder != null) { 375 if (outputAudioFrame != null) { 376 mAudioTrackDecoder.grabSample(outputAudioFrame); 377 } else { 378 mAudioTrackDecoder.clearBuffer(); 379 } 380 } 381 } 382 383 /** 384 * Gets the duration, in nanoseconds. 385 */ 386 public long getDuration() { 387 if (!mStarted) { 388 throw new IllegalStateException("MediaDecoder has not been started"); 389 } 390 391 MediaFormat mediaFormat = mMediaExtractor.getTrackFormat( 392 mVideoTrackIndex != -1 ? mVideoTrackIndex : mAudioTrackIndex); 393 return mediaFormat.getLong(MediaFormat.KEY_DURATION) * 1000; 394 } 395 396 private RenderTarget getRenderTarget() { 397 if (mRenderTarget == null) { 398 mRenderTarget = RenderTarget.newTarget(1, 1); 399 } 400 return mRenderTarget; 401 } 402 403 @Override 404 public void onDecodedOutputAvailable(TrackDecoder decoder) { 405 if (decoder == mVideoTrackDecoder) { 406 mListener.onVideoFrameAvailable(); 407 } else if (decoder == mAudioTrackDecoder) { 408 mListener.onAudioSamplesAvailable(); 409 } 410 } 411 412 @Override 413 public void onEndOfStream(TrackDecoder decoder) { 414 if (decoder == mAudioTrackDecoder) { 415 mSeenEndOfAudioOutput = true; 416 } else if (decoder == mVideoTrackDecoder) { 417 mSeenEndOfVideoOutput = true; 418 } 419 420 if ((mAudioTrackDecoder == null || mSeenEndOfAudioOutput) 421 && (mVideoTrackDecoder == null || mSeenEndOfVideoOutput)) { 422 stop(false); 423 } 424 } 425 426 } 427