1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts; 17 18 import android.media.AudioTrack; 19 import android.media.MediaCodec; 20 import android.media.MediaExtractor; 21 import android.media.MediaFormat; 22 import android.util.Log; 23 24 import java.nio.ByteBuffer; 25 import java.util.LinkedList; 26 27 /** 28 * Class for directly managing both audio and video playback by 29 * using {@link MediaCodec} and {@link AudioTrack}. 30 */ 31 public class CodecState { 32 private static final String TAG = CodecState.class.getSimpleName(); 33 34 private boolean mSawInputEOS, mSawOutputEOS; 35 private boolean mLimitQueueDepth; 36 private boolean mTunneled; 37 private boolean mIsAudio; 38 private int mAudioSessionId; 39 private ByteBuffer[] mCodecInputBuffers; 40 private ByteBuffer[] mCodecOutputBuffers; 41 private int mTrackIndex; 42 private LinkedList<Integer> mAvailableInputBufferIndices; 43 private LinkedList<Integer> mAvailableOutputBufferIndices; 44 private LinkedList<MediaCodec.BufferInfo> mAvailableOutputBufferInfos; 45 private long mPresentationTimeUs; 46 private long mSampleBaseTimeUs; 47 private MediaCodec mCodec; 48 private MediaTimeProvider mMediaTimeProvider; 49 private MediaExtractor mExtractor; 50 private MediaFormat mFormat; 51 private MediaFormat mOutputFormat; 52 private NonBlockingAudioTrack mAudioTrack; 53 54 /** 55 * Manages audio and video playback using MediaCodec and AudioTrack. 56 */ 57 public CodecState( 58 MediaTimeProvider mediaTimeProvider, 59 MediaExtractor extractor, 60 int trackIndex, 61 MediaFormat format, 62 MediaCodec codec, 63 boolean limitQueueDepth, 64 boolean tunneled, 65 int audioSessionId) { 66 mMediaTimeProvider = mediaTimeProvider; 67 mExtractor = extractor; 68 mTrackIndex = trackIndex; 69 mFormat = format; 70 mSawInputEOS = mSawOutputEOS = false; 71 mLimitQueueDepth = limitQueueDepth; 72 mTunneled = tunneled; 73 mAudioSessionId = audioSessionId; 74 mSampleBaseTimeUs = -1; 75 76 mCodec = codec; 77 78 mAvailableInputBufferIndices = new LinkedList<Integer>(); 79 mAvailableOutputBufferIndices = new LinkedList<Integer>(); 80 mAvailableOutputBufferInfos = new LinkedList<MediaCodec.BufferInfo>(); 81 82 mPresentationTimeUs = 0; 83 84 String mime = mFormat.getString(MediaFormat.KEY_MIME); 85 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 86 mIsAudio = mime.startsWith("audio/"); 87 } 88 89 public void release() { 90 mCodec.stop(); 91 mCodecInputBuffers = null; 92 mCodecOutputBuffers = null; 93 mOutputFormat = null; 94 95 mAvailableInputBufferIndices.clear(); 96 mAvailableOutputBufferIndices.clear(); 97 mAvailableOutputBufferInfos.clear(); 98 99 mAvailableInputBufferIndices = null; 100 mAvailableOutputBufferIndices = null; 101 mAvailableOutputBufferInfos = null; 102 103 mCodec.release(); 104 mCodec = null; 105 106 if (mAudioTrack != null) { 107 mAudioTrack.release(); 108 mAudioTrack = null; 109 } 110 } 111 112 public void start() { 113 mCodec.start(); 114 mCodecInputBuffers = mCodec.getInputBuffers(); 115 if (!mTunneled || mIsAudio) { 116 mCodecOutputBuffers = mCodec.getOutputBuffers(); 117 } 118 119 if (mAudioTrack != null) { 120 mAudioTrack.play(); 121 } 122 } 123 124 public void pause() { 125 if (mAudioTrack != null) { 126 mAudioTrack.pause(); 127 } 128 } 129 130 public long getCurrentPositionUs() { 131 return mPresentationTimeUs; 132 } 133 134 public void flush() { 135 mAvailableInputBufferIndices.clear(); 136 if (!mTunneled || mIsAudio) { 137 mAvailableOutputBufferIndices.clear(); 138 mAvailableOutputBufferInfos.clear(); 139 } 140 141 mSawInputEOS = false; 142 mSawOutputEOS = false; 143 144 if (mAudioTrack != null 145 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { 146 mAudioTrack.flush(); 147 } 148 149 mCodec.flush(); 150 } 151 152 public boolean isEnded() { 153 return mSawInputEOS && mSawOutputEOS; 154 } 155 156 /** 157 * doSomeWork() is the worker function that does all buffer handling and decoding works. 158 * It first reads data from {@link MediaExtractor} and pushes it into {@link MediaCodec}; 159 * it then dequeues buffer from {@link MediaCodec}, consumes it and pushes back to its own 160 * buffer queue for next round reading data from {@link MediaExtractor}. 161 */ 162 public void doSomeWork() { 163 int indexInput = mCodec.dequeueInputBuffer(0 /* timeoutUs */); 164 165 if (indexInput != MediaCodec.INFO_TRY_AGAIN_LATER) { 166 mAvailableInputBufferIndices.add(indexInput); 167 } 168 169 while (feedInputBuffer()) { 170 } 171 172 if (mIsAudio || !mTunneled) { 173 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 174 int indexOutput = mCodec.dequeueOutputBuffer(info, 0 /* timeoutUs */); 175 176 if (indexOutput == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 177 mOutputFormat = mCodec.getOutputFormat(); 178 onOutputFormatChanged(); 179 } else if (indexOutput == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 180 mCodecOutputBuffers = mCodec.getOutputBuffers(); 181 } else if (indexOutput != MediaCodec.INFO_TRY_AGAIN_LATER) { 182 mAvailableOutputBufferIndices.add(indexOutput); 183 mAvailableOutputBufferInfos.add(info); 184 } 185 186 while (drainOutputBuffer()) { 187 } 188 } 189 } 190 191 /** Returns true if more input data could be fed. */ 192 private boolean feedInputBuffer() throws MediaCodec.CryptoException, IllegalStateException { 193 if (mSawInputEOS || mAvailableInputBufferIndices.isEmpty()) { 194 return false; 195 } 196 197 // stalls read if audio queue is larger than 2MB full so we will not occupy too much heap 198 if (mLimitQueueDepth && mAudioTrack != null && 199 mAudioTrack.getNumBytesQueued() > 2 * 1024 * 1024) { 200 return false; 201 } 202 203 int index = mAvailableInputBufferIndices.peekFirst().intValue(); 204 205 ByteBuffer codecData = mCodecInputBuffers[index]; 206 207 int trackIndex = mExtractor.getSampleTrackIndex(); 208 209 if (trackIndex == mTrackIndex) { 210 int sampleSize = 211 mExtractor.readSampleData(codecData, 0 /* offset */); 212 213 long sampleTime = mExtractor.getSampleTime(); 214 215 int sampleFlags = mExtractor.getSampleFlags(); 216 217 if (sampleSize <= 0) { 218 Log.d(TAG, "sampleSize: " + sampleSize + " trackIndex:" + trackIndex + 219 " sampleTime:" + sampleTime + " sampleFlags:" + sampleFlags); 220 mSawInputEOS = true; 221 // FIX-ME: in tunneled mode we currently use input EOS as output EOS indicator 222 // we should use stream duration 223 if (mTunneled && !mIsAudio) { 224 mSawOutputEOS = true; 225 } 226 return false; 227 } 228 229 if (mTunneled && !mIsAudio) { 230 if (mSampleBaseTimeUs == -1) { 231 mSampleBaseTimeUs = sampleTime; 232 } 233 sampleTime -= mSampleBaseTimeUs; 234 // FIX-ME: in tunneled mode we currently use input buffer time 235 // as video presentation time. This is not accurate and should be fixed 236 mPresentationTimeUs = sampleTime; 237 } 238 239 if ((sampleFlags & MediaExtractor.SAMPLE_FLAG_ENCRYPTED) != 0) { 240 MediaCodec.CryptoInfo info = new MediaCodec.CryptoInfo(); 241 mExtractor.getSampleCryptoInfo(info); 242 243 mCodec.queueSecureInputBuffer( 244 index, 0 /* offset */, info, sampleTime, 0 /* flags */); 245 } else { 246 mCodec.queueInputBuffer( 247 index, 0 /* offset */, sampleSize, sampleTime, 0 /* flags */); 248 } 249 250 mAvailableInputBufferIndices.removeFirst(); 251 mExtractor.advance(); 252 253 return true; 254 } else if (trackIndex < 0) { 255 Log.d(TAG, "saw input EOS on track " + mTrackIndex); 256 257 mSawInputEOS = true; 258 // FIX-ME: in tunneled mode we currently use input EOS as output EOS indicator 259 // we should use stream duration 260 if (mTunneled && !mIsAudio) { 261 mSawOutputEOS = true; 262 } 263 264 mCodec.queueInputBuffer( 265 index, 0 /* offset */, 0 /* sampleSize */, 266 0 /* sampleTime */, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 267 268 mAvailableInputBufferIndices.removeFirst(); 269 } 270 271 return false; 272 } 273 274 private void onOutputFormatChanged() { 275 String mime = mOutputFormat.getString(MediaFormat.KEY_MIME); 276 // b/9250789 277 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 278 279 mIsAudio = false; 280 if (mime.startsWith("audio/")) { 281 mIsAudio = true; 282 int sampleRate = 283 mOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); 284 285 int channelCount = 286 mOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 287 288 Log.d(TAG, "CodecState::onOutputFormatChanged Audio" + 289 " sampleRate:" + sampleRate + " channels:" + channelCount); 290 // We do sanity check here after we receive data from MediaExtractor and before 291 // we pass them down to AudioTrack. If MediaExtractor works properly, this 292 // sanity-check is not necessary, however, in our tests, we found that there 293 // are a few cases where ch=0 and samplerate=0 were returned by MediaExtractor. 294 if (channelCount < 1 || channelCount > 8 || 295 sampleRate < 8000 || sampleRate > 128000) { 296 return; 297 } 298 mAudioTrack = new NonBlockingAudioTrack(sampleRate, channelCount, 299 mTunneled, mAudioSessionId); 300 mAudioTrack.play(); 301 } 302 303 if (mime.startsWith("video/")) { 304 int width = mOutputFormat.getInteger(MediaFormat.KEY_WIDTH); 305 int height = mOutputFormat.getInteger(MediaFormat.KEY_HEIGHT); 306 Log.d(TAG, "CodecState::onOutputFormatChanged Video" + 307 " width:" + width + " height:" + height); 308 } 309 } 310 311 /** Returns true if more output data could be drained. */ 312 private boolean drainOutputBuffer() { 313 if (mSawOutputEOS || mAvailableOutputBufferIndices.isEmpty()) { 314 return false; 315 } 316 317 int index = mAvailableOutputBufferIndices.peekFirst().intValue(); 318 MediaCodec.BufferInfo info = mAvailableOutputBufferInfos.peekFirst(); 319 320 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 321 Log.d(TAG, "saw output EOS on track " + mTrackIndex); 322 323 mSawOutputEOS = true; 324 325 // We need to stop the audio track so that all audio frames are played 326 // and the video codec can consume all of its data. 327 // After audio track stop, getAudioTimeUs will return 0. 328 if (mAudioTrack != null) { 329 mAudioTrack.stop(); 330 } 331 return false; 332 } 333 334 long realTimeUs = 335 mMediaTimeProvider.getRealTimeUsForMediaTime(info.presentationTimeUs); 336 337 long nowUs = mMediaTimeProvider.getNowUs(); 338 339 long lateUs = nowUs - realTimeUs; 340 341 if (mAudioTrack != null) { 342 ByteBuffer buffer = mCodecOutputBuffers[index]; 343 buffer.clear(); 344 ByteBuffer audioBuffer = ByteBuffer.allocate(buffer.remaining()); 345 audioBuffer.put(buffer); 346 audioBuffer.clear(); 347 348 mAudioTrack.write(audioBuffer, info.size, info.presentationTimeUs*1000); 349 350 mCodec.releaseOutputBuffer(index, false /* render */); 351 352 mPresentationTimeUs = info.presentationTimeUs; 353 354 mAvailableOutputBufferIndices.removeFirst(); 355 mAvailableOutputBufferInfos.removeFirst(); 356 return true; 357 } else { 358 // video 359 boolean render; 360 361 if (lateUs < -45000) { 362 // too early; 363 return false; 364 } else if (lateUs > 30000) { 365 Log.d(TAG, "video late by " + lateUs + " us."); 366 render = false; 367 } else { 368 render = true; 369 mPresentationTimeUs = info.presentationTimeUs; 370 } 371 372 mCodec.releaseOutputBuffer(index, render); 373 374 mAvailableOutputBufferIndices.removeFirst(); 375 mAvailableOutputBufferInfos.removeFirst(); 376 return true; 377 } 378 } 379 380 public long getAudioTimeUs() { 381 if (mAudioTrack == null) { 382 return 0; 383 } 384 385 return mAudioTrack.getAudioTimeUs(); 386 } 387 388 public void process() { 389 if (mAudioTrack != null) { 390 mAudioTrack.process(); 391 } 392 } 393 } 394
