1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts; 17 18 import android.media.AudioTrack; 19 import android.media.MediaCodec; 20 import android.media.MediaExtractor; 21 import android.media.MediaFormat; 22 import android.util.Log; 23 24 import java.nio.ByteBuffer; 25 import java.util.LinkedList; 26 27 /** 28 * Class for directly managing both audio and video playback by 29 * using {@link MediaCodec} and {@link AudioTrack}. 30 */ 31 public class CodecState { 32 private static final String TAG = CodecState.class.getSimpleName(); 33 34 private boolean mSawInputEOS, mSawOutputEOS; 35 private boolean mLimitQueueDepth; 36 private boolean mTunneled; 37 private boolean mIsAudio; 38 private int mAudioSessionId; 39 private ByteBuffer[] mCodecInputBuffers; 40 private ByteBuffer[] mCodecOutputBuffers; 41 private int mTrackIndex; 42 private LinkedList<Integer> mAvailableInputBufferIndices; 43 private LinkedList<Integer> mAvailableOutputBufferIndices; 44 private LinkedList<MediaCodec.BufferInfo> mAvailableOutputBufferInfos; 45 private long mPresentationTimeUs; 46 private long mSampleBaseTimeUs; 47 private MediaCodec mCodec; 48 private MediaTimeProvider mMediaTimeProvider; 49 private MediaExtractor mExtractor; 50 private MediaFormat mFormat; 51 private MediaFormat mOutputFormat; 52 private NonBlockingAudioTrack mAudioTrack; 53 54 /** 55 * Manages audio and video playback using MediaCodec and AudioTrack. 56 */ 57 public CodecState( 58 MediaTimeProvider mediaTimeProvider, 59 MediaExtractor extractor, 60 int trackIndex, 61 MediaFormat format, 62 MediaCodec codec, 63 boolean limitQueueDepth, 64 boolean tunneled, 65 int audioSessionId) { 66 mMediaTimeProvider = mediaTimeProvider; 67 mExtractor = extractor; 68 mTrackIndex = trackIndex; 69 mFormat = format; 70 mSawInputEOS = mSawOutputEOS = false; 71 mLimitQueueDepth = limitQueueDepth; 72 mTunneled = tunneled; 73 mAudioSessionId = audioSessionId; 74 mSampleBaseTimeUs = -1; 75 76 mCodec = codec; 77 78 mAvailableInputBufferIndices = new LinkedList<Integer>(); 79 mAvailableOutputBufferIndices = new LinkedList<Integer>(); 80 mAvailableOutputBufferInfos = new LinkedList<MediaCodec.BufferInfo>(); 81 82 mPresentationTimeUs = 0; 83 84 String mime = mFormat.getString(MediaFormat.KEY_MIME); 85 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 86 mIsAudio = mime.startsWith("audio/"); 87 } 88 89 public void release() { 90 mCodec.stop(); 91 mCodecInputBuffers = null; 92 mCodecOutputBuffers = null; 93 mOutputFormat = null; 94 95 mAvailableInputBufferIndices.clear(); 96 mAvailableOutputBufferIndices.clear(); 97 mAvailableOutputBufferInfos.clear(); 98 99 mAvailableInputBufferIndices = null; 100 mAvailableOutputBufferIndices = null; 101 mAvailableOutputBufferInfos = null; 102 103 mCodec.release(); 104 mCodec = null; 105 106 if (mAudioTrack != null) { 107 mAudioTrack.release(); 108 mAudioTrack = null; 109 } 110 } 111 112 public void start() { 113 mCodec.start(); 114 mCodecInputBuffers = mCodec.getInputBuffers(); 115 if (!mTunneled || mIsAudio) { 116 mCodecOutputBuffers = mCodec.getOutputBuffers(); 117 } 118 119 if (mAudioTrack != null) { 120 mAudioTrack.play(); 121 } 122 } 123 124 public void pause() { 125 if (mAudioTrack != null) { 126 mAudioTrack.pause(); 127 } 128 } 129 130 public long getCurrentPositionUs() { 131 return mPresentationTimeUs; 132 } 133 134 public void flush() { 135 mAvailableInputBufferIndices.clear(); 136 if (!mTunneled || mIsAudio) { 137 mAvailableOutputBufferIndices.clear(); 138 mAvailableOutputBufferInfos.clear(); 139 } 140 141 mSawInputEOS = false; 142 mSawOutputEOS = false; 143 144 if (mAudioTrack != null 145 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { 146 mAudioTrack.flush(); 147 } 148 149 mCodec.flush(); 150 } 151 152 public boolean isEnded() { 153 return mSawInputEOS && mSawOutputEOS; 154 } 155 156 /** 157 * doSomeWork() is the worker function that does all buffer handling and decoding works. 158 * It first reads data from {@link MediaExtractor} and pushes it into {@link MediaCodec}; 159 * it then dequeues buffer from {@link MediaCodec}, consumes it and pushes back to its own 160 * buffer queue for next round reading data from {@link MediaExtractor}. 161 */ 162 public void doSomeWork() { 163 int indexInput = mCodec.dequeueInputBuffer(0 /* timeoutUs */); 164 165 if (indexInput != MediaCodec.INFO_TRY_AGAIN_LATER) { 166 mAvailableInputBufferIndices.add(indexInput); 167 } 168 169 while (feedInputBuffer()) { 170 } 171 172 if (mIsAudio || !mTunneled) { 173 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 174 int indexOutput = mCodec.dequeueOutputBuffer(info, 0 /* timeoutUs */); 175 176 if (indexOutput == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 177 mOutputFormat = mCodec.getOutputFormat(); 178 onOutputFormatChanged(); 179 } else if (indexOutput == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 180 mCodecOutputBuffers = mCodec.getOutputBuffers(); 181 } else if (indexOutput != MediaCodec.INFO_TRY_AGAIN_LATER) { 182 mAvailableOutputBufferIndices.add(indexOutput); 183 mAvailableOutputBufferInfos.add(info); 184 } 185 186 while (drainOutputBuffer()) { 187 } 188 } 189 } 190 191 /** Returns true if more input data could be fed. */ 192 private boolean feedInputBuffer() throws MediaCodec.CryptoException, IllegalStateException { 193 if (mSawInputEOS || mAvailableInputBufferIndices.isEmpty()) { 194 return false; 195 } 196 197 // stalls read if audio queue is larger than 2MB full so we will not occupy too much heap 198 if (mLimitQueueDepth && mAudioTrack != null && 199 mAudioTrack.getNumBytesQueued() > 2 * 1024 * 1024) { 200 return false; 201 } 202 203 int index = mAvailableInputBufferIndices.peekFirst().intValue(); 204 205 ByteBuffer codecData = mCodecInputBuffers[index]; 206 207 int trackIndex = mExtractor.getSampleTrackIndex(); 208 209 if (trackIndex == mTrackIndex) { 210 int sampleSize = 211 mExtractor.readSampleData(codecData, 0 /* offset */); 212 213 long sampleTime = mExtractor.getSampleTime(); 214 215 int sampleFlags = mExtractor.getSampleFlags(); 216 217 if (sampleSize <= 0) { 218 Log.d(TAG, "sampleSize: " + sampleSize + " trackIndex:" + trackIndex + 219 " sampleTime:" + sampleTime + " sampleFlags:" + sampleFlags); 220 mSawInputEOS = true; 221 // FIX-ME: in tunneled mode we currently use input EOS as output EOS indicator 222 // we should stream duration 223 if (mTunneled && !mIsAudio) { 224 mSawOutputEOS = true; 225 } 226 return false; 227 } 228 229 if (mTunneled && !mIsAudio) { 230 if (mSampleBaseTimeUs == -1) { 231 mSampleBaseTimeUs = sampleTime; 232 } 233 sampleTime -= mSampleBaseTimeUs; 234 // FIX-ME: in tunneled mode we currently use input buffer time 235 // as video presentation time. This is not accurate and should be fixed 236 mPresentationTimeUs = sampleTime; 237 } 238 239 if ((sampleFlags & MediaExtractor.SAMPLE_FLAG_ENCRYPTED) != 0) { 240 MediaCodec.CryptoInfo info = new MediaCodec.CryptoInfo(); 241 mExtractor.getSampleCryptoInfo(info); 242 243 mCodec.queueSecureInputBuffer( 244 index, 0 /* offset */, info, sampleTime, 0 /* flags */); 245 } else { 246 mCodec.queueInputBuffer( 247 index, 0 /* offset */, sampleSize, sampleTime, 0 /* flags */); 248 } 249 250 mAvailableInputBufferIndices.removeFirst(); 251 mExtractor.advance(); 252 253 return true; 254 } else if (trackIndex < 0) { 255 Log.d(TAG, "saw input EOS on track " + mTrackIndex); 256 257 mSawInputEOS = true; 258 259 mCodec.queueInputBuffer( 260 index, 0 /* offset */, 0 /* sampleSize */, 261 0 /* sampleTime */, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 262 263 mAvailableInputBufferIndices.removeFirst(); 264 } 265 266 return false; 267 } 268 269 private void onOutputFormatChanged() { 270 String mime = mOutputFormat.getString(MediaFormat.KEY_MIME); 271 // b/9250789 272 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 273 274 if (mime.startsWith("audio/")) { 275 int sampleRate = 276 mOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); 277 278 int channelCount = 279 mOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 280 281 Log.d(TAG, "CodecState::onOutputFormatChanged Audio" + 282 " sampleRate:" + sampleRate + " channels:" + channelCount); 283 // We do sanity check here after we receive data from MediaExtractor and before 284 // we pass them down to AudioTrack. If MediaExtractor works properly, this 285 // sanity-check is not necessary, however, in our tests, we found that there 286 // are a few cases where ch=0 and samplerate=0 were returned by MediaExtractor. 287 if (channelCount < 1 || channelCount > 8 || 288 sampleRate < 8000 || sampleRate > 128000) { 289 return; 290 } 291 mAudioTrack = new NonBlockingAudioTrack(sampleRate, channelCount, 292 mTunneled, mAudioSessionId); 293 mAudioTrack.play(); 294 } 295 296 if (mime.startsWith("video/")) { 297 int width = mOutputFormat.getInteger(MediaFormat.KEY_WIDTH); 298 int height = mOutputFormat.getInteger(MediaFormat.KEY_HEIGHT); 299 Log.d(TAG, "CodecState::onOutputFormatChanged Video" + 300 " width:" + width + " height:" + height); 301 } 302 } 303 304 /** Returns true if more output data could be drained. */ 305 private boolean drainOutputBuffer() { 306 if (mSawOutputEOS || mAvailableOutputBufferIndices.isEmpty()) { 307 return false; 308 } 309 310 int index = mAvailableOutputBufferIndices.peekFirst().intValue(); 311 MediaCodec.BufferInfo info = mAvailableOutputBufferInfos.peekFirst(); 312 313 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 314 Log.d(TAG, "saw output EOS on track " + mTrackIndex); 315 316 mSawOutputEOS = true; 317 318 return false; 319 } 320 321 long realTimeUs = 322 mMediaTimeProvider.getRealTimeUsForMediaTime(info.presentationTimeUs); 323 324 long nowUs = mMediaTimeProvider.getNowUs(); 325 326 long lateUs = nowUs - realTimeUs; 327 328 if (mAudioTrack != null) { 329 ByteBuffer buffer = mCodecOutputBuffers[index]; 330 buffer.clear(); 331 ByteBuffer audioBuffer = ByteBuffer.allocate(buffer.remaining()); 332 audioBuffer.put(buffer); 333 audioBuffer.clear(); 334 335 mAudioTrack.write(audioBuffer, info.size, info.presentationTimeUs*1000); 336 337 mCodec.releaseOutputBuffer(index, false /* render */); 338 339 mPresentationTimeUs = info.presentationTimeUs; 340 341 mAvailableOutputBufferIndices.removeFirst(); 342 mAvailableOutputBufferInfos.removeFirst(); 343 return true; 344 } else { 345 // video 346 boolean render; 347 348 if (lateUs < -45000) { 349 // too early; 350 return false; 351 } else if (lateUs > 30000) { 352 Log.d(TAG, "video late by " + lateUs + " us."); 353 render = false; 354 } else { 355 render = true; 356 mPresentationTimeUs = info.presentationTimeUs; 357 } 358 359 mCodec.releaseOutputBuffer(index, render); 360 361 mAvailableOutputBufferIndices.removeFirst(); 362 mAvailableOutputBufferInfos.removeFirst(); 363 return true; 364 } 365 } 366 367 public long getAudioTimeUs() { 368 if (mAudioTrack == null) { 369 return 0; 370 } 371 372 return mAudioTrack.getAudioTimeUs(); 373 } 374 375 public void process() { 376 if (mAudioTrack != null) { 377 mAudioTrack.process(); 378 } 379 } 380 } 381