1 /* 2 * Copyright 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef NUPLAYER2_RENDERER_H_ 18 19 #define NUPLAYER2_RENDERER_H_ 20 21 #include <media/AudioResamplerPublic.h> 22 #include <media/AVSyncSettings.h> 23 #include <mediaplayer2/JObjectHolder.h> 24 25 #include "NuPlayer2.h" 26 27 namespace android { 28 29 class JWakeLock; 30 struct MediaClock; 31 class MediaCodecBuffer; 32 struct VideoFrameSchedulerBase; 33 34 struct NuPlayer2::Renderer : public AHandler { 35 enum Flags { 36 FLAG_REAL_TIME = 1, 37 FLAG_OFFLOAD_AUDIO = 2, 38 }; 39 Renderer(const sp<MediaPlayer2Interface::AudioSink> &sink, 40 const sp<MediaClock> &mediaClock, 41 const sp<AMessage> ¬ify, 42 const sp<JObjectHolder> &context, 43 uint32_t flags = 0); 44 45 static size_t AudioSinkCallback( 46 MediaPlayer2Interface::AudioSink *audioSink, 47 void *data, size_t size, void *me, 48 MediaPlayer2Interface::AudioSink::cb_event_t event); 49 50 void queueBuffer( 51 bool audio, 52 const sp<MediaCodecBuffer> &buffer, 53 const sp<AMessage> ¬ifyConsumed); 54 55 void queueEOS(bool audio, status_t finalResult); 56 57 status_t setPlaybackSettings(const AudioPlaybackRate &rate /* sanitized */); 58 status_t getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 59 status_t setSyncSettings(const AVSyncSettings &sync, float videoFpsHint); 60 status_t getSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 61 62 void flush(bool audio, bool notifyComplete); 63 64 void signalTimeDiscontinuity(); 65 66 void signalDisableOffloadAudio(); 67 void signalEnableOffloadAudio(); 68 69 void pause(); 70 void resume(); 71 72 void setVideoFrameRate(float fps); 73 74 status_t getCurrentPosition(int64_t *mediaUs); 75 int64_t getVideoLateByUs(); 76 77 status_t openAudioSink( 78 const sp<AMessage> &format, 79 bool offloadOnly, 80 bool hasVideo, 81 uint32_t flags, 82 bool *isOffloaded, 83 bool isStreaming); 84 void closeAudioSink(); 85 86 // re-open audio sink after all pending audio buffers played. 87 void changeAudioFormat( 88 const sp<AMessage> &format, 89 bool offloadOnly, 90 bool hasVideo, 91 uint32_t flags, 92 bool isStreaming, 93 const sp<AMessage> ¬ify); 94 95 enum { 96 kWhatEOS = 'eos ', 97 kWhatFlushComplete = 'fluC', 98 kWhatPosition = 'posi', 99 kWhatVideoRenderingStart = 'vdrd', 100 kWhatMediaRenderingStart = 'mdrd', 101 kWhatAudioTearDown = 'adTD', 102 kWhatAudioOffloadPauseTimeout = 'aOPT', 103 }; 104 105 enum AudioTearDownReason { 106 kDueToError = 0, // Could restart with either offload or non-offload. 107 kDueToTimeout, 108 kForceNonOffload, // Restart only with non-offload. 109 }; 110 111 protected: 112 virtual ~Renderer(); 113 114 virtual void onMessageReceived(const sp<AMessage> &msg); 115 116 private: 117 enum { 118 kWhatDrainAudioQueue = 'draA', 119 kWhatDrainVideoQueue = 'draV', 120 kWhatPostDrainVideoQueue = 'pDVQ', 121 kWhatQueueBuffer = 'queB', 122 kWhatQueueEOS = 'qEOS', 123 kWhatConfigPlayback = 'cfPB', 124 kWhatConfigSync = 'cfSy', 125 kWhatGetPlaybackSettings = 'gPbS', 126 kWhatGetSyncSettings = 'gSyS', 127 kWhatFlush = 'flus', 128 kWhatPause = 'paus', 129 kWhatResume = 'resm', 130 kWhatOpenAudioSink = 'opnA', 131 kWhatCloseAudioSink = 'clsA', 132 kWhatChangeAudioFormat = 'chgA', 133 kWhatStopAudioSink = 'stpA', 134 kWhatDisableOffloadAudio = 'noOA', 135 kWhatEnableOffloadAudio = 'enOA', 136 kWhatSetVideoFrameRate = 'sVFR', 137 }; 138 139 // if mBuffer != nullptr, it's a buffer containing real data. 140 // else if mNotifyConsumed == nullptr, it's EOS. 141 // else it's a tag for re-opening audio sink in different format. 142 struct QueueEntry { 143 sp<MediaCodecBuffer> mBuffer; 144 sp<AMessage> mMeta; 145 sp<AMessage> mNotifyConsumed; 146 size_t mOffset; 147 status_t mFinalResult; 148 int32_t mBufferOrdinal; 149 }; 150 151 static const int64_t kMinPositionUpdateDelayUs; 152 153 sp<MediaPlayer2Interface::AudioSink> mAudioSink; 154 bool mUseVirtualAudioSink; 155 sp<AMessage> mNotify; 156 Mutex mLock; 157 uint32_t mFlags; 158 List<QueueEntry> mAudioQueue; 159 List<QueueEntry> mVideoQueue; 160 uint32_t mNumFramesWritten; 161 sp<VideoFrameSchedulerBase> mVideoScheduler; 162 163 bool mDrainAudioQueuePending; 164 bool mDrainVideoQueuePending; 165 int32_t mAudioQueueGeneration; 166 int32_t mVideoQueueGeneration; 167 int32_t mAudioDrainGeneration; 168 int32_t mVideoDrainGeneration; 169 int32_t mAudioEOSGeneration; 170 171 const sp<MediaClock> mMediaClock; 172 173 AudioPlaybackRate mPlaybackSettings; 174 AVSyncSettings mSyncSettings; 175 float mVideoFpsHint; 176 177 int64_t mAudioFirstAnchorTimeMediaUs; 178 int64_t mAnchorTimeMediaUs; 179 int64_t mAnchorNumFramesWritten; 180 int64_t mVideoLateByUs; 181 int64_t mNextVideoTimeMediaUs; 182 bool mHasAudio; 183 bool mHasVideo; 184 185 bool mNotifyCompleteAudio; 186 bool mNotifyCompleteVideo; 187 188 bool mSyncQueues; 189 190 // modified on only renderer's thread. 191 bool mPaused; 192 int64_t mPauseDrainAudioAllowedUs; // time when we can drain/deliver audio in pause mode. 193 194 bool mVideoSampleReceived; 195 bool mVideoRenderingStarted; 196 int32_t mVideoRenderingStartGeneration; 197 int32_t mAudioRenderingStartGeneration; 198 bool mRenderingDataDelivered; 199 200 int64_t mNextAudioClockUpdateTimeUs; 201 // the media timestamp of last audio sample right before EOS. 202 int64_t mLastAudioMediaTimeUs; 203 204 int32_t mAudioOffloadPauseTimeoutGeneration; 205 bool mAudioTornDown; 206 audio_offload_info_t mCurrentOffloadInfo; 207 208 struct PcmInfo { 209 audio_channel_mask_t mChannelMask; 210 audio_output_flags_t mFlags; 211 audio_format_t mFormat; 212 int32_t mNumChannels; 213 int32_t mSampleRate; 214 }; 215 PcmInfo mCurrentPcmInfo; 216 static const PcmInfo AUDIO_PCMINFO_INITIALIZER; 217 218 int32_t mTotalBuffersQueued; 219 int32_t mLastAudioBufferDrained; 220 bool mUseAudioCallback; 221 222 sp<JWakeLock> mWakeLock; 223 224 status_t getCurrentPositionOnLooper(int64_t *mediaUs); 225 status_t getCurrentPositionOnLooper( 226 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 227 bool getCurrentPositionIfPaused_l(int64_t *mediaUs); 228 status_t getCurrentPositionFromAnchor( 229 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 230 231 void notifyEOSCallback(); 232 size_t fillAudioBuffer(void *buffer, size_t size); 233 234 bool onDrainAudioQueue(); 235 void drainAudioQueueUntilLastEOS(); 236 int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); 237 void postDrainAudioQueue_l(int64_t delayUs = 0); 238 239 void clearAnchorTime(); 240 void clearAudioFirstAnchorTime_l(); 241 void setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs); 242 void setVideoLateByUs(int64_t lateUs); 243 244 void onNewAudioMediaTime(int64_t mediaTimeUs); 245 int64_t getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs); 246 247 void onDrainVideoQueue(); 248 void postDrainVideoQueue(); 249 250 void prepareForMediaRenderingStart_l(); 251 void notifyIfMediaRenderingStarted_l(); 252 253 void onQueueBuffer(const sp<AMessage> &msg); 254 void onQueueEOS(const sp<AMessage> &msg); 255 void onFlush(const sp<AMessage> &msg); 256 void onAudioSinkChanged(); 257 void onDisableOffloadAudio(); 258 void onEnableOffloadAudio(); 259 status_t onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */); 260 status_t onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 261 status_t onConfigSync(const AVSyncSettings &sync, float videoFpsHint); 262 status_t onGetSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 263 264 void onPause(); 265 void onResume(); 266 void onSetVideoFrameRate(float fps); 267 int32_t getQueueGeneration(bool audio); 268 int32_t getDrainGeneration(bool audio); 269 bool getSyncQueues(); 270 void onAudioTearDown(AudioTearDownReason reason); 271 status_t onOpenAudioSink( 272 const sp<AMessage> &format, 273 bool offloadOnly, 274 bool hasVideo, 275 uint32_t flags, 276 bool isStreaming); 277 void onCloseAudioSink(); 278 void onChangeAudioFormat(const sp<AMessage> &meta, const sp<AMessage> ¬ify); 279 280 void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); 281 void notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs = 0); 282 void notifyFlushComplete(bool audio); 283 void notifyPosition(); 284 void notifyVideoLateBy(int64_t lateByUs); 285 void notifyVideoRenderingStart(); 286 void notifyAudioTearDown(AudioTearDownReason reason); 287 288 void flushQueue(List<QueueEntry> *queue); 289 bool dropBufferIfStale(bool audio, const sp<AMessage> &msg); 290 void syncQueuesDone_l(); 291 292 bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; } 293 294 void startAudioOffloadPauseTimeout(); 295 void cancelAudioOffloadPauseTimeout(); 296 297 int64_t getDurationUsIfPlayedAtSampleRate(uint32_t numFrames); 298 299 DISALLOW_EVIL_CONSTRUCTORS(Renderer); 300 }; 301 302 } // namespace android 303 304 #endif // NUPLAYER2_RENDERER_H_ 305
