1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/renderer/pepper/pepper_media_stream_audio_track_host.h" 6 7 #include <algorithm> 8 9 #include "base/bind.h" 10 #include "base/location.h" 11 #include "base/logging.h" 12 #include "base/macros.h" 13 #include "base/message_loop/message_loop_proxy.h" 14 #include "base/numerics/safe_math.h" 15 #include "ppapi/c/pp_errors.h" 16 #include "ppapi/c/ppb_audio_buffer.h" 17 #include "ppapi/host/dispatch_host_message.h" 18 #include "ppapi/host/host_message_context.h" 19 #include "ppapi/host/ppapi_host.h" 20 #include "ppapi/proxy/ppapi_messages.h" 21 #include "ppapi/shared_impl/media_stream_audio_track_shared.h" 22 #include "ppapi/shared_impl/media_stream_buffer.h" 23 24 using media::AudioParameters; 25 using ppapi::host::HostMessageContext; 26 using ppapi::MediaStreamAudioTrackShared; 27 28 namespace { 29 30 // Audio buffer durations in milliseconds. 31 const uint32_t kMinDuration = 10; 32 const uint32_t kDefaultDuration = 10; 33 34 const int32_t kDefaultNumberOfBuffers = 4; 35 const int32_t kMaxNumberOfBuffers = 1000; // 10 sec 36 37 // Returns true if the |sample_rate| is supported in 38 // |PP_AudioBuffer_SampleRate|, otherwise false. 39 PP_AudioBuffer_SampleRate GetPPSampleRate(int sample_rate) { 40 switch (sample_rate) { 41 case 8000: 42 return PP_AUDIOBUFFER_SAMPLERATE_8000; 43 case 16000: 44 return PP_AUDIOBUFFER_SAMPLERATE_16000; 45 case 22050: 46 return PP_AUDIOBUFFER_SAMPLERATE_22050; 47 case 32000: 48 return PP_AUDIOBUFFER_SAMPLERATE_32000; 49 case 44100: 50 return PP_AUDIOBUFFER_SAMPLERATE_44100; 51 case 48000: 52 return PP_AUDIOBUFFER_SAMPLERATE_48000; 53 case 96000: 54 return PP_AUDIOBUFFER_SAMPLERATE_96000; 55 case 192000: 56 return PP_AUDIOBUFFER_SAMPLERATE_192000; 57 default: 58 return PP_AUDIOBUFFER_SAMPLERATE_UNKNOWN; 59 } 60 } 61 62 } // namespace 63 64 namespace content { 65 66 PepperMediaStreamAudioTrackHost::AudioSink::AudioSink( 67 PepperMediaStreamAudioTrackHost* host) 68 : host_(host), 69 buffer_data_size_(0), 70 active_buffer_index_(-1), 71 active_buffers_generation_(0), 72 active_buffer_offset_(0), 73 buffers_generation_(0), 74 main_message_loop_proxy_(base::MessageLoopProxy::current()), 75 number_of_buffers_(kDefaultNumberOfBuffers), 76 bytes_per_second_(0), 77 user_buffer_duration_(kDefaultDuration), 78 weak_factory_(this) {} 79 80 PepperMediaStreamAudioTrackHost::AudioSink::~AudioSink() { 81 DCHECK_EQ(main_message_loop_proxy_, base::MessageLoopProxy::current()); 82 } 83 84 void PepperMediaStreamAudioTrackHost::AudioSink::EnqueueBuffer(int32_t index) { 85 DCHECK_EQ(main_message_loop_proxy_, base::MessageLoopProxy::current()); 86 DCHECK_GE(index, 0); 87 DCHECK_LT(index, host_->buffer_manager()->number_of_buffers()); 88 base::AutoLock lock(lock_); 89 buffers_.push_back(index); 90 } 91 92 int32_t PepperMediaStreamAudioTrackHost::AudioSink::Configure( 93 int32_t number_of_buffers, int32_t duration, 94 const ppapi::host::ReplyMessageContext& context) { 95 DCHECK_EQ(main_message_loop_proxy_, base::MessageLoopProxy::current()); 96 97 if (pending_configure_reply_.is_valid()) { 98 return PP_ERROR_INPROGRESS; 99 } 100 pending_configure_reply_ = context; 101 102 bool changed = false; 103 if (number_of_buffers != number_of_buffers_) 104 changed = true; 105 if (duration != 0 && duration != user_buffer_duration_) { 106 user_buffer_duration_ = duration; 107 changed = true; 108 } 109 number_of_buffers_ = number_of_buffers; 110 111 if (changed) { 112 // Initialize later in OnSetFormat if bytes_per_second_ is not known yet. 113 if (bytes_per_second_ > 0 && bytes_per_frame_ > 0) 114 InitBuffers(); 115 } else { 116 SendConfigureReply(PP_OK); 117 } 118 return PP_OK_COMPLETIONPENDING; 119 } 120 121 void PepperMediaStreamAudioTrackHost::AudioSink::SendConfigureReply( 122 int32_t result) { 123 if (pending_configure_reply_.is_valid()) { 124 pending_configure_reply_.params.set_result(result); 125 host_->host()->SendReply( 126 pending_configure_reply_, 127 PpapiPluginMsg_MediaStreamAudioTrack_ConfigureReply()); 128 pending_configure_reply_ = ppapi::host::ReplyMessageContext(); 129 } 130 } 131 132 void PepperMediaStreamAudioTrackHost::AudioSink::SetFormatOnMainThread( 133 int bytes_per_second, int bytes_per_frame) { 134 bytes_per_second_ = bytes_per_second; 135 bytes_per_frame_ = bytes_per_frame; 136 InitBuffers(); 137 } 138 139 void PepperMediaStreamAudioTrackHost::AudioSink::InitBuffers() { 140 DCHECK_EQ(main_message_loop_proxy_, base::MessageLoopProxy::current()); 141 { 142 base::AutoLock lock(lock_); 143 // Clear |buffers_|, so the audio thread will drop all incoming audio data. 144 buffers_.clear(); 145 buffers_generation_++; 146 } 147 int32_t frame_rate = bytes_per_second_ / bytes_per_frame_; 148 base::CheckedNumeric<int32_t> frames_per_buffer = user_buffer_duration_; 149 frames_per_buffer *= frame_rate; 150 frames_per_buffer /= base::Time::kMillisecondsPerSecond; 151 base::CheckedNumeric<int32_t> buffer_audio_size = 152 frames_per_buffer * bytes_per_frame_; 153 // The size is slightly bigger than necessary, because 8 extra bytes are 154 // added into the struct. Also see |MediaStreamBuffer|. Also, the size of the 155 // buffer may be larger than requested, since the size of each buffer will be 156 // 4-byte aligned. 157 base::CheckedNumeric<int32_t> buffer_size = buffer_audio_size; 158 buffer_size += sizeof(ppapi::MediaStreamBuffer::Audio); 159 DCHECK_GT(buffer_size.ValueOrDie(), 0); 160 161 // We don't need to hold |lock_| during |host->InitBuffers()| call, because 162 // we just cleared |buffers_| , so the audio thread will drop all incoming 163 // audio data, and not use buffers in |host_|. 164 bool result = host_->InitBuffers(number_of_buffers_, 165 buffer_size.ValueOrDie(), 166 kRead); 167 if (!result) { 168 SendConfigureReply(PP_ERROR_NOMEMORY); 169 return; 170 } 171 172 // Fill the |buffers_|, so the audio thread can continue receiving audio data. 173 base::AutoLock lock(lock_); 174 output_buffer_size_ = buffer_audio_size.ValueOrDie(); 175 for (int32_t i = 0; i < number_of_buffers_; ++i) { 176 int32_t index = host_->buffer_manager()->DequeueBuffer(); 177 DCHECK_GE(index, 0); 178 buffers_.push_back(index); 179 } 180 181 SendConfigureReply(PP_OK); 182 } 183 184 void PepperMediaStreamAudioTrackHost::AudioSink:: 185 SendEnqueueBufferMessageOnMainThread(int32_t index, 186 int32_t buffers_generation) { 187 DCHECK_EQ(main_message_loop_proxy_, base::MessageLoopProxy::current()); 188 // If |InitBuffers()| is called after this task being posted from the audio 189 // thread, the buffer should become invalid already. We should ignore it. 190 // And because only the main thread modifies the |buffers_generation_|, 191 // so we don't need to lock |lock_| here (main thread). 192 if (buffers_generation == buffers_generation_) 193 host_->SendEnqueueBufferMessageToPlugin(index); 194 } 195 196 void PepperMediaStreamAudioTrackHost::AudioSink::OnData(const int16* audio_data, 197 int sample_rate, 198 int number_of_channels, 199 int number_of_frames) { 200 DCHECK(audio_thread_checker_.CalledOnValidThread()); 201 DCHECK(audio_data); 202 DCHECK_EQ(sample_rate, audio_params_.sample_rate()); 203 DCHECK_EQ(number_of_channels, audio_params_.channels()); 204 // Here, |number_of_frames| and |audio_params_.frames_per_buffer()| refer to 205 // the incomming audio buffer. However, this doesn't necessarily equal 206 // |buffer->number_of_samples|, which is configured by the user when they set 207 // buffer duration. 208 DCHECK_EQ(number_of_frames, audio_params_.frames_per_buffer()); 209 210 const uint32_t bytes_per_frame = number_of_channels * 211 audio_params_.bits_per_sample() / 8; 212 213 int frames_remaining = number_of_frames; 214 base::TimeDelta timestamp_offset; 215 216 base::AutoLock lock(lock_); 217 while (frames_remaining) { 218 if (active_buffers_generation_ != buffers_generation_) { 219 // Buffers have changed, so drop the active buffer. 220 active_buffer_index_ = -1; 221 } 222 if (active_buffer_index_ == -1 && !buffers_.empty()) { 223 active_buffers_generation_ = buffers_generation_; 224 active_buffer_offset_ = 0; 225 active_buffer_index_ = buffers_.front(); 226 buffers_.pop_front(); 227 } 228 if (active_buffer_index_ == -1) { 229 // Eek! We're dropping frames. Bad, bad, bad! 230 break; 231 } 232 233 // TODO(penghuang): support re-sampling, etc. 234 ppapi::MediaStreamBuffer::Audio* buffer = 235 &(host_->buffer_manager()->GetBufferPointer(active_buffer_index_) 236 ->audio); 237 if (active_buffer_offset_ == 0) { 238 // The active buffer is new, so initialise the header and metadata fields. 239 buffer->header.size = host_->buffer_manager()->buffer_size(); 240 buffer->header.type = ppapi::MediaStreamBuffer::TYPE_AUDIO; 241 buffer->timestamp = (timestamp_ + timestamp_offset).InMillisecondsF(); 242 buffer->sample_rate = static_cast<PP_AudioBuffer_SampleRate>(sample_rate); 243 buffer->data_size = output_buffer_size_; 244 buffer->number_of_channels = number_of_channels; 245 buffer->number_of_samples = buffer->data_size * number_of_channels / 246 bytes_per_frame; 247 } 248 uint32_t buffer_bytes_remaining = 249 buffer->data_size - active_buffer_offset_; 250 DCHECK_EQ(buffer_bytes_remaining % bytes_per_frame, 0U); 251 uint32_t incoming_bytes_remaining = frames_remaining * bytes_per_frame; 252 uint32_t bytes_to_copy = std::min(buffer_bytes_remaining, 253 incoming_bytes_remaining); 254 uint32_t frames_to_copy = bytes_to_copy / bytes_per_frame; 255 DCHECK_EQ(bytes_to_copy % bytes_per_frame, 0U); 256 memcpy(buffer->data + active_buffer_offset_, 257 audio_data, bytes_to_copy); 258 active_buffer_offset_ += bytes_to_copy; 259 audio_data += bytes_to_copy / sizeof(*audio_data); 260 frames_remaining -= frames_to_copy; 261 timestamp_offset += base::TimeDelta::FromMilliseconds( 262 frames_to_copy * base::Time::kMillisecondsPerSecond / sample_rate); 263 264 DCHECK_LE(active_buffer_offset_, buffer->data_size); 265 if (active_buffer_offset_ == buffer->data_size) { 266 main_message_loop_proxy_->PostTask( 267 FROM_HERE, 268 base::Bind(&AudioSink::SendEnqueueBufferMessageOnMainThread, 269 weak_factory_.GetWeakPtr(), 270 active_buffer_index_, 271 buffers_generation_)); 272 active_buffer_index_ = -1; 273 } 274 } 275 timestamp_ += buffer_duration_; 276 } 277 278 void PepperMediaStreamAudioTrackHost::AudioSink::OnSetFormat( 279 const AudioParameters& params) { 280 DCHECK(params.IsValid()); 281 // TODO(amistry): How do you handle the case where the user configures a 282 // duration that's shorter than the received buffer duration? One option is to 283 // double buffer, where the size of the intermediate ring buffer is at least 284 // max(user requested duration, received buffer duration). There are other 285 // ways of dealing with it, but which one is "correct"? 286 DCHECK_LE(params.GetBufferDuration().InMilliseconds(), kMinDuration); 287 DCHECK_EQ(params.bits_per_sample(), 16); 288 DCHECK_NE(GetPPSampleRate(params.sample_rate()), 289 PP_AUDIOBUFFER_SAMPLERATE_UNKNOWN); 290 291 audio_params_ = params; 292 293 // TODO(penghuang): support setting format more than once. 294 buffer_duration_ = params.GetBufferDuration(); 295 buffer_data_size_ = params.GetBytesPerBuffer(); 296 297 if (original_audio_params_.IsValid()) { 298 DCHECK_EQ(params.sample_rate(), original_audio_params_.sample_rate()); 299 DCHECK_EQ(params.bits_per_sample(), 300 original_audio_params_.bits_per_sample()); 301 DCHECK_EQ(params.channels(), original_audio_params_.channels()); 302 } else { 303 audio_thread_checker_.DetachFromThread(); 304 original_audio_params_ = params; 305 306 int bytes_per_frame = params.channels() * params.bits_per_sample() / 8; 307 main_message_loop_proxy_->PostTask( 308 FROM_HERE, 309 base::Bind(&AudioSink::SetFormatOnMainThread, 310 weak_factory_.GetWeakPtr(), 311 params.GetBytesPerSecond(), 312 bytes_per_frame)); 313 } 314 } 315 316 PepperMediaStreamAudioTrackHost::PepperMediaStreamAudioTrackHost( 317 RendererPpapiHost* host, 318 PP_Instance instance, 319 PP_Resource resource, 320 const blink::WebMediaStreamTrack& track) 321 : PepperMediaStreamTrackHostBase(host, instance, resource), 322 track_(track), 323 connected_(false), 324 audio_sink_(this) { 325 DCHECK(!track_.isNull()); 326 } 327 328 PepperMediaStreamAudioTrackHost::~PepperMediaStreamAudioTrackHost() { 329 OnClose(); 330 } 331 332 int32_t PepperMediaStreamAudioTrackHost::OnResourceMessageReceived( 333 const IPC::Message& msg, 334 HostMessageContext* context) { 335 PPAPI_BEGIN_MESSAGE_MAP(PepperMediaStreamAudioTrackHost, msg) 336 PPAPI_DISPATCH_HOST_RESOURCE_CALL( 337 PpapiHostMsg_MediaStreamAudioTrack_Configure, OnHostMsgConfigure) 338 PPAPI_END_MESSAGE_MAP() 339 return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg, 340 context); 341 } 342 343 int32_t PepperMediaStreamAudioTrackHost::OnHostMsgConfigure( 344 HostMessageContext* context, 345 const MediaStreamAudioTrackShared::Attributes& attributes) { 346 if (!MediaStreamAudioTrackShared::VerifyAttributes(attributes)) 347 return PP_ERROR_BADARGUMENT; 348 349 int32_t buffers = attributes.buffers 350 ? std::min(kMaxNumberOfBuffers, attributes.buffers) 351 : kDefaultNumberOfBuffers; 352 return audio_sink_.Configure(buffers, attributes.duration, 353 context->MakeReplyMessageContext()); 354 } 355 356 void PepperMediaStreamAudioTrackHost::OnClose() { 357 if (connected_) { 358 MediaStreamAudioSink::RemoveFromAudioTrack(&audio_sink_, track_); 359 connected_ = false; 360 } 361 audio_sink_.SendConfigureReply(PP_ERROR_ABORTED); 362 } 363 364 void PepperMediaStreamAudioTrackHost::OnNewBufferEnqueued() { 365 int32_t index = buffer_manager()->DequeueBuffer(); 366 DCHECK_GE(index, 0); 367 audio_sink_.EnqueueBuffer(index); 368 } 369 370 void PepperMediaStreamAudioTrackHost::DidConnectPendingHostToResource() { 371 if (!connected_) { 372 MediaStreamAudioSink::AddToAudioTrack(&audio_sink_, track_); 373 connected_ = true; 374 } 375 } 376 377 } // namespace content 378