1 /* 2 * libjingle 3 * Copyright 2012, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "talk/app/webrtc/videosource.h" 29 30 #include <vector> 31 #include <cstdlib> 32 33 #include "talk/app/webrtc/mediaconstraintsinterface.h" 34 #include "talk/session/media/channelmanager.h" 35 36 using cricket::CaptureState; 37 using webrtc::MediaConstraintsInterface; 38 using webrtc::MediaSourceInterface; 39 40 namespace { 41 42 const double kRoundingTruncation = 0.0005; 43 44 enum { 45 MSG_VIDEOCAPTURESTATECONNECT, 46 MSG_VIDEOCAPTURESTATEDISCONNECT, 47 MSG_VIDEOCAPTURESTATECHANGE, 48 }; 49 50 // Default resolution. If no constraint is specified, this is the resolution we 51 // will use. 52 static const cricket::VideoFormatPod kDefaultFormat = 53 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}; 54 55 // List of formats used if the camera doesn't support capability enumeration. 56 static const cricket::VideoFormatPod kVideoFormats[] = { 57 {1920, 1080, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 58 {1280, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 59 {960, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 60 {640, 360, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 61 {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 62 {320, 240, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}, 63 {320, 180, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY} 64 }; 65 66 MediaSourceInterface::SourceState 67 GetReadyState(cricket::CaptureState state) { 68 switch (state) { 69 case cricket::CS_STARTING: 70 return MediaSourceInterface::kInitializing; 71 case cricket::CS_RUNNING: 72 return MediaSourceInterface::kLive; 73 case cricket::CS_FAILED: 74 case cricket::CS_NO_DEVICE: 75 case cricket::CS_STOPPED: 76 return MediaSourceInterface::kEnded; 77 case cricket::CS_PAUSED: 78 return MediaSourceInterface::kMuted; 79 default: 80 ASSERT(false && "GetReadyState unknown state"); 81 } 82 return MediaSourceInterface::kEnded; 83 } 84 85 void SetUpperLimit(int new_limit, int* original_limit) { 86 if (*original_limit < 0 || new_limit < *original_limit) 87 *original_limit = new_limit; 88 } 89 90 // Updates |format_upper_limit| from |constraint|. 91 // If constraint.maxFoo is smaller than format_upper_limit.foo, 92 // set format_upper_limit.foo to constraint.maxFoo. 93 void SetUpperLimitFromConstraint( 94 const MediaConstraintsInterface::Constraint& constraint, 95 cricket::VideoFormat* format_upper_limit) { 96 if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 97 int value = rtc::FromString<int>(constraint.value); 98 SetUpperLimit(value, &(format_upper_limit->width)); 99 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 100 int value = rtc::FromString<int>(constraint.value); 101 SetUpperLimit(value, &(format_upper_limit->height)); 102 } 103 } 104 105 // Fills |format_out| with the max width and height allowed by |constraints|. 106 void FromConstraintsForScreencast( 107 const MediaConstraintsInterface::Constraints& constraints, 108 cricket::VideoFormat* format_out) { 109 typedef MediaConstraintsInterface::Constraints::const_iterator 110 ConstraintsIterator; 111 112 cricket::VideoFormat upper_limit(-1, -1, 0, 0); 113 for (ConstraintsIterator constraints_it = constraints.begin(); 114 constraints_it != constraints.end(); ++constraints_it) 115 SetUpperLimitFromConstraint(*constraints_it, &upper_limit); 116 117 if (upper_limit.width >= 0) 118 format_out->width = upper_limit.width; 119 if (upper_limit.height >= 0) 120 format_out->height = upper_limit.height; 121 } 122 123 // Returns true if |constraint| is fulfilled. |format_out| can differ from 124 // |format_in| if the format is changed by the constraint. Ie - the frame rate 125 // can be changed by setting maxFrameRate. 126 bool NewFormatWithConstraints( 127 const MediaConstraintsInterface::Constraint& constraint, 128 const cricket::VideoFormat& format_in, 129 bool mandatory, 130 cricket::VideoFormat* format_out) { 131 ASSERT(format_out != NULL); 132 *format_out = format_in; 133 134 if (constraint.key == MediaConstraintsInterface::kMinWidth) { 135 int value = rtc::FromString<int>(constraint.value); 136 return (value <= format_in.width); 137 } else if (constraint.key == MediaConstraintsInterface::kMaxWidth) { 138 int value = rtc::FromString<int>(constraint.value); 139 return (value >= format_in.width); 140 } else if (constraint.key == MediaConstraintsInterface::kMinHeight) { 141 int value = rtc::FromString<int>(constraint.value); 142 return (value <= format_in.height); 143 } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) { 144 int value = rtc::FromString<int>(constraint.value); 145 return (value >= format_in.height); 146 } else if (constraint.key == MediaConstraintsInterface::kMinFrameRate) { 147 int value = rtc::FromString<int>(constraint.value); 148 return (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)); 149 } else if (constraint.key == MediaConstraintsInterface::kMaxFrameRate) { 150 int value = rtc::FromString<int>(constraint.value); 151 if (value == 0) { 152 if (mandatory) { 153 // TODO(ronghuawu): Convert the constraint value to float when sub-1fps 154 // is supported by the capturer. 155 return false; 156 } else { 157 value = 1; 158 } 159 } 160 if (value <= cricket::VideoFormat::IntervalToFps(format_in.interval)) { 161 format_out->interval = cricket::VideoFormat::FpsToInterval(value); 162 return true; 163 } else { 164 return false; 165 } 166 } else if (constraint.key == MediaConstraintsInterface::kMinAspectRatio) { 167 double value = rtc::FromString<double>(constraint.value); 168 // The aspect ratio in |constraint.value| has been converted to a string and 169 // back to a double, so it may have a rounding error. 170 // E.g if the value 1/3 is converted to a string, the string will not have 171 // infinite length. 172 // We add a margin of 0.0005 which is high enough to detect the same aspect 173 // ratio but small enough to avoid matching wrong aspect ratios. 174 double ratio = static_cast<double>(format_in.width) / format_in.height; 175 return (value <= ratio + kRoundingTruncation); 176 } else if (constraint.key == MediaConstraintsInterface::kMaxAspectRatio) { 177 double value = rtc::FromString<double>(constraint.value); 178 double ratio = static_cast<double>(format_in.width) / format_in.height; 179 // Subtract 0.0005 to avoid rounding problems. Same as above. 180 const double kRoundingTruncation = 0.0005; 181 return (value >= ratio - kRoundingTruncation); 182 } else if (constraint.key == MediaConstraintsInterface::kNoiseReduction || 183 constraint.key == MediaConstraintsInterface::kLeakyBucket || 184 constraint.key == 185 MediaConstraintsInterface::kTemporalLayeredScreencast) { 186 // These are actually options, not constraints, so they can be satisfied 187 // regardless of the format. 188 return true; 189 } 190 LOG(LS_WARNING) << "Found unknown MediaStream constraint. Name:" 191 << constraint.key << " Value:" << constraint.value; 192 return false; 193 } 194 195 // Removes cricket::VideoFormats from |formats| that don't meet |constraint|. 196 void FilterFormatsByConstraint( 197 const MediaConstraintsInterface::Constraint& constraint, 198 bool mandatory, 199 std::vector<cricket::VideoFormat>* formats) { 200 std::vector<cricket::VideoFormat>::iterator format_it = 201 formats->begin(); 202 while (format_it != formats->end()) { 203 // Modify the format_it to fulfill the constraint if possible. 204 // Delete it otherwise. 205 if (!NewFormatWithConstraints(constraint, (*format_it), 206 mandatory, &(*format_it))) { 207 format_it = formats->erase(format_it); 208 } else { 209 ++format_it; 210 } 211 } 212 } 213 214 // Returns a vector of cricket::VideoFormat that best match |constraints|. 215 std::vector<cricket::VideoFormat> FilterFormats( 216 const MediaConstraintsInterface::Constraints& mandatory, 217 const MediaConstraintsInterface::Constraints& optional, 218 const std::vector<cricket::VideoFormat>& supported_formats) { 219 typedef MediaConstraintsInterface::Constraints::const_iterator 220 ConstraintsIterator; 221 std::vector<cricket::VideoFormat> candidates = supported_formats; 222 223 for (ConstraintsIterator constraints_it = mandatory.begin(); 224 constraints_it != mandatory.end(); ++constraints_it) 225 FilterFormatsByConstraint(*constraints_it, true, &candidates); 226 227 if (candidates.size() == 0) 228 return candidates; 229 230 // Ok - all mandatory checked and we still have a candidate. 231 // Let's try filtering using the optional constraints. 232 for (ConstraintsIterator constraints_it = optional.begin(); 233 constraints_it != optional.end(); ++constraints_it) { 234 std::vector<cricket::VideoFormat> current_candidates = candidates; 235 FilterFormatsByConstraint(*constraints_it, false, ¤t_candidates); 236 if (current_candidates.size() > 0) { 237 candidates = current_candidates; 238 } 239 } 240 241 // We have done as good as we can to filter the supported resolutions. 242 return candidates; 243 } 244 245 // Find the format that best matches the default video size. 246 // Constraints are optional and since the performance of a video call 247 // might be bad due to bitrate limitations, CPU, and camera performance, 248 // it is better to select a resolution that is as close as possible to our 249 // default and still meets the contraints. 250 const cricket::VideoFormat& GetBestCaptureFormat( 251 const std::vector<cricket::VideoFormat>& formats) { 252 ASSERT(formats.size() > 0); 253 254 int default_area = kDefaultFormat.width * kDefaultFormat.height; 255 256 std::vector<cricket::VideoFormat>::const_iterator it = formats.begin(); 257 std::vector<cricket::VideoFormat>::const_iterator best_it = formats.begin(); 258 int best_diff_area = std::abs(default_area - it->width * it->height); 259 int64 best_diff_interval = kDefaultFormat.interval; 260 for (; it != formats.end(); ++it) { 261 int diff_area = std::abs(default_area - it->width * it->height); 262 int64 diff_interval = std::abs(kDefaultFormat.interval - it->interval); 263 if (diff_area < best_diff_area || 264 (diff_area == best_diff_area && diff_interval < best_diff_interval)) { 265 best_diff_area = diff_area; 266 best_diff_interval = diff_interval; 267 best_it = it; 268 } 269 } 270 return *best_it; 271 } 272 273 // Set |option| to the highest-priority value of |key| in the constraints. 274 // Return false if the key is mandatory, and the value is invalid. 275 bool ExtractOption(const MediaConstraintsInterface* all_constraints, 276 const std::string& key, cricket::Settable<bool>* option) { 277 size_t mandatory = 0; 278 bool value; 279 if (FindConstraint(all_constraints, key, &value, &mandatory)) { 280 option->Set(value); 281 return true; 282 } 283 284 return mandatory == 0; 285 } 286 287 // Search |all_constraints| for known video options. Apply all options that are 288 // found with valid values, and return false if any mandatory video option was 289 // found with an invalid value. 290 bool ExtractVideoOptions(const MediaConstraintsInterface* all_constraints, 291 cricket::VideoOptions* options) { 292 bool all_valid = true; 293 294 all_valid &= ExtractOption(all_constraints, 295 MediaConstraintsInterface::kNoiseReduction, 296 &(options->video_noise_reduction)); 297 all_valid &= ExtractOption(all_constraints, 298 MediaConstraintsInterface::kLeakyBucket, 299 &(options->video_leaky_bucket)); 300 all_valid &= ExtractOption(all_constraints, 301 MediaConstraintsInterface::kTemporalLayeredScreencast, 302 &(options->video_temporal_layer_screencast)); 303 304 return all_valid; 305 } 306 307 class FrameInputWrapper : public cricket::VideoRenderer { 308 public: 309 explicit FrameInputWrapper(cricket::VideoCapturer* capturer) 310 : capturer_(capturer) { 311 ASSERT(capturer_ != NULL); 312 } 313 314 virtual ~FrameInputWrapper() {} 315 316 // VideoRenderer implementation. 317 virtual bool SetSize(int width, int height, int reserved) OVERRIDE { 318 return true; 319 } 320 321 virtual bool RenderFrame(const cricket::VideoFrame* frame) OVERRIDE { 322 if (!capturer_->IsRunning()) { 323 return true; 324 } 325 326 // This signal will be made on media engine render thread. The clients 327 // of this signal should have no assumptions on what thread this signal 328 // come from. 329 capturer_->SignalVideoFrame(capturer_, frame); 330 return true; 331 } 332 333 private: 334 cricket::VideoCapturer* capturer_; 335 int width_; 336 int height_; 337 338 DISALLOW_COPY_AND_ASSIGN(FrameInputWrapper); 339 }; 340 341 } // anonymous namespace 342 343 namespace webrtc { 344 345 rtc::scoped_refptr<VideoSource> VideoSource::Create( 346 cricket::ChannelManager* channel_manager, 347 cricket::VideoCapturer* capturer, 348 const webrtc::MediaConstraintsInterface* constraints) { 349 ASSERT(channel_manager != NULL); 350 ASSERT(capturer != NULL); 351 rtc::scoped_refptr<VideoSource> source( 352 new rtc::RefCountedObject<VideoSource>(channel_manager, 353 capturer)); 354 source->Initialize(constraints); 355 return source; 356 } 357 358 VideoSource::VideoSource(cricket::ChannelManager* channel_manager, 359 cricket::VideoCapturer* capturer) 360 : channel_manager_(channel_manager), 361 video_capturer_(capturer), 362 state_(kInitializing) { 363 channel_manager_->SignalVideoCaptureStateChange.connect( 364 this, &VideoSource::OnStateChange); 365 } 366 367 VideoSource::~VideoSource() { 368 channel_manager_->StopVideoCapture(video_capturer_.get(), format_); 369 channel_manager_->SignalVideoCaptureStateChange.disconnect(this); 370 } 371 372 void VideoSource::Initialize( 373 const webrtc::MediaConstraintsInterface* constraints) { 374 375 std::vector<cricket::VideoFormat> formats; 376 if (video_capturer_->GetSupportedFormats() && 377 video_capturer_->GetSupportedFormats()->size() > 0) { 378 formats = *video_capturer_->GetSupportedFormats(); 379 } else if (video_capturer_->IsScreencast()) { 380 // The screen capturer can accept any resolution and we will derive the 381 // format from the constraints if any. 382 // Note that this only affects tab capturing, not desktop capturing, 383 // since desktop capturer does not respect the VideoFormat passed in. 384 formats.push_back(cricket::VideoFormat(kDefaultFormat)); 385 } else { 386 // The VideoCapturer implementation doesn't support capability enumeration. 387 // We need to guess what the camera support. 388 for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) { 389 formats.push_back(cricket::VideoFormat(kVideoFormats[i])); 390 } 391 } 392 393 if (constraints) { 394 MediaConstraintsInterface::Constraints mandatory_constraints = 395 constraints->GetMandatory(); 396 MediaConstraintsInterface::Constraints optional_constraints; 397 optional_constraints = constraints->GetOptional(); 398 399 if (video_capturer_->IsScreencast()) { 400 // Use the maxWidth and maxHeight allowed by constraints for screencast. 401 FromConstraintsForScreencast(mandatory_constraints, &(formats[0])); 402 } 403 404 formats = FilterFormats(mandatory_constraints, optional_constraints, 405 formats); 406 } 407 408 if (formats.size() == 0) { 409 LOG(LS_WARNING) << "Failed to find a suitable video format."; 410 SetState(kEnded); 411 return; 412 } 413 414 cricket::VideoOptions options; 415 if (!ExtractVideoOptions(constraints, &options)) { 416 LOG(LS_WARNING) << "Could not satisfy mandatory options."; 417 SetState(kEnded); 418 return; 419 } 420 options_.SetAll(options); 421 422 format_ = GetBestCaptureFormat(formats); 423 // Start the camera with our best guess. 424 // TODO(perkj): Should we try again with another format it it turns out that 425 // the camera doesn't produce frames with the correct format? Or will 426 // cricket::VideCapturer be able to re-scale / crop to the requested 427 // resolution? 428 if (!channel_manager_->StartVideoCapture(video_capturer_.get(), format_)) { 429 SetState(kEnded); 430 return; 431 } 432 // Initialize hasn't succeeded until a successful state change has occurred. 433 } 434 435 cricket::VideoRenderer* VideoSource::FrameInput() { 436 // Defer creation of frame_input_ until it's needed, e.g. the local video 437 // sources will never need it. 438 if (!frame_input_) { 439 frame_input_.reset(new FrameInputWrapper(video_capturer_.get())); 440 } 441 return frame_input_.get(); 442 } 443 444 void VideoSource::AddSink(cricket::VideoRenderer* output) { 445 channel_manager_->AddVideoRenderer(video_capturer_.get(), output); 446 } 447 448 void VideoSource::RemoveSink(cricket::VideoRenderer* output) { 449 channel_manager_->RemoveVideoRenderer(video_capturer_.get(), output); 450 } 451 452 // OnStateChange listens to the ChannelManager::SignalVideoCaptureStateChange. 453 // This signal is triggered for all video capturers. Not only the one we are 454 // interested in. 455 void VideoSource::OnStateChange(cricket::VideoCapturer* capturer, 456 cricket::CaptureState capture_state) { 457 if (capturer == video_capturer_.get()) { 458 SetState(GetReadyState(capture_state)); 459 } 460 } 461 462 void VideoSource::SetState(SourceState new_state) { 463 if (VERIFY(state_ != new_state)) { 464 state_ = new_state; 465 FireOnChanged(); 466 } 467 } 468 469 } // namespace webrtc 470