Home | History | Annotate | Download | only in media 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/renderer/media/media_stream_video_source.h"
      6 
      7 #include <algorithm>
      8 #include <limits>
      9 #include <string>
     10 
     11 #include "base/debug/trace_event.h"
     12 #include "base/logging.h"
     13 #include "base/strings/string_number_conversions.h"
     14 #include "content/child/child_process.h"
     15 #include "content/renderer/media/media_stream_constraints_util.h"
     16 #include "content/renderer/media/media_stream_video_track.h"
     17 #include "content/renderer/media/video_track_adapter.h"
     18 
     19 namespace content {
     20 
     21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
     22 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
     23 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
     24 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
     25 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
     26 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
     27 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
     28 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
     29 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
     30 
     31 const char* kSupportedConstraints[] = {
     32   MediaStreamVideoSource::kMaxAspectRatio,
     33   MediaStreamVideoSource::kMinAspectRatio,
     34   MediaStreamVideoSource::kMaxWidth,
     35   MediaStreamVideoSource::kMinWidth,
     36   MediaStreamVideoSource::kMaxHeight,
     37   MediaStreamVideoSource::kMinHeight,
     38   MediaStreamVideoSource::kMaxFrameRate,
     39   MediaStreamVideoSource::kMinFrameRate,
     40 };
     41 
     42 const int MediaStreamVideoSource::kDefaultWidth = 640;
     43 const int MediaStreamVideoSource::kDefaultHeight = 480;
     44 const int MediaStreamVideoSource::kDefaultFrameRate = 30;
     45 
     46 namespace {
     47 
     48 // Google-specific key prefix. Constraints with this prefix are ignored if they
     49 // are unknown.
     50 const char kGooglePrefix[] = "goog";
     51 
     52 // Returns true if |constraint| has mandatory constraints.
     53 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
     54   blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
     55   constraints.getMandatoryConstraints(mandatory_constraints);
     56   return !mandatory_constraints.isEmpty();
     57 }
     58 
     59 // Retrieve the desired max width and height from |constraints|. If not set,
     60 // the |desired_width| and |desired_height| are set to
     61 // std::numeric_limits<int>::max();
     62 // If either max width or height is set as a mandatory constraint, the optional
     63 // constraints are not checked.
     64 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
     65                                  int* desired_width, int* desired_height) {
     66   *desired_width = std::numeric_limits<int>::max();
     67   *desired_height = std::numeric_limits<int>::max();
     68 
     69   bool mandatory = GetMandatoryConstraintValueAsInteger(
     70       constraints,
     71       MediaStreamVideoSource::kMaxWidth,
     72       desired_width);
     73   mandatory |= GetMandatoryConstraintValueAsInteger(
     74       constraints,
     75       MediaStreamVideoSource::kMaxHeight,
     76       desired_height);
     77   if (mandatory)
     78     return;
     79 
     80   GetOptionalConstraintValueAsInteger(constraints,
     81                                       MediaStreamVideoSource::kMaxWidth,
     82                                       desired_width);
     83   GetOptionalConstraintValueAsInteger(constraints,
     84                                       MediaStreamVideoSource::kMaxHeight,
     85                                       desired_height);
     86 }
     87 
     88 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
     89 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
     90 // std::numeric_limits<double>::max();
     91 // If either min or max aspect ratio is set as a mandatory constraint, the
     92 // optional constraints are not checked.
     93 void GetDesiredMinAndMaxAspectRatio(
     94     const blink::WebMediaConstraints& constraints,
     95     double* min_aspect_ratio,
     96     double* max_aspect_ratio) {
     97   *min_aspect_ratio = 0;
     98   *max_aspect_ratio = std::numeric_limits<double>::max();
     99 
    100   bool mandatory = GetMandatoryConstraintValueAsDouble(
    101       constraints,
    102       MediaStreamVideoSource::kMinAspectRatio,
    103       min_aspect_ratio);
    104   mandatory |= GetMandatoryConstraintValueAsDouble(
    105       constraints,
    106       MediaStreamVideoSource::kMaxAspectRatio,
    107       max_aspect_ratio);
    108   if (mandatory)
    109     return;
    110 
    111   GetOptionalConstraintValueAsDouble(
    112       constraints,
    113       MediaStreamVideoSource::kMinAspectRatio,
    114       min_aspect_ratio);
    115   GetOptionalConstraintValueAsDouble(
    116       constraints,
    117       MediaStreamVideoSource::kMaxAspectRatio,
    118       max_aspect_ratio);
    119 }
    120 
    121 // Returns true if |constraint| is fulfilled. |format| can be changed by a
    122 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
    123 bool UpdateFormatForConstraint(
    124     const blink::WebMediaConstraint& constraint,
    125     bool mandatory,
    126     media::VideoCaptureFormat* format) {
    127   DCHECK(format != NULL);
    128 
    129   if (!format->IsValid())
    130     return false;
    131 
    132   std::string constraint_name = constraint.m_name.utf8();
    133   std::string constraint_value = constraint.m_value.utf8();
    134 
    135   if (constraint_name.find(kGooglePrefix) == 0) {
    136     // These are actually options, not constraints, so they can be satisfied
    137     // regardless of the format.
    138     return true;
    139   }
    140 
    141   if (constraint_name == MediaStreamSource::kSourceId) {
    142     // This is a constraint that doesn't affect the format.
    143     return true;
    144   }
    145 
    146   // Ignore Chrome specific Tab capture constraints.
    147   if (constraint_name == kMediaStreamSource ||
    148       constraint_name == kMediaStreamSourceId)
    149     return true;
    150 
    151   if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
    152       constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
    153     // These constraints are handled by cropping if the camera outputs the wrong
    154     // aspect ratio.
    155     double value;
    156     return base::StringToDouble(constraint_value, &value);
    157   }
    158 
    159   double value = 0.0;
    160   if (!base::StringToDouble(constraint_value, &value)) {
    161     DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
    162                   <<  constraint_name << " Value:" << constraint_value;
    163     return false;
    164   }
    165 
    166   if (constraint_name == MediaStreamVideoSource::kMinWidth) {
    167     return (value <= format->frame_size.width());
    168   } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
    169     return value > 0.0;
    170   } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
    171     return (value <= format->frame_size.height());
    172   } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
    173      return value > 0.0;
    174   } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
    175     return (value <= format->frame_rate);
    176   } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
    177     if (value == 0.0) {
    178       // The frame rate is set by constraint.
    179       // Don't allow 0 as frame rate if it is a mandatory constraint.
    180       // Set the frame rate to 1 if it is not mandatory.
    181       if (mandatory) {
    182         return false;
    183       } else {
    184         value = 1.0;
    185       }
    186     }
    187     format->frame_rate =
    188         (format->frame_rate > value) ? value : format->frame_rate;
    189     return true;
    190   } else {
    191     LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
    192                  <<  constraint_name << " Value:" << constraint_value;
    193     return false;
    194   }
    195 }
    196 
    197 // Removes media::VideoCaptureFormats from |formats| that don't meet
    198 // |constraint|.
    199 void FilterFormatsByConstraint(
    200     const blink::WebMediaConstraint& constraint,
    201     bool mandatory,
    202     media::VideoCaptureFormats* formats) {
    203   DVLOG(3) << "FilterFormatsByConstraint("
    204            << "{ constraint.m_name = " << constraint.m_name.utf8()
    205            << "  constraint.m_value = " << constraint.m_value.utf8()
    206            << "  mandatory =  " << mandatory << "})";
    207   media::VideoCaptureFormats::iterator format_it = formats->begin();
    208   while (format_it != formats->end()) {
    209     // Modify the format_it to fulfill the constraint if possible.
    210     // Delete it otherwise.
    211     if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it))) {
    212       format_it = formats->erase(format_it);
    213     } else {
    214       ++format_it;
    215     }
    216   }
    217 }
    218 
    219 // Returns the media::VideoCaptureFormats that matches |constraints|.
    220 media::VideoCaptureFormats FilterFormats(
    221     const blink::WebMediaConstraints& constraints,
    222     const media::VideoCaptureFormats& supported_formats) {
    223   if (constraints.isNull()) {
    224     return supported_formats;
    225   }
    226 
    227   double max_aspect_ratio;
    228   double min_aspect_ratio;
    229   GetDesiredMinAndMaxAspectRatio(constraints,
    230                                  &min_aspect_ratio,
    231                                  &max_aspect_ratio);
    232 
    233   if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
    234     DLOG(WARNING) << "Wrong requested aspect ratio.";
    235     return media::VideoCaptureFormats();
    236   }
    237 
    238   int min_width = 0;
    239   GetMandatoryConstraintValueAsInteger(constraints,
    240                                        MediaStreamVideoSource::kMinWidth,
    241                                        &min_width);
    242   int min_height = 0;
    243   GetMandatoryConstraintValueAsInteger(constraints,
    244                                        MediaStreamVideoSource::kMinHeight,
    245                                        &min_height);
    246   int max_width;
    247   int max_height;
    248   GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
    249 
    250   if (min_width > max_width || min_height > max_height)
    251     return media::VideoCaptureFormats();
    252 
    253   blink::WebVector<blink::WebMediaConstraint> mandatory;
    254   blink::WebVector<blink::WebMediaConstraint> optional;
    255   constraints.getMandatoryConstraints(mandatory);
    256   constraints.getOptionalConstraints(optional);
    257   media::VideoCaptureFormats candidates = supported_formats;
    258   for (size_t i = 0; i < mandatory.size(); ++i)
    259     FilterFormatsByConstraint(mandatory[i], true, &candidates);
    260 
    261   if (candidates.empty())
    262     return candidates;
    263 
    264   // Ok - all mandatory checked and we still have candidates.
    265   // Let's try filtering using the optional constraints. The optional
    266   // constraints must be filtered in the order they occur in |optional|.
    267   // But if a constraint produce zero candidates, the constraint is ignored and
    268   // the next constraint is tested.
    269   // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
    270   for (size_t i = 0; i < optional.size(); ++i) {
    271     media::VideoCaptureFormats current_candidates = candidates;
    272     FilterFormatsByConstraint(optional[i], false, &current_candidates);
    273     if (!current_candidates.empty()) {
    274       candidates = current_candidates;
    275     }
    276   }
    277 
    278   // We have done as good as we can to filter the supported resolutions.
    279   return candidates;
    280 }
    281 
    282 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
    283     const media::VideoCaptureFormats& formats,
    284     int area) {
    285   media::VideoCaptureFormats::const_iterator it = formats.begin();
    286   media::VideoCaptureFormats::const_iterator best_it = formats.begin();
    287   int best_diff = std::numeric_limits<int>::max();
    288   for (; it != formats.end(); ++it) {
    289     int diff = abs(area - it->frame_size.width() * it->frame_size.height());
    290     if (diff < best_diff) {
    291       best_diff = diff;
    292       best_it = it;
    293     }
    294   }
    295   return *best_it;
    296 }
    297 
    298 // Find the format that best matches the default video size.
    299 // This algorithm is chosen since a resolution must be picked even if no
    300 // constraints are provided. We don't just select the maximum supported
    301 // resolution since higher resolutions cost more in terms of complexity and
    302 // many cameras have lower frame rate and have more noise in the image at
    303 // their maximum supported resolution.
    304 void GetBestCaptureFormat(
    305     const media::VideoCaptureFormats& formats,
    306     const blink::WebMediaConstraints& constraints,
    307     media::VideoCaptureFormat* capture_format) {
    308   DCHECK(!formats.empty());
    309 
    310   int max_width;
    311   int max_height;
    312   GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
    313 
    314   *capture_format = GetBestFormatBasedOnArea(
    315       formats,
    316       std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
    317       std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
    318 }
    319 
    320 }  // anonymous namespace
    321 
    322 // static
    323 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
    324     const blink::WebMediaStreamSource& source) {
    325   return static_cast<MediaStreamVideoSource*>(source.extraData());
    326 }
    327 
    328 // static
    329 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
    330   for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
    331     if (kSupportedConstraints[i] == name)
    332       return true;
    333   }
    334   return false;
    335 }
    336 
    337 MediaStreamVideoSource::MediaStreamVideoSource()
    338     : state_(NEW),
    339       track_adapter_(new VideoTrackAdapter(
    340           ChildProcess::current()->io_message_loop_proxy())),
    341       weak_factory_(this) {
    342 }
    343 
    344 MediaStreamVideoSource::~MediaStreamVideoSource() {
    345   DVLOG(3) << "~MediaStreamVideoSource()";
    346 }
    347 
    348 void MediaStreamVideoSource::AddTrack(
    349     MediaStreamVideoTrack* track,
    350     const VideoCaptureDeliverFrameCB& frame_callback,
    351     const blink::WebMediaConstraints& constraints,
    352     const ConstraintsCallback& callback) {
    353   DCHECK(CalledOnValidThread());
    354   DCHECK(!constraints.isNull());
    355   DCHECK(std::find(tracks_.begin(), tracks_.end(),
    356                    track) == tracks_.end());
    357   tracks_.push_back(track);
    358 
    359   requested_constraints_.push_back(
    360       RequestedConstraints(track, frame_callback, constraints, callback));
    361 
    362   switch (state_) {
    363     case NEW: {
    364       // Tab capture and Screen capture needs the maximum requested height
    365       // and width to decide on the resolution.
    366       int max_requested_width = 0;
    367       GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
    368                                            &max_requested_width);
    369 
    370       int max_requested_height = 0;
    371       GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
    372                                            &max_requested_height);
    373 
    374       state_ = RETRIEVING_CAPABILITIES;
    375       GetCurrentSupportedFormats(
    376           max_requested_width,
    377           max_requested_height,
    378           base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
    379                      weak_factory_.GetWeakPtr()));
    380 
    381       break;
    382     }
    383     case STARTING:
    384     case RETRIEVING_CAPABILITIES: {
    385       // The |callback| will be triggered once the source has started or
    386       // the capabilities have been retrieved.
    387       break;
    388     }
    389     case ENDED:
    390     case STARTED: {
    391       // Currently, reconfiguring the source is not supported.
    392       FinalizeAddTrack();
    393     }
    394   }
    395 }
    396 
    397 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
    398   DCHECK(CalledOnValidThread());
    399   std::vector<MediaStreamVideoTrack*>::iterator it =
    400       std::find(tracks_.begin(), tracks_.end(), video_track);
    401   DCHECK(it != tracks_.end());
    402   tracks_.erase(it);
    403 
    404   // Check if |video_track| is waiting for applying new constraints and remove
    405   // the request in that case.
    406   for (std::vector<RequestedConstraints>::iterator it =
    407            requested_constraints_.begin();
    408        it != requested_constraints_.end(); ++it) {
    409     if (it->track == video_track) {
    410       requested_constraints_.erase(it);
    411       break;
    412     }
    413   }
    414   // Call |frame_adapter_->RemoveTrack| here even if adding the track has
    415   // failed and |frame_adapter_->AddCallback| has not been called.
    416   track_adapter_->RemoveTrack(video_track);
    417 
    418   if (tracks_.empty())
    419     StopSource();
    420 }
    421 
    422 const scoped_refptr<base::MessageLoopProxy>&
    423 MediaStreamVideoSource::io_message_loop() const {
    424   DCHECK(CalledOnValidThread());
    425   return track_adapter_->io_message_loop();
    426 }
    427 
    428 void MediaStreamVideoSource::DoStopSource() {
    429   DCHECK(CalledOnValidThread());
    430   DVLOG(3) << "DoStopSource()";
    431   if (state_ == ENDED)
    432     return;
    433   StopSourceImpl();
    434   state_ = ENDED;
    435   SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
    436 }
    437 
    438 void MediaStreamVideoSource::OnSupportedFormats(
    439     const media::VideoCaptureFormats& formats) {
    440   DCHECK(CalledOnValidThread());
    441   DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
    442 
    443   supported_formats_ = formats;
    444   if (!FindBestFormatWithConstraints(supported_formats_,
    445                                      &current_format_)) {
    446     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
    447     // This object can be deleted after calling FinalizeAddTrack. See comment
    448     // in the header file.
    449     FinalizeAddTrack();
    450     return;
    451   }
    452 
    453   state_ = STARTING;
    454   DVLOG(3) << "Starting the capturer with"
    455            << " width = " << current_format_.frame_size.width()
    456            << " height = " << current_format_.frame_size.height()
    457            << " frame rate = " << current_format_.frame_rate;
    458 
    459   media::VideoCaptureParams params;
    460   params.requested_format = current_format_;
    461   StartSourceImpl(
    462       params,
    463       base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
    464 }
    465 
    466 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
    467     const media::VideoCaptureFormats& formats,
    468     media::VideoCaptureFormat* best_format) {
    469   // Find the first constraints that we can fulfill.
    470   for (std::vector<RequestedConstraints>::iterator request_it =
    471            requested_constraints_.begin();
    472        request_it != requested_constraints_.end(); ++request_it) {
    473     const blink::WebMediaConstraints& requested_constraints =
    474         request_it->constraints;
    475 
    476     // If the source doesn't support capability enumeration it is still ok if
    477     // no mandatory constraints have been specified. That just means that
    478     // we will start with whatever format is native to the source.
    479     if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
    480       *best_format = media::VideoCaptureFormat();
    481       return true;
    482     }
    483     media::VideoCaptureFormats filtered_formats =
    484         FilterFormats(requested_constraints, formats);
    485     if (filtered_formats.size() > 0) {
    486       // A request with constraints that can be fulfilled.
    487       GetBestCaptureFormat(filtered_formats,
    488                            requested_constraints,
    489                            best_format);
    490       return true;
    491     }
    492   }
    493   return false;
    494 }
    495 
    496 void MediaStreamVideoSource::OnStartDone(bool success) {
    497   DCHECK(CalledOnValidThread());
    498   DVLOG(3) << "OnStartDone({success =" << success << "})";
    499   if (success) {
    500     DCHECK_EQ(STARTING, state_);
    501     state_ = STARTED;
    502     SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
    503   } else {
    504     state_ = ENDED;
    505     SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
    506     StopSourceImpl();
    507   }
    508 
    509   // This object can be deleted after calling FinalizeAddTrack. See comment in
    510   // the header file.
    511   FinalizeAddTrack();
    512 }
    513 
    514 void MediaStreamVideoSource::FinalizeAddTrack() {
    515   media::VideoCaptureFormats formats;
    516   formats.push_back(current_format_);
    517 
    518   std::vector<RequestedConstraints> callbacks;
    519   callbacks.swap(requested_constraints_);
    520   for (std::vector<RequestedConstraints>::iterator it = callbacks.begin();
    521        it != callbacks.end(); ++it) {
    522     // The track has been added successfully if the source has started and
    523     // there are either no mandatory constraints and the source doesn't expose
    524     // its format capabilities, or the constraints and the format match.
    525     // For example, a remote source doesn't expose its format capabilities.
    526     bool success =
    527         state_ == STARTED &&
    528         ((!current_format_.IsValid() && !HasMandatoryConstraints(
    529             it->constraints)) ||
    530          !FilterFormats(it->constraints, formats).empty());
    531 
    532     if (success) {
    533       int max_width;
    534       int max_height;
    535       GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
    536       double max_aspect_ratio;
    537       double min_aspect_ratio;
    538       GetDesiredMinAndMaxAspectRatio(it->constraints,
    539                                      &min_aspect_ratio,
    540                                      &max_aspect_ratio);
    541       track_adapter_->AddTrack(it->track,it->frame_callback,
    542                                max_width, max_height,
    543                                min_aspect_ratio, max_aspect_ratio);
    544     }
    545 
    546     DVLOG(3) << "FinalizeAddTrack() success " << success;
    547 
    548     if (!it->callback.is_null())
    549       it->callback.Run(this, success);
    550   }
    551 }
    552 
    553 void MediaStreamVideoSource::SetReadyState(
    554     blink::WebMediaStreamSource::ReadyState state) {
    555   if (!owner().isNull()) {
    556     owner().setReadyState(state);
    557   }
    558   for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
    559        it != tracks_.end(); ++it) {
    560     (*it)->OnReadyStateChanged(state);
    561   }
    562 }
    563 
    564 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
    565     MediaStreamVideoTrack* track,
    566     const VideoCaptureDeliverFrameCB& frame_callback,
    567     const blink::WebMediaConstraints& constraints,
    568     const ConstraintsCallback& callback)
    569     : track(track),
    570       frame_callback(frame_callback),
    571       constraints(constraints),
    572       callback(callback) {
    573 }
    574 
    575 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
    576 }
    577 
    578 }  // namespace content
    579