Home | History | Annotate | Download | only in webrtc 
      1 /*
      2  * libjingle
      3  * Copyright 2011 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #include "talk/media/webrtc/webrtcvideocapturer.h"
     29 
     30 #ifdef HAVE_CONFIG_H
     31 #include <config.h>
     32 #endif
     33 
     34 #ifdef HAVE_WEBRTC_VIDEO
     35 #include "talk/media/webrtc/webrtcvideoframe.h"
     36 #include "talk/media/webrtc/webrtcvideoframefactory.h"
     37 #include "webrtc/base/arraysize.h"
     38 #include "webrtc/base/bind.h"
     39 #include "webrtc/base/checks.h"
     40 #include "webrtc/base/criticalsection.h"
     41 #include "webrtc/base/logging.h"
     42 #include "webrtc/base/safe_conversions.h"
     43 #include "webrtc/base/thread.h"
     44 #include "webrtc/base/timeutils.h"
     45 
     46 #include "webrtc/base/win32.h"  // Need this to #include the impl files.
     47 #include "webrtc/modules/video_capture/video_capture_factory.h"
     48 #include "webrtc/system_wrappers/include/field_trial.h"
     49 
     50 namespace cricket {
     51 
     52 struct kVideoFourCCEntry {
     53   uint32_t fourcc;
     54   webrtc::RawVideoType webrtc_type;
     55 };
     56 
     57 // This indicates our format preferences and defines a mapping between
     58 // webrtc::RawVideoType (from video_capture_defines.h) to our FOURCCs.
     59 static kVideoFourCCEntry kSupportedFourCCs[] = {
     60   { FOURCC_I420, webrtc::kVideoI420 },   // 12 bpp, no conversion.
     61   { FOURCC_YV12, webrtc::kVideoYV12 },   // 12 bpp, no conversion.
     62   { FOURCC_YUY2, webrtc::kVideoYUY2 },   // 16 bpp, fast conversion.
     63   { FOURCC_UYVY, webrtc::kVideoUYVY },   // 16 bpp, fast conversion.
     64   { FOURCC_NV12, webrtc::kVideoNV12 },   // 12 bpp, fast conversion.
     65   { FOURCC_NV21, webrtc::kVideoNV21 },   // 12 bpp, fast conversion.
     66   { FOURCC_MJPG, webrtc::kVideoMJPEG },  // compressed, slow conversion.
     67   { FOURCC_ARGB, webrtc::kVideoARGB },   // 32 bpp, slow conversion.
     68   { FOURCC_24BG, webrtc::kVideoRGB24 },  // 24 bpp, slow conversion.
     69 };
     70 
     71 class WebRtcVcmFactory : public WebRtcVcmFactoryInterface {
     72  public:
     73   virtual webrtc::VideoCaptureModule* Create(int id, const char* device) {
     74     return webrtc::VideoCaptureFactory::Create(id, device);
     75   }
     76   virtual webrtc::VideoCaptureModule::DeviceInfo* CreateDeviceInfo(int id) {
     77     return webrtc::VideoCaptureFactory::CreateDeviceInfo(id);
     78   }
     79   virtual void DestroyDeviceInfo(webrtc::VideoCaptureModule::DeviceInfo* info) {
     80     delete info;
     81   }
     82 };
     83 
     84 static bool CapabilityToFormat(const webrtc::VideoCaptureCapability& cap,
     85                                VideoFormat* format) {
     86   uint32_t fourcc = 0;
     87   for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
     88     if (kSupportedFourCCs[i].webrtc_type == cap.rawType) {
     89       fourcc = kSupportedFourCCs[i].fourcc;
     90       break;
     91     }
     92   }
     93   if (fourcc == 0) {
     94     return false;
     95   }
     96 
     97   format->fourcc = fourcc;
     98   format->width = cap.width;
     99   format->height = cap.height;
    100   format->interval = VideoFormat::FpsToInterval(cap.maxFPS);
    101   return true;
    102 }
    103 
    104 static bool FormatToCapability(const VideoFormat& format,
    105                                webrtc::VideoCaptureCapability* cap) {
    106   webrtc::RawVideoType webrtc_type = webrtc::kVideoUnknown;
    107   for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
    108     if (kSupportedFourCCs[i].fourcc == format.fourcc) {
    109       webrtc_type = kSupportedFourCCs[i].webrtc_type;
    110       break;
    111     }
    112   }
    113   if (webrtc_type == webrtc::kVideoUnknown) {
    114     return false;
    115   }
    116 
    117   cap->width = format.width;
    118   cap->height = format.height;
    119   cap->maxFPS = VideoFormat::IntervalToFps(format.interval);
    120   cap->expectedCaptureDelay = 0;
    121   cap->rawType = webrtc_type;
    122   cap->codecType = webrtc::kVideoCodecUnknown;
    123   cap->interlaced = false;
    124   return true;
    125 }
    126 
    127 ///////////////////////////////////////////////////////////////////////////
    128 // Implementation of class WebRtcVideoCapturer
    129 ///////////////////////////////////////////////////////////////////////////
    130 
    131 WebRtcVideoCapturer::WebRtcVideoCapturer()
    132     : factory_(new WebRtcVcmFactory),
    133       module_(nullptr),
    134       captured_frames_(0),
    135       start_thread_(nullptr),
    136       async_invoker_(nullptr) {
    137   set_frame_factory(new WebRtcVideoFrameFactory());
    138 }
    139 
    140 WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory)
    141     : factory_(factory),
    142       module_(nullptr),
    143       captured_frames_(0),
    144       start_thread_(nullptr),
    145       async_invoker_(nullptr) {
    146   set_frame_factory(new WebRtcVideoFrameFactory());
    147 }
    148 
    149 WebRtcVideoCapturer::~WebRtcVideoCapturer() {
    150   if (module_) {
    151     module_->Release();
    152   }
    153 }
    154 
    155 bool WebRtcVideoCapturer::Init(const Device& device) {
    156   RTC_DCHECK(!start_thread_);
    157   if (module_) {
    158     LOG(LS_ERROR) << "The capturer is already initialized";
    159     return false;
    160   }
    161 
    162   webrtc::VideoCaptureModule::DeviceInfo* info = factory_->CreateDeviceInfo(0);
    163   if (!info) {
    164     return false;
    165   }
    166 
    167   // Find the desired camera, by name.
    168   // In the future, comparing IDs will be more robust.
    169   // TODO(juberti): Figure what's needed to allow this.
    170   int num_cams = info->NumberOfDevices();
    171   char vcm_id[256] = "";
    172   bool found = false;
    173   for (int index = 0; index < num_cams; ++index) {
    174     char vcm_name[256];
    175     if (info->GetDeviceName(index, vcm_name, arraysize(vcm_name), vcm_id,
    176                             arraysize(vcm_id)) != -1) {
    177       if (device.name == reinterpret_cast<char*>(vcm_name)) {
    178         found = true;
    179         break;
    180       }
    181     }
    182   }
    183   if (!found) {
    184     LOG(LS_WARNING) << "Failed to find capturer for id: " << device.id;
    185     factory_->DestroyDeviceInfo(info);
    186     return false;
    187   }
    188 
    189   // Enumerate the supported formats.
    190   // TODO(juberti): Find out why this starts/stops the camera...
    191   std::vector<VideoFormat> supported;
    192   int32_t num_caps = info->NumberOfCapabilities(vcm_id);
    193   for (int32_t i = 0; i < num_caps; ++i) {
    194     webrtc::VideoCaptureCapability cap;
    195     if (info->GetCapability(vcm_id, i, cap) != -1) {
    196       VideoFormat format;
    197       if (CapabilityToFormat(cap, &format)) {
    198         supported.push_back(format);
    199       } else {
    200         LOG(LS_WARNING) << "Ignoring unsupported WebRTC capture format "
    201                         << cap.rawType;
    202       }
    203     }
    204   }
    205   factory_->DestroyDeviceInfo(info);
    206 
    207   if (supported.empty()) {
    208     LOG(LS_ERROR) << "Failed to find usable formats for id: " << device.id;
    209     return false;
    210   }
    211 
    212   module_ = factory_->Create(0, vcm_id);
    213   if (!module_) {
    214     LOG(LS_ERROR) << "Failed to create capturer for id: " << device.id;
    215     return false;
    216   }
    217 
    218   // It is safe to change member attributes now.
    219   module_->AddRef();
    220   SetId(device.id);
    221   SetSupportedFormats(supported);
    222 
    223   // Ensure these 2 have the same value.
    224   SetApplyRotation(module_->GetApplyRotation());
    225 
    226   return true;
    227 }
    228 
    229 bool WebRtcVideoCapturer::Init(webrtc::VideoCaptureModule* module) {
    230   RTC_DCHECK(!start_thread_);
    231   if (module_) {
    232     LOG(LS_ERROR) << "The capturer is already initialized";
    233     return false;
    234   }
    235   if (!module) {
    236     LOG(LS_ERROR) << "Invalid VCM supplied";
    237     return false;
    238   }
    239   // TODO(juberti): Set id and formats.
    240   (module_ = module)->AddRef();
    241   return true;
    242 }
    243 
    244 bool WebRtcVideoCapturer::GetBestCaptureFormat(const VideoFormat& desired,
    245                                                VideoFormat* best_format) {
    246   if (!best_format) {
    247     return false;
    248   }
    249 
    250   if (!VideoCapturer::GetBestCaptureFormat(desired, best_format)) {
    251     // We maybe using a manually injected VCM which doesn't support enum.
    252     // Use the desired format as the best format.
    253     best_format->width = desired.width;
    254     best_format->height = desired.height;
    255     best_format->fourcc = FOURCC_I420;
    256     best_format->interval = desired.interval;
    257     LOG(LS_INFO) << "Failed to find best capture format,"
    258                  << " fall back to the requested format "
    259                  << best_format->ToString();
    260   }
    261   return true;
    262 }
    263 bool WebRtcVideoCapturer::SetApplyRotation(bool enable) {
    264   // Can't take lock here as this will cause deadlock with
    265   // OnIncomingCapturedFrame. In fact, the whole method, including methods it
    266   // calls, can't take lock.
    267   RTC_DCHECK(module_);
    268 
    269   const std::string group_name =
    270       webrtc::field_trial::FindFullName("WebRTC-CVO");
    271 
    272   if (group_name == "Disabled") {
    273     return true;
    274   }
    275 
    276   if (!VideoCapturer::SetApplyRotation(enable)) {
    277     return false;
    278   }
    279   return module_->SetApplyRotation(enable);
    280 }
    281 
    282 CaptureState WebRtcVideoCapturer::Start(const VideoFormat& capture_format) {
    283   if (!module_) {
    284     LOG(LS_ERROR) << "The capturer has not been initialized";
    285     return CS_NO_DEVICE;
    286   }
    287   if (start_thread_) {
    288     LOG(LS_ERROR) << "The capturer is already running";
    289     RTC_DCHECK(start_thread_->IsCurrent())
    290         << "Trying to start capturer on different threads";
    291     return CS_FAILED;
    292   }
    293 
    294   start_thread_ = rtc::Thread::Current();
    295   RTC_DCHECK(!async_invoker_);
    296   async_invoker_.reset(new rtc::AsyncInvoker());
    297   captured_frames_ = 0;
    298 
    299   SetCaptureFormat(&capture_format);
    300 
    301   webrtc::VideoCaptureCapability cap;
    302   if (!FormatToCapability(capture_format, &cap)) {
    303     LOG(LS_ERROR) << "Invalid capture format specified";
    304     return CS_FAILED;
    305   }
    306 
    307   uint32_t start = rtc::Time();
    308   module_->RegisterCaptureDataCallback(*this);
    309   if (module_->StartCapture(cap) != 0) {
    310     LOG(LS_ERROR) << "Camera '" << GetId() << "' failed to start";
    311     module_->DeRegisterCaptureDataCallback();
    312     async_invoker_.reset();
    313     SetCaptureFormat(nullptr);
    314     start_thread_ = nullptr;
    315     return CS_FAILED;
    316   }
    317 
    318   LOG(LS_INFO) << "Camera '" << GetId() << "' started with format "
    319                << capture_format.ToString() << ", elapsed time "
    320                << rtc::TimeSince(start) << " ms";
    321 
    322   SetCaptureState(CS_RUNNING);
    323   return CS_STARTING;
    324 }
    325 
    326 void WebRtcVideoCapturer::Stop() {
    327   if (!start_thread_) {
    328     LOG(LS_ERROR) << "The capturer is already stopped";
    329     return;
    330   }
    331   RTC_DCHECK(start_thread_);
    332   RTC_DCHECK(start_thread_->IsCurrent());
    333   RTC_DCHECK(async_invoker_);
    334   if (IsRunning()) {
    335     // The module is responsible for OnIncomingCapturedFrame being called, if
    336     // we stop it we will get no further callbacks.
    337     module_->StopCapture();
    338   }
    339   module_->DeRegisterCaptureDataCallback();
    340 
    341   // TODO(juberti): Determine if the VCM exposes any drop stats we can use.
    342   double drop_ratio = 0.0;
    343   LOG(LS_INFO) << "Camera '" << GetId() << "' stopped after capturing "
    344                << captured_frames_ << " frames and dropping "
    345                << drop_ratio << "%";
    346 
    347   // Clear any pending async invokes (that OnIncomingCapturedFrame may have
    348   // caused).
    349   async_invoker_.reset();
    350 
    351   SetCaptureFormat(NULL);
    352   start_thread_ = nullptr;
    353   SetCaptureState(CS_STOPPED);
    354 }
    355 
    356 bool WebRtcVideoCapturer::IsRunning() {
    357   return (module_ != NULL && module_->CaptureStarted());
    358 }
    359 
    360 bool WebRtcVideoCapturer::GetPreferredFourccs(std::vector<uint32_t>* fourccs) {
    361   if (!fourccs) {
    362     return false;
    363   }
    364 
    365   fourccs->clear();
    366   for (size_t i = 0; i < arraysize(kSupportedFourCCs); ++i) {
    367     fourccs->push_back(kSupportedFourCCs[i].fourcc);
    368   }
    369   return true;
    370 }
    371 
    372 void WebRtcVideoCapturer::OnIncomingCapturedFrame(
    373     const int32_t id,
    374     const webrtc::VideoFrame& sample) {
    375   // This can only happen between Start() and Stop().
    376   RTC_DCHECK(start_thread_);
    377   RTC_DCHECK(async_invoker_);
    378   if (start_thread_->IsCurrent()) {
    379     SignalFrameCapturedOnStartThread(sample);
    380   } else {
    381     // This currently happens on with at least VideoCaptureModuleV4L2 and
    382     // possibly other implementations of WebRTC's VideoCaptureModule.
    383     // In order to maintain the threading contract with the upper layers and
    384     // consistency with other capturers such as in Chrome, we need to do a
    385     // thread hop.
    386     // Note that Stop() can cause the async invoke call to be cancelled.
    387     async_invoker_->AsyncInvoke<void>(
    388         start_thread_,
    389         // Note that Bind captures by value, so there's an intermediate copy
    390         // of sample.
    391         rtc::Bind(&WebRtcVideoCapturer::SignalFrameCapturedOnStartThread, this,
    392                   sample));
    393   }
    394 }
    395 
    396 void WebRtcVideoCapturer::OnCaptureDelayChanged(const int32_t id,
    397                                                 const int32_t delay) {
    398   LOG(LS_INFO) << "Capture delay changed to " << delay << " ms";
    399 }
    400 
    401 void WebRtcVideoCapturer::SignalFrameCapturedOnStartThread(
    402     const webrtc::VideoFrame& frame) {
    403   // This can only happen between Start() and Stop().
    404   RTC_DCHECK(start_thread_);
    405   RTC_DCHECK(start_thread_->IsCurrent());
    406   RTC_DCHECK(async_invoker_);
    407 
    408   ++captured_frames_;
    409   // Log the size and pixel aspect ratio of the first captured frame.
    410   if (1 == captured_frames_) {
    411     LOG(LS_INFO) << "Captured frame size "
    412                  << frame.width() << "x" << frame.height()
    413                  << ". Expected format " << GetCaptureFormat()->ToString();
    414   }
    415 
    416   // Signal down stream components on captured frame.
    417   // The CapturedFrame class doesn't support planes. We have to ExtractBuffer
    418   // to one block for it.
    419   size_t length =
    420       webrtc::CalcBufferSize(webrtc::kI420, frame.width(), frame.height());
    421   capture_buffer_.resize(length);
    422   // TODO(magjed): Refactor the WebRtcCapturedFrame to avoid memory copy or
    423   // take over ownership of the buffer held by |frame| if that's possible.
    424   webrtc::ExtractBuffer(frame, length, &capture_buffer_[0]);
    425   WebRtcCapturedFrame webrtc_frame(frame, &capture_buffer_[0], length);
    426   SignalFrameCaptured(this, &webrtc_frame);
    427 }
    428 
    429 // WebRtcCapturedFrame
    430 WebRtcCapturedFrame::WebRtcCapturedFrame(const webrtc::VideoFrame& sample,
    431                                          void* buffer,
    432                                          size_t length) {
    433   width = sample.width();
    434   height = sample.height();
    435   fourcc = FOURCC_I420;
    436   // TODO(hellner): Support pixel aspect ratio (for OSX).
    437   pixel_width = 1;
    438   pixel_height = 1;
    439   // Convert units from VideoFrame RenderTimeMs to CapturedFrame (nanoseconds).
    440   time_stamp = sample.render_time_ms() * rtc::kNumNanosecsPerMillisec;
    441   data_size = rtc::checked_cast<uint32_t>(length);
    442   data = buffer;
    443   rotation = sample.rotation();
    444 }
    445 
    446 }  // namespace cricket
    447 
    448 #endif  // HAVE_WEBRTC_VIDEO
    449