Home | History | Annotate | Download | only in objc 
      1 /*
      2  * libjingle
      3  * Copyright 2015 Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #include "talk/app/webrtc/objc/avfoundationvideocapturer.h"
     29 
     30 #include "webrtc/base/bind.h"
     31 
     32 #import <AVFoundation/AVFoundation.h>
     33 #import <Foundation/Foundation.h>
     34 #import <UIKit/UIKit.h>
     35 
     36 #import "webrtc/base/objc/RTCDispatcher.h"
     37 
     38 // TODO(tkchin): support other formats.
     39 static NSString* const kDefaultPreset = AVCaptureSessionPreset640x480;
     40 static cricket::VideoFormat const kDefaultFormat =
     41     cricket::VideoFormat(640,
     42                          480,
     43                          cricket::VideoFormat::FpsToInterval(30),
     44                          cricket::FOURCC_NV12);
     45 
     46 // This class used to capture frames using AVFoundation APIs on iOS. It is meant
     47 // to be owned by an instance of AVFoundationVideoCapturer. The reason for this
     48 // because other webrtc objects own cricket::VideoCapturer, which is not
     49 // ref counted. To prevent bad behavior we do not expose this class directly.
     50 @interface RTCAVFoundationVideoCapturerInternal : NSObject
     51     <AVCaptureVideoDataOutputSampleBufferDelegate>
     52 
     53 @property(nonatomic, readonly) AVCaptureSession* captureSession;
     54 @property(nonatomic, readonly) BOOL isRunning;
     55 @property(nonatomic, assign) BOOL useBackCamera;  // Defaults to NO.
     56 
     57 // We keep a pointer back to AVFoundationVideoCapturer to make callbacks on it
     58 // when we receive frames. This is safe because this object should be owned by
     59 // it.
     60 - (instancetype)initWithCapturer:(webrtc::AVFoundationVideoCapturer*)capturer;
     61 - (void)startCaptureAsync;
     62 - (void)stopCaptureAsync;
     63 
     64 @end
     65 
     66 @implementation RTCAVFoundationVideoCapturerInternal {
     67   // Keep pointers to inputs for convenience.
     68   AVCaptureDeviceInput* _frontDeviceInput;
     69   AVCaptureDeviceInput* _backDeviceInput;
     70   AVCaptureVideoDataOutput* _videoOutput;
     71   // The cricket::VideoCapturer that owns this class. Should never be NULL.
     72   webrtc::AVFoundationVideoCapturer* _capturer;
     73   BOOL _orientationHasChanged;
     74 }
     75 
     76 @synthesize captureSession = _captureSession;
     77 @synthesize useBackCamera = _useBackCamera;
     78 @synthesize isRunning = _isRunning;
     79 
     80 - (instancetype)initWithCapturer:(webrtc::AVFoundationVideoCapturer*)capturer {
     81   NSParameterAssert(capturer);
     82   if (self = [super init]) {
     83     _capturer = capturer;
     84     if (![self setupCaptureSession]) {
     85       return nil;
     86     }
     87     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
     88     [center addObserver:self
     89                selector:@selector(deviceOrientationDidChange:)
     90                    name:UIDeviceOrientationDidChangeNotification
     91                  object:nil];
     92     [center addObserverForName:AVCaptureSessionRuntimeErrorNotification
     93                         object:nil
     94                          queue:nil
     95                     usingBlock:^(NSNotification* notification) {
     96       NSLog(@"Capture session error: %@", notification.userInfo);
     97     }];
     98   }
     99   return self;
    100 }
    101 
    102 - (void)dealloc {
    103   [self stopCaptureAsync];
    104   [[NSNotificationCenter defaultCenter] removeObserver:self];
    105   _capturer = nullptr;
    106 }
    107 
    108 - (void)setUseBackCamera:(BOOL)useBackCamera {
    109   if (_useBackCamera == useBackCamera) {
    110     return;
    111   }
    112   _useBackCamera = useBackCamera;
    113   [self updateSessionInput];
    114 }
    115 
    116 - (void)startCaptureAsync {
    117   if (_isRunning) {
    118     return;
    119   }
    120   _orientationHasChanged = NO;
    121   [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
    122   AVCaptureSession* session = _captureSession;
    123   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
    124                                block:^{
    125     [session startRunning];
    126   }];
    127   _isRunning = YES;
    128 }
    129 
    130 - (void)stopCaptureAsync {
    131   if (!_isRunning) {
    132     return;
    133   }
    134   [_videoOutput setSampleBufferDelegate:nil queue:nullptr];
    135   AVCaptureSession* session = _captureSession;
    136   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
    137                                block:^{
    138     [session stopRunning];
    139   }];
    140   [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
    141   _isRunning = NO;
    142 }
    143 
    144 #pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
    145 
    146 - (void)captureOutput:(AVCaptureOutput*)captureOutput
    147     didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
    148            fromConnection:(AVCaptureConnection*)connection {
    149   NSParameterAssert(captureOutput == _videoOutput);
    150   if (!_isRunning) {
    151     return;
    152   }
    153   _capturer->CaptureSampleBuffer(sampleBuffer);
    154 }
    155 
    156 - (void)captureOutput:(AVCaptureOutput*)captureOutput
    157     didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
    158     fromConnection:(AVCaptureConnection*)connection {
    159   NSLog(@"Dropped sample buffer.");
    160 }
    161 
    162 #pragma mark - Private
    163 
    164 - (BOOL)setupCaptureSession {
    165   _captureSession = [[AVCaptureSession alloc] init];
    166 #if defined(__IPHONE_7_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_7_0
    167   NSString* version = [[UIDevice currentDevice] systemVersion];
    168   if ([version integerValue] >= 7) {
    169     _captureSession.usesApplicationAudioSession = NO;
    170   }
    171 #endif
    172   if (![_captureSession canSetSessionPreset:kDefaultPreset]) {
    173     NSLog(@"Default video capture preset unsupported.");
    174     return NO;
    175   }
    176   _captureSession.sessionPreset = kDefaultPreset;
    177 
    178   // Make the capturer output NV12. Ideally we want I420 but that's not
    179   // currently supported on iPhone / iPad.
    180   _videoOutput = [[AVCaptureVideoDataOutput alloc] init];
    181   _videoOutput.videoSettings = @{
    182     (NSString*)kCVPixelBufferPixelFormatTypeKey :
    183         @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
    184   };
    185   _videoOutput.alwaysDiscardsLateVideoFrames = NO;
    186   [_videoOutput setSampleBufferDelegate:self
    187                                   queue:dispatch_get_main_queue()];
    188   if (![_captureSession canAddOutput:_videoOutput]) {
    189     NSLog(@"Default video capture output unsupported.");
    190     return NO;
    191   }
    192   [_captureSession addOutput:_videoOutput];
    193 
    194   // Find the capture devices.
    195   AVCaptureDevice* frontCaptureDevice = nil;
    196   AVCaptureDevice* backCaptureDevice = nil;
    197   for (AVCaptureDevice* captureDevice in
    198        [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {
    199     if (captureDevice.position == AVCaptureDevicePositionBack) {
    200       backCaptureDevice = captureDevice;
    201     }
    202     if (captureDevice.position == AVCaptureDevicePositionFront) {
    203       frontCaptureDevice = captureDevice;
    204     }
    205   }
    206   if (!frontCaptureDevice || !backCaptureDevice) {
    207     NSLog(@"Failed to get capture devices.");
    208     return NO;
    209   }
    210 
    211   // Set up the session inputs.
    212   NSError* error = nil;
    213   _frontDeviceInput =
    214       [AVCaptureDeviceInput deviceInputWithDevice:frontCaptureDevice
    215                                             error:&error];
    216   if (!_frontDeviceInput) {
    217     NSLog(@"Failed to get capture device input: %@",
    218           error.localizedDescription);
    219     return NO;
    220   }
    221   _backDeviceInput =
    222       [AVCaptureDeviceInput deviceInputWithDevice:backCaptureDevice
    223                                             error:&error];
    224   if (!_backDeviceInput) {
    225     NSLog(@"Failed to get capture device input: %@",
    226           error.localizedDescription);
    227     return NO;
    228   }
    229 
    230   // Add the inputs.
    231   if (![_captureSession canAddInput:_frontDeviceInput] ||
    232       ![_captureSession canAddInput:_backDeviceInput]) {
    233     NSLog(@"Session does not support capture inputs.");
    234     return NO;
    235   }
    236   [self updateSessionInput];
    237 
    238   return YES;
    239 }
    240 
    241 - (void)deviceOrientationDidChange:(NSNotification*)notification {
    242   _orientationHasChanged = YES;
    243   [self updateOrientation];
    244 }
    245 
    246 - (void)updateOrientation {
    247   AVCaptureConnection* connection =
    248       [_videoOutput connectionWithMediaType:AVMediaTypeVideo];
    249   if (!connection.supportsVideoOrientation) {
    250     // TODO(tkchin): set rotation bit on frames.
    251     return;
    252   }
    253   AVCaptureVideoOrientation orientation = AVCaptureVideoOrientationPortrait;
    254   switch ([UIDevice currentDevice].orientation) {
    255     case UIDeviceOrientationPortrait:
    256       orientation = AVCaptureVideoOrientationPortrait;
    257       break;
    258     case UIDeviceOrientationPortraitUpsideDown:
    259       orientation = AVCaptureVideoOrientationPortraitUpsideDown;
    260       break;
    261     case UIDeviceOrientationLandscapeLeft:
    262       orientation = AVCaptureVideoOrientationLandscapeRight;
    263       break;
    264     case UIDeviceOrientationLandscapeRight:
    265       orientation = AVCaptureVideoOrientationLandscapeLeft;
    266       break;
    267     case UIDeviceOrientationFaceUp:
    268     case UIDeviceOrientationFaceDown:
    269     case UIDeviceOrientationUnknown:
    270       if (!_orientationHasChanged) {
    271         connection.videoOrientation = orientation;
    272       }
    273       return;
    274   }
    275   connection.videoOrientation = orientation;
    276 }
    277 
    278 - (void)updateSessionInput {
    279   // Update the current session input to match what's stored in _useBackCamera.
    280   [_captureSession beginConfiguration];
    281   AVCaptureDeviceInput* oldInput = _backDeviceInput;
    282   AVCaptureDeviceInput* newInput = _frontDeviceInput;
    283   if (_useBackCamera) {
    284     oldInput = _frontDeviceInput;
    285     newInput = _backDeviceInput;
    286   }
    287   // Ok to remove this even if it's not attached. Will be no-op.
    288   [_captureSession removeInput:oldInput];
    289   [_captureSession addInput:newInput];
    290   [self updateOrientation];
    291   [_captureSession commitConfiguration];
    292 }
    293 
    294 @end
    295 
    296 namespace webrtc {
    297 
    298 AVFoundationVideoCapturer::AVFoundationVideoCapturer()
    299     : _capturer(nil), _startThread(nullptr) {
    300   // Set our supported formats. This matches kDefaultPreset.
    301   std::vector<cricket::VideoFormat> supportedFormats;
    302   supportedFormats.push_back(cricket::VideoFormat(kDefaultFormat));
    303   SetSupportedFormats(supportedFormats);
    304   _capturer =
    305       [[RTCAVFoundationVideoCapturerInternal alloc] initWithCapturer:this];
    306 }
    307 
    308 AVFoundationVideoCapturer::~AVFoundationVideoCapturer() {
    309   _capturer = nil;
    310 }
    311 
    312 cricket::CaptureState AVFoundationVideoCapturer::Start(
    313     const cricket::VideoFormat& format) {
    314   if (!_capturer) {
    315     LOG(LS_ERROR) << "Failed to create AVFoundation capturer.";
    316     return cricket::CaptureState::CS_FAILED;
    317   }
    318   if (_capturer.isRunning) {
    319     LOG(LS_ERROR) << "The capturer is already running.";
    320     return cricket::CaptureState::CS_FAILED;
    321   }
    322   if (format != kDefaultFormat) {
    323     LOG(LS_ERROR) << "Unsupported format provided.";
    324     return cricket::CaptureState::CS_FAILED;
    325   }
    326 
    327   // Keep track of which thread capture started on. This is the thread that
    328   // frames need to be sent to.
    329   RTC_DCHECK(!_startThread);
    330   _startThread = rtc::Thread::Current();
    331 
    332   SetCaptureFormat(&format);
    333   // This isn't super accurate because it takes a while for the AVCaptureSession
    334   // to spin up, and this call returns async.
    335   // TODO(tkchin): make this better.
    336   [_capturer startCaptureAsync];
    337   SetCaptureState(cricket::CaptureState::CS_RUNNING);
    338 
    339   return cricket::CaptureState::CS_STARTING;
    340 }
    341 
    342 void AVFoundationVideoCapturer::Stop() {
    343   [_capturer stopCaptureAsync];
    344   SetCaptureFormat(NULL);
    345   _startThread = nullptr;
    346 }
    347 
    348 bool AVFoundationVideoCapturer::IsRunning() {
    349   return _capturer.isRunning;
    350 }
    351 
    352 AVCaptureSession* AVFoundationVideoCapturer::GetCaptureSession() {
    353   return _capturer.captureSession;
    354 }
    355 
    356 void AVFoundationVideoCapturer::SetUseBackCamera(bool useBackCamera) {
    357   _capturer.useBackCamera = useBackCamera;
    358 }
    359 
    360 bool AVFoundationVideoCapturer::GetUseBackCamera() const {
    361   return _capturer.useBackCamera;
    362 }
    363 
    364 void AVFoundationVideoCapturer::CaptureSampleBuffer(
    365     CMSampleBufferRef sampleBuffer) {
    366   if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 ||
    367       !CMSampleBufferIsValid(sampleBuffer) ||
    368       !CMSampleBufferDataIsReady(sampleBuffer)) {
    369     return;
    370   }
    371 
    372   CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    373   if (imageBuffer == NULL) {
    374     return;
    375   }
    376 
    377   // Base address must be unlocked to access frame data.
    378   CVOptionFlags lockFlags = kCVPixelBufferLock_ReadOnly;
    379   CVReturn ret = CVPixelBufferLockBaseAddress(imageBuffer, lockFlags);
    380   if (ret != kCVReturnSuccess) {
    381     return;
    382   }
    383 
    384   static size_t const kYPlaneIndex = 0;
    385   static size_t const kUVPlaneIndex = 1;
    386   uint8_t* yPlaneAddress =
    387       (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(imageBuffer, kYPlaneIndex);
    388   size_t yPlaneHeight =
    389       CVPixelBufferGetHeightOfPlane(imageBuffer, kYPlaneIndex);
    390   size_t yPlaneWidth =
    391       CVPixelBufferGetWidthOfPlane(imageBuffer, kYPlaneIndex);
    392   size_t yPlaneBytesPerRow =
    393       CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, kYPlaneIndex);
    394   size_t uvPlaneHeight =
    395       CVPixelBufferGetHeightOfPlane(imageBuffer, kUVPlaneIndex);
    396   size_t uvPlaneBytesPerRow =
    397       CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, kUVPlaneIndex);
    398   size_t frameSize =
    399       yPlaneBytesPerRow * yPlaneHeight + uvPlaneBytesPerRow * uvPlaneHeight;
    400 
    401   // Sanity check assumption that planar bytes are contiguous.
    402   uint8_t* uvPlaneAddress =
    403       (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(imageBuffer, kUVPlaneIndex);
    404   RTC_DCHECK(
    405       uvPlaneAddress == yPlaneAddress + yPlaneHeight * yPlaneBytesPerRow);
    406 
    407   // Stuff data into a cricket::CapturedFrame.
    408   int64_t currentTime = rtc::TimeNanos();
    409   cricket::CapturedFrame frame;
    410   frame.width = yPlaneWidth;
    411   frame.height = yPlaneHeight;
    412   frame.pixel_width = 1;
    413   frame.pixel_height = 1;
    414   frame.fourcc = static_cast<uint32_t>(cricket::FOURCC_NV12);
    415   frame.time_stamp = currentTime;
    416   frame.data = yPlaneAddress;
    417   frame.data_size = frameSize;
    418 
    419   if (_startThread->IsCurrent()) {
    420     SignalFrameCaptured(this, &frame);
    421   } else {
    422     _startThread->Invoke<void>(
    423         rtc::Bind(&AVFoundationVideoCapturer::SignalFrameCapturedOnStartThread,
    424                   this, &frame));
    425   }
    426   CVPixelBufferUnlockBaseAddress(imageBuffer, lockFlags);
    427 }
    428 
    429 void AVFoundationVideoCapturer::SignalFrameCapturedOnStartThread(
    430     const cricket::CapturedFrame* frame) {
    431   RTC_DCHECK(_startThread->IsCurrent());
    432   // This will call a superclass method that will perform the frame conversion
    433   // to I420.
    434   SignalFrameCaptured(this, frame);
    435 }
    436 
    437 }  // namespace webrtc
    438