summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m')
-rw-r--r--third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m535
1 files changed, 535 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m
new file mode 100644
index 0000000000..98d3cf9f45
--- /dev/null
+++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m
@@ -0,0 +1,535 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCCameraVideoCapturer.h"
+#import "base/RTCLogging.h"
+#import "base/RTCVideoFrameBuffer.h"
+#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
+
+#if TARGET_OS_IPHONE
+#import "helpers/UIDevice+RTCDevice.h"
+#endif
+
+#import "helpers/AVCaptureSession+DevicePosition.h"
+#import "helpers/RTCDispatcher+Private.h"
+#include "rtc_base/system/gcd_helpers.h"
+
+const int64_t kNanosecondsPerSecond = 1000000000;
+
+@interface RTC_OBJC_TYPE (RTCCameraVideoCapturer)
+()<AVCaptureVideoDataOutputSampleBufferDelegate> @property(nonatomic,
+ readonly) dispatch_queue_t frameQueue;
+@property(nonatomic, strong) AVCaptureDevice *currentDevice;
+@property(nonatomic, assign) BOOL hasRetriedOnFatalError;
+@property(nonatomic, assign) BOOL isRunning;
+// Will the session be running once all asynchronous operations have been completed?
+@property(nonatomic, assign) BOOL willBeRunning;
+@end
+
+@implementation RTC_OBJC_TYPE (RTCCameraVideoCapturer) {
+ AVCaptureVideoDataOutput *_videoDataOutput;
+ AVCaptureSession *_captureSession;
+ FourCharCode _preferredOutputPixelFormat;
+ FourCharCode _outputPixelFormat;
+ RTCVideoRotation _rotation;
+#if TARGET_OS_IPHONE
+ UIDeviceOrientation _orientation;
+ BOOL _generatingOrientationNotifications;
+#endif
+}
+
+@synthesize frameQueue = _frameQueue;
+@synthesize captureSession = _captureSession;
+@synthesize currentDevice = _currentDevice;
+@synthesize hasRetriedOnFatalError = _hasRetriedOnFatalError;
+@synthesize isRunning = _isRunning;
+@synthesize willBeRunning = _willBeRunning;
+
+- (instancetype)init {
+ return [self initWithDelegate:nil captureSession:[[AVCaptureSession alloc] init]];
+}
+
+- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate {
+ return [self initWithDelegate:delegate captureSession:[[AVCaptureSession alloc] init]];
+}
+
+// This initializer is used for testing.
+- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate
+ captureSession:(AVCaptureSession *)captureSession {
+ if (self = [super initWithDelegate:delegate]) {
+ // Create the capture session and all relevant inputs and outputs. We need
+ // to do this in init because the application may want the capture session
+ // before we start the capturer for e.g. AVCapturePreviewLayer. All objects
+ // created here are retained until dealloc and never recreated.
+ if (![self setupCaptureSession:captureSession]) {
+ return nil;
+ }
+ NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+#if TARGET_OS_IPHONE
+ _orientation = UIDeviceOrientationPortrait;
+ _rotation = RTCVideoRotation_90;
+ [center addObserver:self
+ selector:@selector(deviceOrientationDidChange:)
+ name:UIDeviceOrientationDidChangeNotification
+ object:nil];
+ [center addObserver:self
+ selector:@selector(handleCaptureSessionInterruption:)
+ name:AVCaptureSessionWasInterruptedNotification
+ object:_captureSession];
+ [center addObserver:self
+ selector:@selector(handleCaptureSessionInterruptionEnded:)
+ name:AVCaptureSessionInterruptionEndedNotification
+ object:_captureSession];
+ [center addObserver:self
+ selector:@selector(handleApplicationDidBecomeActive:)
+ name:UIApplicationDidBecomeActiveNotification
+ object:[UIApplication sharedApplication]];
+#endif
+ [center addObserver:self
+ selector:@selector(handleCaptureSessionRuntimeError:)
+ name:AVCaptureSessionRuntimeErrorNotification
+ object:_captureSession];
+ [center addObserver:self
+ selector:@selector(handleCaptureSessionDidStartRunning:)
+ name:AVCaptureSessionDidStartRunningNotification
+ object:_captureSession];
+ [center addObserver:self
+ selector:@selector(handleCaptureSessionDidStopRunning:)
+ name:AVCaptureSessionDidStopRunningNotification
+ object:_captureSession];
+ }
+ return self;
+}
+
+- (void)dealloc {
+ NSAssert(!_willBeRunning,
+ @"Session was still running in RTC_OBJC_TYPE(RTCCameraVideoCapturer) dealloc. Forgot to "
+ @"call stopCapture?");
+ [[NSNotificationCenter defaultCenter] removeObserver:self];
+}
+
++ (NSArray<AVCaptureDevice *> *)captureDevices {
+#if defined(WEBRTC_IOS) && defined(__IPHONE_10_0) && \
+ __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_10_0
+ AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession
+ discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ]
+ mediaType:AVMediaTypeVideo
+ position:AVCaptureDevicePositionUnspecified];
+ return session.devices;
+#else
+ return [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+#endif
+}
+
++ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
+ // Support opening the device in any format. We make sure it's converted to a format we
+ // can handle, if needed, in the method `-setupVideoDataOutput`.
+ return device.formats;
+}
+
+- (FourCharCode)preferredOutputPixelFormat {
+ return _preferredOutputPixelFormat;
+}
+
+- (void)startCaptureWithDevice:(AVCaptureDevice *)device
+ format:(AVCaptureDeviceFormat *)format
+ fps:(NSInteger)fps {
+ [self startCaptureWithDevice:device format:format fps:fps completionHandler:nil];
+}
+
+- (void)stopCapture {
+ [self stopCaptureWithCompletionHandler:nil];
+}
+
+- (void)startCaptureWithDevice:(AVCaptureDevice *)device
+ format:(AVCaptureDeviceFormat *)format
+ fps:(NSInteger)fps
+ completionHandler:(nullable void (^)(NSError *_Nullable error))completionHandler {
+ _willBeRunning = YES;
+ [RTC_OBJC_TYPE(RTCDispatcher)
+ dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps);
+
+#if TARGET_OS_IPHONE
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (!self->_generatingOrientationNotifications) {
+ [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
+ self->_generatingOrientationNotifications = YES;
+ }
+ });
+#endif
+
+ self.currentDevice = device;
+
+ NSError *error = nil;
+ if (![self.currentDevice lockForConfiguration:&error]) {
+ RTCLogError(@"Failed to lock device %@. Error: %@",
+ self.currentDevice,
+ error.userInfo);
+ if (completionHandler) {
+ completionHandler(error);
+ }
+ self.willBeRunning = NO;
+ return;
+ }
+ [self reconfigureCaptureSessionInput];
+ [self updateOrientation];
+ [self updateDeviceCaptureFormat:format fps:fps];
+ [self updateVideoDataOutputPixelFormat:format];
+ [self.captureSession startRunning];
+ [self.currentDevice unlockForConfiguration];
+ self.isRunning = YES;
+ if (completionHandler) {
+ completionHandler(nil);
+ }
+ }];
+}
+
+- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler {
+ _willBeRunning = NO;
+ [RTC_OBJC_TYPE(RTCDispatcher)
+ dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ RTCLogInfo("Stop");
+ self.currentDevice = nil;
+ for (AVCaptureDeviceInput *oldInput in [self.captureSession.inputs copy]) {
+ [self.captureSession removeInput:oldInput];
+ }
+ [self.captureSession stopRunning];
+
+#if TARGET_OS_IPHONE
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_generatingOrientationNotifications) {
+ [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
+ self->_generatingOrientationNotifications = NO;
+ }
+ });
+#endif
+ self.isRunning = NO;
+ if (completionHandler) {
+ completionHandler();
+ }
+ }];
+}
+
+#pragma mark iOS notifications
+
+#if TARGET_OS_IPHONE
+- (void)deviceOrientationDidChange:(NSNotification *)notification {
+ [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ [self updateOrientation];
+ }];
+}
+#endif
+
+#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+ fromConnection:(AVCaptureConnection *)connection {
+ NSParameterAssert(captureOutput == _videoDataOutput);
+
+ if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
+ !CMSampleBufferDataIsReady(sampleBuffer)) {
+ return;
+ }
+
+ CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+ if (pixelBuffer == nil) {
+ return;
+ }
+
+#if TARGET_OS_IPHONE
+ // Default to portrait orientation on iPhone.
+ BOOL usingFrontCamera = NO;
+ // Check the image's EXIF for the camera the image came from as the image could have been
+ // delayed as we set alwaysDiscardsLateVideoFrames to NO.
+ AVCaptureDevicePosition cameraPosition =
+ [AVCaptureSession devicePositionForSampleBuffer:sampleBuffer];
+ if (cameraPosition != AVCaptureDevicePositionUnspecified) {
+ usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition;
+ } else {
+ AVCaptureDeviceInput *deviceInput =
+ (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
+ usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position;
+ }
+ switch (_orientation) {
+ case UIDeviceOrientationPortrait:
+ _rotation = RTCVideoRotation_90;
+ break;
+ case UIDeviceOrientationPortraitUpsideDown:
+ _rotation = RTCVideoRotation_270;
+ break;
+ case UIDeviceOrientationLandscapeLeft:
+ _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
+ break;
+ case UIDeviceOrientationLandscapeRight:
+ _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
+ break;
+ case UIDeviceOrientationFaceUp:
+ case UIDeviceOrientationFaceDown:
+ case UIDeviceOrientationUnknown:
+ // Ignore.
+ break;
+ }
+#else
+ // No rotation on Mac.
+ _rotation = RTCVideoRotation_0;
+#endif
+
+ RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer =
+ [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer];
+ int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
+ kNanosecondsPerSecond;
+ RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame =
+ [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer
+ rotation:_rotation
+ timeStampNs:timeStampNs];
+ [self.delegate capturer:self didCaptureVideoFrame:videoFrame];
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+ didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
+ fromConnection:(AVCaptureConnection *)connection {
+#if TARGET_OS_IPHONE
+ CFStringRef droppedReason =
+ CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil);
+#else
+ // DroppedFrameReason unavailable on macOS.
+ CFStringRef droppedReason = nil;
+#endif
+ RTCLogError(@"Dropped sample buffer. Reason: %@", (__bridge NSString *)droppedReason);
+}
+
+#pragma mark - AVCaptureSession notifications
+
+- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
+ NSString *reasonString = nil;
+#if TARGET_OS_IPHONE
+ NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey];
+ if (reason) {
+ switch (reason.intValue) {
+ case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground:
+ reasonString = @"VideoDeviceNotAvailableInBackground";
+ break;
+ case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient:
+ reasonString = @"AudioDeviceInUseByAnotherClient";
+ break;
+ case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient:
+ reasonString = @"VideoDeviceInUseByAnotherClient";
+ break;
+ case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps:
+ reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps";
+ break;
+ }
+ }
+#endif
+ RTCLog(@"Capture session interrupted: %@", reasonString);
+}
+
+- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
+ RTCLog(@"Capture session interruption ended.");
+}
+
+- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
+ NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
+ RTCLogError(@"Capture session runtime error: %@", error);
+
+ [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+#if TARGET_OS_IPHONE
+ if (error.code == AVErrorMediaServicesWereReset) {
+ [self handleNonFatalError];
+ } else {
+ [self handleFatalError];
+ }
+#else
+ [self handleFatalError];
+#endif
+ }];
+}
+
+- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
+ RTCLog(@"Capture session started.");
+
+ [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ // If we successfully restarted after an unknown
+ // error, allow future retries on fatal errors.
+ self.hasRetriedOnFatalError = NO;
+ }];
+}
+
+- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
+ RTCLog(@"Capture session stopped.");
+}
+
+- (void)handleFatalError {
+ [RTC_OBJC_TYPE(RTCDispatcher)
+ dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ if (!self.hasRetriedOnFatalError) {
+ RTCLogWarning(@"Attempting to recover from fatal capture error.");
+ [self handleNonFatalError];
+ self.hasRetriedOnFatalError = YES;
+ } else {
+ RTCLogError(@"Previous fatal error recovery failed.");
+ }
+ }];
+}
+
+- (void)handleNonFatalError {
+ [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ RTCLog(@"Restarting capture session after error.");
+ if (self.isRunning) {
+ [self.captureSession startRunning];
+ }
+ }];
+}
+
+#if TARGET_OS_IPHONE
+
+#pragma mark - UIApplication notifications
+
+- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
+ [RTC_OBJC_TYPE(RTCDispatcher)
+ dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+ block:^{
+ if (self.isRunning && !self.captureSession.isRunning) {
+ RTCLog(@"Restarting capture session on active.");
+ [self.captureSession startRunning];
+ }
+ }];
+}
+
+#endif // TARGET_OS_IPHONE
+
+#pragma mark - Private
+
+- (dispatch_queue_t)frameQueue {
+ if (!_frameQueue) {
+ _frameQueue = RTCDispatchQueueCreateWithTarget(
+ "org.webrtc.cameravideocapturer.video",
+ DISPATCH_QUEUE_SERIAL,
+ dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
+ }
+ return _frameQueue;
+}
+
+- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession {
+ NSAssert(_captureSession == nil, @"Setup capture session called twice.");
+ _captureSession = captureSession;
+#if defined(WEBRTC_IOS)
+ _captureSession.sessionPreset = AVCaptureSessionPresetInputPriority;
+ _captureSession.usesApplicationAudioSession = NO;
+#endif
+ [self setupVideoDataOutput];
+ // Add the output.
+ if (![_captureSession canAddOutput:_videoDataOutput]) {
+ RTCLogError(@"Video data output unsupported.");
+ return NO;
+ }
+ [_captureSession addOutput:_videoDataOutput];
+
+ return YES;
+}
+
+- (void)setupVideoDataOutput {
+ NSAssert(_videoDataOutput == nil, @"Setup video data output called twice.");
+ AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
+
+ // `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the
+ // device with the most efficient output format first. Find the first format that we support.
+ NSSet<NSNumber *> *supportedPixelFormats =
+ [RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats];
+ NSMutableOrderedSet *availablePixelFormats =
+ [NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes];
+ [availablePixelFormats intersectSet:supportedPixelFormats];
+ NSNumber *pixelFormat = availablePixelFormats.firstObject;
+ NSAssert(pixelFormat, @"Output device has no supported formats.");
+
+ _preferredOutputPixelFormat = [pixelFormat unsignedIntValue];
+ _outputPixelFormat = _preferredOutputPixelFormat;
+ videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat};
+ videoDataOutput.alwaysDiscardsLateVideoFrames = NO;
+ [videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
+ _videoDataOutput = videoDataOutput;
+}
+
+- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format {
+ FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+ if (![[RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats] containsObject:@(mediaSubType)]) {
+ mediaSubType = _preferredOutputPixelFormat;
+ }
+
+ if (mediaSubType != _outputPixelFormat) {
+ _outputPixelFormat = mediaSubType;
+ }
+
+ // Update videoSettings with dimensions, as some virtual cameras, e.g. Snap Camera, may not work
+ // otherwise.
+ CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
+ _videoDataOutput.videoSettings = @{
+ (id)kCVPixelBufferWidthKey : @(dimensions.width),
+ (id)kCVPixelBufferHeightKey : @(dimensions.height),
+ (id)kCVPixelBufferPixelFormatTypeKey : @(_outputPixelFormat),
+ };
+}
+
+#pragma mark - Private, called inside capture queue
+
+- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps {
+ NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession],
+ @"updateDeviceCaptureFormat must be called on the capture queue.");
+ @try {
+ _currentDevice.activeFormat = format;
+ _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps);
+ } @catch (NSException *exception) {
+ RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo);
+ return;
+ }
+}
+
+- (void)reconfigureCaptureSessionInput {
+ NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession],
+ @"reconfigureCaptureSessionInput must be called on the capture queue.");
+ NSError *error = nil;
+ AVCaptureDeviceInput *input =
+ [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error];
+ if (!input) {
+ RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
+ return;
+ }
+ [_captureSession beginConfiguration];
+ for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
+ [_captureSession removeInput:oldInput];
+ }
+ if ([_captureSession canAddInput:input]) {
+ [_captureSession addInput:input];
+ } else {
+ RTCLogError(@"Cannot add camera as an input to the session.");
+ }
+ [_captureSession commitConfiguration];
+}
+
+- (void)updateOrientation {
+ NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession],
+ @"updateOrientation must be called on the capture queue.");
+#if TARGET_OS_IPHONE
+ _orientation = [UIDevice currentDevice].orientation;
+#endif
+}
+
+@end