diff options
Diffstat (limited to 'third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m')
| -rw-r--r-- | third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m | 543 |
1 files changed, 543 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m new file mode 100644 index 0000000000..1361207faf --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m @@ -0,0 +1,543 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCameraVideoCapturer.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#if TARGET_OS_IPHONE +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#import "helpers/AVCaptureSession+DevicePosition.h" +#import "helpers/RTCDispatcher+Private.h" +#include "rtc_base/system/gcd_helpers.h" + +const int64_t kNanosecondsPerSecond = 1000000000; + +@interface RTC_OBJC_TYPE (RTCCameraVideoCapturer) +()<AVCaptureVideoDataOutputSampleBufferDelegate> @property(nonatomic, + readonly) dispatch_queue_t frameQueue; +@property(nonatomic, strong) AVCaptureDevice *currentDevice; +@property(nonatomic, assign) BOOL hasRetriedOnFatalError; +@property(nonatomic, assign) BOOL isRunning; +// Will the session be running once all asynchronous operations have been completed? +@property(nonatomic, assign) BOOL willBeRunning; +@end + +@implementation RTC_OBJC_TYPE (RTCCameraVideoCapturer) { + AVCaptureVideoDataOutput *_videoDataOutput; + AVCaptureSession *_captureSession; + FourCharCode _preferredOutputPixelFormat; + FourCharCode _outputPixelFormat; + RTCVideoRotation _rotation; +#if TARGET_OS_IPHONE + UIDeviceOrientation _orientation; + BOOL _generatingOrientationNotifications; +#endif +} + +@synthesize frameQueue = _frameQueue; +@synthesize captureSession = _captureSession; +@synthesize currentDevice = _currentDevice; +@synthesize hasRetriedOnFatalError = _hasRetriedOnFatalError; +@synthesize isRunning = _isRunning; +@synthesize willBeRunning = _willBeRunning; + +- (instancetype)init { + return [self initWithDelegate:nil captureSession:[[AVCaptureSession alloc] init]]; +} + +- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate { + return [self initWithDelegate:delegate captureSession:[[AVCaptureSession alloc] init]]; +} + +// This initializer is used for testing. +- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate + captureSession:(AVCaptureSession *)captureSession { + if (self = [super initWithDelegate:delegate]) { + // Create the capture session and all relevant inputs and outputs. We need + // to do this in init because the application may want the capture session + // before we start the capturer for e.g. AVCapturePreviewLayer. All objects + // created here are retained until dealloc and never recreated. + if (![self setupCaptureSession:captureSession]) { + return nil; + } + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; +#if TARGET_OS_IPHONE + _orientation = UIDeviceOrientationPortrait; + _rotation = RTCVideoRotation_90; + [center addObserver:self + selector:@selector(deviceOrientationDidChange:) + name:UIDeviceOrientationDidChangeNotification + object:nil]; + [center addObserver:self + selector:@selector(handleCaptureSessionInterruption:) + name:AVCaptureSessionWasInterruptedNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionInterruptionEnded:) + name:AVCaptureSessionInterruptionEndedNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleApplicationDidBecomeActive:) + name:UIApplicationDidBecomeActiveNotification + object:[UIApplication sharedApplication]]; +#endif + [center addObserver:self + selector:@selector(handleCaptureSessionRuntimeError:) + name:AVCaptureSessionRuntimeErrorNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionDidStartRunning:) + name:AVCaptureSessionDidStartRunningNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionDidStopRunning:) + name:AVCaptureSessionDidStopRunningNotification + object:_captureSession]; + } + return self; +} + +- (void)dealloc { + NSAssert(!_willBeRunning, + @"Session was still running in RTC_OBJC_TYPE(RTCCameraVideoCapturer) dealloc. Forgot to " + @"call stopCapture?"); + [[NSNotificationCenter defaultCenter] removeObserver:self]; +} + ++ (NSArray<AVCaptureDevice *> *)captureDevicesWithDeviceTypes: + (NSArray<AVCaptureDeviceType> *)deviceTypes { + AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:deviceTypes + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + return session.devices; +} + ++ (NSArray<AVCaptureDeviceType> *)defaultCaptureDeviceTypes { + NSArray *types = @[ AVCaptureDeviceTypeBuiltInWideAngleCamera ]; +#if !defined(WEBRTC_IOS) + if (@available(macOS 14.0, *)) { + types = [types arrayByAddingObject:AVCaptureDeviceTypeExternal]; + } else { + types = [types arrayByAddingObject:AVCaptureDeviceTypeExternalUnknown]; + } +#endif + return types; +} + ++ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device { + // Support opening the device in any format. We make sure it's converted to a format we + // can handle, if needed, in the method `-setupVideoDataOutput`. + return device.formats; +} + +- (FourCharCode)preferredOutputPixelFormat { + return _preferredOutputPixelFormat; +} + +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps { + [self startCaptureWithDevice:device format:format fps:fps completionHandler:nil]; +} + +- (void)stopCapture { + [self stopCaptureWithCompletionHandler:nil]; +} + +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps + completionHandler:(nullable void (^)(NSError *_Nullable error))completionHandler { + _willBeRunning = YES; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps); + +#if TARGET_OS_IPHONE + dispatch_async(dispatch_get_main_queue(), ^{ + if (!self->_generatingOrientationNotifications) { + [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications]; + self->_generatingOrientationNotifications = YES; + } + }); +#endif + + self.currentDevice = device; + + NSError *error = nil; + if (![self.currentDevice lockForConfiguration:&error]) { + RTCLogError(@"Failed to lock device %@. Error: %@", + self.currentDevice, + error.userInfo); + if (completionHandler) { + completionHandler(error); + } + self.willBeRunning = NO; + return; + } + [self reconfigureCaptureSessionInput]; + [self updateOrientation]; + [self updateDeviceCaptureFormat:format fps:fps]; + [self updateVideoDataOutputPixelFormat:format]; + [self.captureSession startRunning]; + [self.currentDevice unlockForConfiguration]; + self.isRunning = YES; + if (completionHandler) { + completionHandler(nil); + } + }]; +} + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler { + _willBeRunning = NO; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLogInfo("Stop"); + self.currentDevice = nil; + for (AVCaptureDeviceInput *oldInput in [self.captureSession.inputs copy]) { + [self.captureSession removeInput:oldInput]; + } + [self.captureSession stopRunning]; + +#if TARGET_OS_IPHONE + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_generatingOrientationNotifications) { + [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications]; + self->_generatingOrientationNotifications = NO; + } + }); +#endif + self.isRunning = NO; + if (completionHandler) { + completionHandler(); + } + }]; +} + +#pragma mark iOS notifications + +#if TARGET_OS_IPHONE +- (void)deviceOrientationDidChange:(NSNotification *)notification { + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + [self updateOrientation]; + }]; +} +#endif + +#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { + NSParameterAssert(captureOutput == _videoDataOutput); + + if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) || + !CMSampleBufferDataIsReady(sampleBuffer)) { + return; + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (pixelBuffer == nil) { + return; + } + +#if TARGET_OS_IPHONE + // Default to portrait orientation on iPhone. + BOOL usingFrontCamera = NO; + // Check the image's EXIF for the camera the image came from as the image could have been + // delayed as we set alwaysDiscardsLateVideoFrames to NO. + AVCaptureDevicePosition cameraPosition = + [AVCaptureSession devicePositionForSampleBuffer:sampleBuffer]; + if (cameraPosition != AVCaptureDevicePositionUnspecified) { + usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition; + } else { + AVCaptureDeviceInput *deviceInput = + (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input; + usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position; + } + switch (_orientation) { + case UIDeviceOrientationPortrait: + _rotation = RTCVideoRotation_90; + break; + case UIDeviceOrientationPortraitUpsideDown: + _rotation = RTCVideoRotation_270; + break; + case UIDeviceOrientationLandscapeLeft: + _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; + break; + case UIDeviceOrientationLandscapeRight: + _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180; + break; + case UIDeviceOrientationFaceUp: + case UIDeviceOrientationFaceDown: + case UIDeviceOrientationUnknown: + // Ignore. + break; + } +#else + // No rotation on Mac. + _rotation = RTCVideoRotation_0; +#endif + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; + int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * + kNanosecondsPerSecond; + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:_rotation + timeStampNs:timeStampNs]; + [self.delegate capturer:self didCaptureVideoFrame:videoFrame]; +} + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { +#if TARGET_OS_IPHONE + CFStringRef droppedReason = + CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil); +#else + // DroppedFrameReason unavailable on macOS. + CFStringRef droppedReason = nil; +#endif + RTCLogError(@"Dropped sample buffer. Reason: %@", (__bridge NSString *)droppedReason); +} + +#pragma mark - AVCaptureSession notifications + +- (void)handleCaptureSessionInterruption:(NSNotification *)notification { + NSString *reasonString = nil; +#if TARGET_OS_IPHONE + NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey]; + if (reason) { + switch (reason.intValue) { + case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground: + reasonString = @"VideoDeviceNotAvailableInBackground"; + break; + case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient: + reasonString = @"AudioDeviceInUseByAnotherClient"; + break; + case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient: + reasonString = @"VideoDeviceInUseByAnotherClient"; + break; + case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps: + reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps"; + break; + } + } +#endif + RTCLog(@"Capture session interrupted: %@", reasonString); +} + +- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification { + RTCLog(@"Capture session interruption ended."); +} + +- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification { + NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey]; + RTCLogError(@"Capture session runtime error: %@", error); + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ +#if TARGET_OS_IPHONE + if (error.code == AVErrorMediaServicesWereReset) { + [self handleNonFatalError]; + } else { + [self handleFatalError]; + } +#else + [self handleFatalError]; +#endif + }]; +} + +- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification { + RTCLog(@"Capture session started."); + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + // If we successfully restarted after an unknown + // error, allow future retries on fatal errors. + self.hasRetriedOnFatalError = NO; + }]; +} + +- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification { + RTCLog(@"Capture session stopped."); +} + +- (void)handleFatalError { + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + if (!self.hasRetriedOnFatalError) { + RTCLogWarning(@"Attempting to recover from fatal capture error."); + [self handleNonFatalError]; + self.hasRetriedOnFatalError = YES; + } else { + RTCLogError(@"Previous fatal error recovery failed."); + } + }]; +} + +- (void)handleNonFatalError { + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLog(@"Restarting capture session after error."); + if (self.isRunning) { + [self.captureSession startRunning]; + } + }]; +} + +#if TARGET_OS_IPHONE + +#pragma mark - UIApplication notifications + +- (void)handleApplicationDidBecomeActive:(NSNotification *)notification { + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + if (self.isRunning && !self.captureSession.isRunning) { + RTCLog(@"Restarting capture session on active."); + [self.captureSession startRunning]; + } + }]; +} + +#endif // TARGET_OS_IPHONE + +#pragma mark - Private + +- (dispatch_queue_t)frameQueue { + if (!_frameQueue) { + _frameQueue = RTCDispatchQueueCreateWithTarget( + "org.webrtc.cameravideocapturer.video", + DISPATCH_QUEUE_SERIAL, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0)); + } + return _frameQueue; +} + +- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession { + NSAssert(_captureSession == nil, @"Setup capture session called twice."); + _captureSession = captureSession; +#if defined(WEBRTC_IOS) + _captureSession.sessionPreset = AVCaptureSessionPresetInputPriority; + _captureSession.usesApplicationAudioSession = NO; +#endif + [self setupVideoDataOutput]; + // Add the output. + if (![_captureSession canAddOutput:_videoDataOutput]) { + RTCLogError(@"Video data output unsupported."); + return NO; + } + [_captureSession addOutput:_videoDataOutput]; + + return YES; +} + +- (void)setupVideoDataOutput { + NSAssert(_videoDataOutput == nil, @"Setup video data output called twice."); + AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; + + // `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the + // device with the most efficient output format first. Find the first format that we support. + NSSet<NSNumber *> *supportedPixelFormats = + [RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats]; + NSMutableOrderedSet *availablePixelFormats = + [NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes]; + [availablePixelFormats intersectSet:supportedPixelFormats]; + NSNumber *pixelFormat = availablePixelFormats.firstObject; + NSAssert(pixelFormat, @"Output device has no supported formats."); + + _preferredOutputPixelFormat = [pixelFormat unsignedIntValue]; + _outputPixelFormat = _preferredOutputPixelFormat; + videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat}; + videoDataOutput.alwaysDiscardsLateVideoFrames = NO; + [videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue]; + _videoDataOutput = videoDataOutput; +} + +- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format { + FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription); + if (![[RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats] containsObject:@(mediaSubType)]) { + mediaSubType = _preferredOutputPixelFormat; + } + + if (mediaSubType != _outputPixelFormat) { + _outputPixelFormat = mediaSubType; + } + + // Update videoSettings with dimensions, as some virtual cameras, e.g. Snap Camera, may not work + // otherwise. + CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription); + _videoDataOutput.videoSettings = @{ + (id)kCVPixelBufferWidthKey : @(dimensions.width), + (id)kCVPixelBufferHeightKey : @(dimensions.height), + (id)kCVPixelBufferPixelFormatTypeKey : @(_outputPixelFormat), + }; +} + +#pragma mark - Private, called inside capture queue + +- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"updateDeviceCaptureFormat must be called on the capture queue."); + @try { + _currentDevice.activeFormat = format; + _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + } @catch (NSException *exception) { + RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo); + return; + } +} + +- (void)reconfigureCaptureSessionInput { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"reconfigureCaptureSessionInput must be called on the capture queue."); + NSError *error = nil; + AVCaptureDeviceInput *input = + [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error]; + if (!input) { + RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); + return; + } + [_captureSession beginConfiguration]; + for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) { + [_captureSession removeInput:oldInput]; + } + if ([_captureSession canAddInput:input]) { + [_captureSession addInput:input]; + } else { + RTCLogError(@"Cannot add camera as an input to the session."); + } + [_captureSession commitConfiguration]; +} + +- (void)updateOrientation { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"updateOrientation must be called on the capture queue."); +#if TARGET_OS_IPHONE + _orientation = [UIDevice currentDevice].orientation; +#endif +} + +@end |