diff options
Diffstat (limited to 'third_party/libwebrtc/sdk/objc/components/video_frame_buffer')
| -rw-r--r-- | third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h | 52 | ||||
| -rw-r--r-- | third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm | 367 |
2 files changed, 419 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h new file mode 100644 index 0000000000..664d9bb904 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h @@ -0,0 +1,52 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrameBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCVideoFrameBuffer containing a CVPixelBufferRef */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCVPixelBuffer) : NSObject <RTC_OBJC_TYPE(RTCVideoFrameBuffer)> + +@property(nonatomic, readonly) CVPixelBufferRef pixelBuffer; +@property(nonatomic, readonly) int cropX; +@property(nonatomic, readonly) int cropY; +@property(nonatomic, readonly) int cropWidth; +@property(nonatomic, readonly) int cropHeight; + ++ (NSSet<NSNumber *> *)supportedPixelFormats; + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer; +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY; + +- (BOOL)requiresCropping; +- (BOOL)requiresScalingToWidth:(int)width height:(int)height; +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height; + +/** The minimum size of the `tmpBuffer` must be the number of bytes returned from the + * bufferSizeForCroppingAndScalingToWidth:height: method. + * If that size is 0, the `tmpBuffer` may be nil. + */ +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t *)tmpBuffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm new file mode 100644 index 0000000000..1a9b672d1a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm @@ -0,0 +1,367 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCVPixelBuffer.h" + +#import "api/video_frame_buffer/RTCNativeMutableI420Buffer.h" + +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv.h" + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +#import <UIKit/UIKit.h> +#import <VideoToolbox/VideoToolbox.h> +#endif + +@implementation RTC_OBJC_TYPE (RTCCVPixelBuffer) { + int _width; + int _height; + int _bufferWidth; + int _bufferHeight; + int _cropWidth; + int _cropHeight; +} + +@synthesize pixelBuffer = _pixelBuffer; +@synthesize cropX = _cropX; +@synthesize cropY = _cropY; +@synthesize cropWidth = _cropWidth; +@synthesize cropHeight = _cropHeight; + ++ (NSSet<NSNumber*>*)supportedPixelFormats { + return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), + @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + @(kCVPixelFormatType_32BGRA), + @(kCVPixelFormatType_32ARGB), + nil]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { + return [self initWithPixelBuffer:pixelBuffer + adaptedWidth:CVPixelBufferGetWidth(pixelBuffer) + adaptedHeight:CVPixelBufferGetHeight(pixelBuffer) + cropWidth:CVPixelBufferGetWidth(pixelBuffer) + cropHeight:CVPixelBufferGetHeight(pixelBuffer) + cropX:0 + cropY:0]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY { + if (self = [super init]) { + _width = adaptedWidth; + _height = adaptedHeight; + _pixelBuffer = pixelBuffer; + _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer); + _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer); + _cropWidth = cropWidth; + _cropHeight = cropHeight; + // Can only crop at even pixels. + _cropX = cropX & ~1; + _cropY = cropY & ~1; + CVBufferRetain(_pixelBuffer); + } + + return self; +} + +- (void)dealloc { + CVBufferRelease(_pixelBuffer); +} + +- (int)width { + return _width; +} + +- (int)height { + return _height; +} + +- (BOOL)requiresCropping { + return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight; +} + +- (BOOL)requiresScalingToWidth:(int)width height:(int)height { + return _cropWidth != width || _cropHeight != height; +} + +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height { + const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + switch (srcPixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + int srcChromaWidth = (_cropWidth + 1) / 2; + int srcChromaHeight = (_cropHeight + 1) / 2; + int dstChromaWidth = (width + 1) / 2; + int dstChromaHeight = (height + 1) / 2; + + return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + return 0; // Scaling RGBA frames does not require a temporary buffer. + } + } + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + return 0; +} + +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t*)tmpBuffer { + const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer); + + switch (srcPixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + if (dstWidth > 0 && dstHeight > 0) { + RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || + dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + if ([self requiresScalingToWidth:dstWidth height:dstHeight]) { + RTC_DCHECK(tmpBuffer); + } + [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer]; + } + break; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + RTC_DCHECK(srcPixelFormat == dstPixelFormat); + [self cropAndScaleARGBTo:outputPixelBuffer]; + break; + } + default: { + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + } + } + + return YES; +} +- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight { + return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] + initWithPixelBuffer:_pixelBuffer + adaptedWidth:scaleWidth + adaptedHeight:scaleHeight + cropWidth:cropWidth * _cropWidth / _width + cropHeight:cropHeight * _cropHeight / _height + cropX:_cropX + offsetX * _cropWidth / _width + cropY:_cropY + offsetY * _cropHeight / _height]; +} + +- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 { + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + + RTC_OBJC_TYPE(RTCMutableI420Buffer)* i420Buffer = + [[RTC_OBJC_TYPE(RTCMutableI420Buffer) alloc] initWithWidth:[self width] height:[self height]]; + + switch (pixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + const uint8_t* srcY = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + // TODO(magjed): Use a frame buffer pool. + webrtc::NV12ToI420Scaler nv12ToI420Scaler; + nv12ToI420Scaler.NV12ToI420Scale(srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + break; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + CVPixelBufferRef scaledPixelBuffer = NULL; + CVPixelBufferRef sourcePixelBuffer = NULL; + if ([self requiresCropping] || + [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) { + CVPixelBufferCreate( + NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer); + [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL]; + + CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + sourcePixelBuffer = scaledPixelBuffer; + } else { + sourcePixelBuffer = _pixelBuffer; + } + const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(sourcePixelBuffer)); + const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer); + + if (pixelFormat == kCVPixelFormatType_32BGRA) { + // Corresponds to libyuv::FOURCC_ARGB + libyuv::ARGBToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + // Corresponds to libyuv::FOURCC_BGRA + libyuv::BGRAToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + } + + if (scaledPixelBuffer) { + CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + CVBufferRelease(scaledPixelBuffer); + } + break; + } + default: { + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + } + } + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + + return i420Buffer; +} + +#pragma mark - Debugging + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +- (id)debugQuickLookObject { + CGImageRef cgImage; + VTCreateCGImageFromCVPixelBuffer(_pixelBuffer, NULL, &cgImage); + UIImage *image = [UIImage imageWithCGImage:cgImage scale:1.0 orientation:UIImageOrientationUp]; + CGImageRelease(cgImage); + return image; +} +#endif + +#pragma mark - Private + +- (void)cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { + // Prepare output pointers. + CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + } + const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + uint8_t* dstY = + reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); + const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); + uint8_t* dstUV = + reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1)); + const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); + + // Prepare source pointers. + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + webrtc::NV12Scale(tmpBuffer, + srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + dstY, + dstYStride, + dstUV, + dstUVStride, + dstWidth, + dstHeight); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); +} + +- (void)cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer { + // Prepare output pointers. + CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + } + const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + + uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer)); + const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer); + + // Prepare source pointers. + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(_pixelBuffer)); + const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer); + + // Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding + // to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly. + const int bytesPerPixel = 4; + src += srcStride * _cropY + (_cropX * bytesPerPixel); + + // kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB + libyuv::ARGBScale(src, + srcStride, + _cropWidth, + _cropHeight, + dst, + dstStride, + dstWidth, + dstHeight, + libyuv::kFilterBox); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); +} + +@end |