/* * Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #import #import #import "components/video_frame_buffer/RTCCVPixelBuffer.h" #import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h" #import "base/RTCVideoFrame.h" #import "base/RTCVideoFrameBuffer.h" #import "frame_buffer_helpers.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "third_party/libyuv/include/libyuv.h" namespace { struct ToI420WithCropAndScaleSetting { int inputWidth; int inputHeight; int offsetX; int offsetY; int cropWidth; int cropHeight; int scaleWidth; int scaleHeight; }; constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = { ToI420WithCropAndScaleSetting{ .inputWidth = 640, .inputHeight = 360, .offsetX = 0, .offsetY = 0, .cropWidth = 640, .cropHeight = 360, .scaleWidth = 320, .scaleHeight = 180, }, ToI420WithCropAndScaleSetting{ .inputWidth = 640, .inputHeight = 360, .offsetX = 160, .offsetY = 90, .cropWidth = 160, .cropHeight = 90, .scaleWidth = 320, .scaleHeight = 180, }, }; } // namespace @interface RTCCVPixelBufferTests : XCTestCase @end @implementation RTCCVPixelBufferTests { } - (void)testRequiresCroppingNoCrop { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertFalse([buffer requiresCropping]); CVBufferRelease(pixelBufferRef); } - (void)testRequiresCroppingWithCrop { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *croppedBuffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:720 adaptedHeight:1280 cropWidth:360 cropHeight:640 cropX:100 cropY:100]; XCTAssertTrue([croppedBuffer requiresCropping]); CVBufferRelease(pixelBufferRef); } - (void)testRequiresScalingNoScale { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertFalse([buffer requiresScalingToWidth:720 height:1280]); CVBufferRelease(pixelBufferRef); } - (void)testRequiresScalingWithScale { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertTrue([buffer requiresScalingToWidth:360 height:640]); CVBufferRelease(pixelBufferRef); } - (void)testRequiresScalingWithScaleAndMatchingCrop { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:720 adaptedHeight:1280 cropWidth:360 cropHeight:640 cropX:100 cropY:100]; XCTAssertFalse([buffer requiresScalingToWidth:360 height:640]); CVBufferRelease(pixelBufferRef); } - (void)testBufferSize_NV12 { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 576000); CVBufferRelease(pixelBufferRef); } - (void)testBufferSize_RGB { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate(NULL, 720, 1280, kCVPixelFormatType_32BGRA, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 0); CVBufferRelease(pixelBufferRef); } - (void)testCropAndScale_NV12 { [self cropAndScaleTestWithNV12]; } - (void)testCropAndScaleNoOp_NV12 { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputSize:CGSizeMake(720, 1280)]; } - (void)testCropAndScale_NV12FullToVideo { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; } - (void)testCropAndScaleZeroSizeFrame_NV12 { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputSize:CGSizeMake(0, 0)]; } - (void)testCropAndScaleToSmallFormat_NV12 { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputSize:CGSizeMake(148, 320)]; } - (void)testCropAndScaleToOddFormat_NV12 { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputSize:CGSizeMake(361, 640)]; } - (void)testCropAndScale_32BGRA { [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32BGRA]; } - (void)testCropAndScale_32ARGB { [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB]; } - (void)testCropAndScaleWithSmallCropInfo_32ARGB { [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:2 cropY:3]; } - (void)testCropAndScaleWithLargeCropInfo_32ARGB { [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:200 cropY:300]; } - (void)testToI420_NV12 { [self toI420WithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; } - (void)testToI420_32BGRA { [self toI420WithPixelFormat:kCVPixelFormatType_32BGRA]; } - (void)testToI420_32ARGB { [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB]; } - (void)testToI420WithCropAndScale_NV12 { for (const auto &setting : kToI420WithCropAndScaleSettings) { [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange setting:setting]; } } - (void)testToI420WithCropAndScale_32BGRA { for (const auto &setting : kToI420WithCropAndScaleSettings) { [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting]; } } - (void)testToI420WithCropAndScale_32ARGB { for (const auto &setting : kToI420WithCropAndScaleSettings) { [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting]; } } - (void)testScaleBufferTest { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); rtc::scoped_refptr i420Buffer = CreateI420Gradient(1920, 1080); CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertEqual(buffer.width, 1920); XCTAssertEqual(buffer.height, 1080); XCTAssertEqual(buffer.cropX, 0); XCTAssertEqual(buffer.cropY, 0); XCTAssertEqual(buffer.cropWidth, 1920); XCTAssertEqual(buffer.cropHeight, 1080); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320 offsetY:180 cropWidth:1280 cropHeight:720 scaleWidth:960 scaleHeight:540]; XCTAssertEqual(buffer2.width, 960); XCTAssertEqual(buffer2.height, 540); XCTAssertEqual(buffer2.cropX, 320); XCTAssertEqual(buffer2.cropY, 180); XCTAssertEqual(buffer2.cropWidth, 1280); XCTAssertEqual(buffer2.cropHeight, 720); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240 offsetY:135 cropWidth:480 cropHeight:270 scaleWidth:320 scaleHeight:180]; XCTAssertEqual(buffer3.width, 320); XCTAssertEqual(buffer3.height, 180); XCTAssertEqual(buffer3.cropX, 640); XCTAssertEqual(buffer3.cropY, 360); XCTAssertEqual(buffer3.cropWidth, 640); XCTAssertEqual(buffer3.cropHeight, 360); CVBufferRelease(pixelBufferRef); } #pragma mark - Shared test code - (void)cropAndScaleTestWithNV12 { [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; } - (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat { [self cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat outputSize:CGSizeMake(360, 640)]; } - (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat outputSize:(CGSize)outputSize { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate(NULL, 720, 1280, inputFormat, NULL, &pixelBufferRef); rtc::scoped_refptr i420Buffer = CreateI420Gradient(720, 1280); CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; XCTAssertEqual(buffer.width, 720); XCTAssertEqual(buffer.height, 1280); CVPixelBufferRef outputPixelBufferRef = NULL; CVPixelBufferCreate( NULL, outputSize.width, outputSize.height, outputFormat, NULL, &outputPixelBufferRef); std::vector frameScaleBuffer; if ([buffer requiresScalingToWidth:outputSize.width height:outputSize.height]) { int size = [buffer bufferSizeForCroppingAndScalingToWidth:outputSize.width height:outputSize.height]; frameScaleBuffer.resize(size); } else { frameScaleBuffer.clear(); } frameScaleBuffer.shrink_to_fit(); [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:frameScaleBuffer.data()]; RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef]; XCTAssertEqual(scaledBuffer.width, outputSize.width); XCTAssertEqual(scaledBuffer.height, outputSize.height); if (outputSize.width > 0 && outputSize.height > 0) { RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420]; RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420]; double psnr = I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); XCTAssertEqual(psnr, webrtc::kPerfectPSNR); } CVBufferRelease(pixelBufferRef); } - (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat { [self cropAndScaleTestWithRGBPixelFormat:pixelFormat cropX:0 cropY:0]; } - (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat cropX:(int)cropX cropY:(int)cropY { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate(NULL, 720, 1280, pixelFormat, NULL, &pixelBufferRef); DrawGradientInRGBPixelBuffer(pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:CVPixelBufferGetWidth(pixelBufferRef) adaptedHeight:CVPixelBufferGetHeight(pixelBufferRef) cropWidth:CVPixelBufferGetWidth(pixelBufferRef) - cropX cropHeight:CVPixelBufferGetHeight(pixelBufferRef) - cropY cropX:cropX cropY:cropY]; XCTAssertEqual(buffer.width, 720); XCTAssertEqual(buffer.height, 1280); CVPixelBufferRef outputPixelBufferRef = NULL; CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &outputPixelBufferRef); [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:NULL]; RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef]; XCTAssertEqual(scaledBuffer.width, 360); XCTAssertEqual(scaledBuffer.height, 640); RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420]; RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420]; double psnr = I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); XCTAssertEqual(psnr, webrtc::kPerfectPSNR); CVBufferRelease(pixelBufferRef); } - (void)toI420WithPixelFormat:(OSType)pixelFormat { rtc::scoped_refptr i420Buffer = CreateI420Gradient(360, 640); CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &pixelBufferRef); CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [buffer toI420]; double psnr = I420PSNR(*i420Buffer, *[fromCVPixelBuffer nativeI420Buffer]); double target = webrtc::kPerfectPSNR; if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { // libyuv's I420ToRGB functions seem to lose some quality. target = 19.0; } XCTAssertGreaterThanOrEqual(psnr, target); CVBufferRelease(pixelBufferRef); } - (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat setting:(const ToI420WithCropAndScaleSetting &)setting { rtc::scoped_refptr i420Buffer = CreateI420Gradient(setting.inputWidth, setting.inputHeight); CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef); CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer = [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer = (RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX offsetY:setting.offsetY cropWidth:setting.cropWidth cropHeight:setting.cropHeight scaleWidth:setting.scaleWidth scaleHeight:setting.scaleHeight]; RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; id scaledBuffer = [buffer cropAndScaleWith:setting.offsetX offsetY:setting.offsetY cropWidth:setting.cropWidth cropHeight:setting.cropHeight scaleWidth:setting.scaleWidth scaleHeight:setting.scaleHeight]; XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420]; double psnr = I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]); double target = webrtc::kPerfectPSNR; if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { // libyuv's I420ToRGB functions seem to lose some quality. target = 19.0; } XCTAssertGreaterThanOrEqual(psnr, target); CVBufferRelease(pixelBufferRef); } @end