diff options
Diffstat (limited to 'third_party/libwebrtc/sdk/objc/components/renderer/metal')
13 files changed, 1432 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h new file mode 100644 index 0000000000..e5987fe22a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h @@ -0,0 +1,17 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLI420Renderer : RTCMTLRenderer +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm new file mode 100644 index 0000000000..f4c76fa313 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm @@ -0,0 +1,170 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLI420Renderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCI420Buffer.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" + +#import "RTCMTLRenderer+Private.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } Varyings; + + vertex Varyings vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + unsigned int vid[[vertex_id]]) { + Varyings out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + + return out; + } + + fragment half4 fragmentColorConversion( + Varyings in[[stage_in]], + texture2d<float, access::sample> textureY[[texture(0)]], + texture2d<float, access::sample> textureU[[texture(1)]], + texture2d<float, access::sample> textureV[[texture(2)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + float y; + float u; + float v; + float r; + float g; + float b; + // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php + y = textureY.sample(s, in.texcoord).r; + u = textureU.sample(s, in.texcoord).r; + v = textureV.sample(s, in.texcoord).r; + u = u - 0.5; + v = v - 0.5; + r = y + 1.403 * v; + g = y - 0.344 * u - 0.714 * v; + b = y + 1.770 * u; + + float4 out = float4(r, g, b, 1.0); + + return half4(out); + }); + +@implementation RTCMTLI420Renderer { + // Textures. + id<MTLTexture> _yTexture; + id<MTLTexture> _uTexture; + id<MTLTexture> _vTexture; + + MTLTextureDescriptor *_descriptor; + MTLTextureDescriptor *_chromaDescriptor; + + int _width; + int _height; + int _chromaWidth; + int _chromaHeight; +} + +#pragma mark - Virtual + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + *width = frame.width; + *height = frame.height; + *cropWidth = frame.width; + *cropHeight = frame.height; + *cropX = 0; + *cropY = 0; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (![super setupTexturesForFrame:frame]) { + return NO; + } + + id<MTLDevice> device = [self currentMetalDevice]; + if (!device) { + return NO; + } + + id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420]; + + // Luma (y) texture. + if (!_descriptor || _width != frame.width || _height != frame.height) { + _width = frame.width; + _height = frame.height; + _descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:_width + height:_height + mipmapped:NO]; + _descriptor.usage = MTLTextureUsageShaderRead; + _yTexture = [device newTextureWithDescriptor:_descriptor]; + } + + // Chroma (u,v) textures + [_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height) + mipmapLevel:0 + withBytes:buffer.dataY + bytesPerRow:buffer.strideY]; + + if (!_chromaDescriptor || _chromaWidth != frame.width / 2 || _chromaHeight != frame.height / 2) { + _chromaWidth = frame.width / 2; + _chromaHeight = frame.height / 2; + _chromaDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:_chromaWidth + height:_chromaHeight + mipmapped:NO]; + _chromaDescriptor.usage = MTLTextureUsageShaderRead; + _uTexture = [device newTextureWithDescriptor:_chromaDescriptor]; + _vTexture = [device newTextureWithDescriptor:_chromaDescriptor]; + } + + [_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) + mipmapLevel:0 + withBytes:buffer.dataU + bytesPerRow:buffer.strideU]; + [_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) + mipmapLevel:0 + withBytes:buffer.dataV + bytesPerRow:buffer.strideV]; + + return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil); +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_yTexture atIndex:0]; + [renderEncoder setFragmentTexture:_uTexture atIndex:1]; + [renderEncoder setFragmentTexture:_vTexture atIndex:2]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h new file mode 100644 index 0000000000..f70e2ad5ee --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AppKit/AppKit.h> + +#import "RTCVideoRenderer.h" + +NS_AVAILABLE_MAC(10.11) + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMTLNSVideoView) : NSView <RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + ++ (BOOL)isMetalAvailable; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m new file mode 100644 index 0000000000..625fb1caa7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m @@ -0,0 +1,122 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLNSVideoView.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCVideoFrame.h" + +#import "RTCMTLI420Renderer.h" + +@interface RTC_OBJC_TYPE (RTCMTLNSVideoView) +()<MTKViewDelegate> @property(nonatomic) id<RTCMTLRenderer> renderer; +@property(nonatomic, strong) MTKView *metalView; +@property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@end + +@implementation RTC_OBJC_TYPE (RTCMTLNSVideoView) { + id<RTCMTLRenderer> _renderer; +} + +@synthesize delegate = _delegate; +@synthesize renderer = _renderer; +@synthesize metalView = _metalView; +@synthesize videoFrame = _videoFrame; + +- (instancetype)initWithFrame:(CGRect)frameRect { + self = [super initWithFrame:frameRect]; + if (self) { + [self configure]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aCoder { + self = [super initWithCoder:aCoder]; + if (self) { + [self configure]; + } + return self; +} + +#pragma mark - Private + ++ (BOOL)isMetalAvailable { + return [MTLCopyAllDevices() count] > 0; +} + +- (void)configure { + if ([[self class] isMetalAvailable]) { + _metalView = [[MTKView alloc] initWithFrame:self.bounds]; + [self addSubview:_metalView]; + _metalView.layerContentsPlacement = NSViewLayerContentsPlacementScaleProportionallyToFit; + _metalView.translatesAutoresizingMaskIntoConstraints = NO; + _metalView.framebufferOnly = YES; + _metalView.delegate = self; + + _renderer = [[RTCMTLI420Renderer alloc] init]; + if (![(RTCMTLI420Renderer *)_renderer addRenderingDestination:_metalView]) { + _renderer = nil; + }; + } +} + +- (void)updateConstraints { + NSDictionary *views = NSDictionaryOfVariableBindings(_metalView); + + NSArray *constraintsHorizontal = + [NSLayoutConstraint constraintsWithVisualFormat:@"H:|-0-[_metalView]-0-|" + options:0 + metrics:nil + views:views]; + [self addConstraints:constraintsHorizontal]; + + NSArray *constraintsVertical = + [NSLayoutConstraint constraintsWithVisualFormat:@"V:|-0-[_metalView]-0-|" + options:0 + metrics:nil + views:views]; + [self addConstraints:constraintsVertical]; + [super updateConstraints]; +} + +#pragma mark - MTKViewDelegate methods +- (void)drawInMTKView:(nonnull MTKView *)view { + if (self.videoFrame == nil) { + return; + } + if (view == self.metalView) { + [_renderer drawFrame:self.videoFrame]; + } +} + +- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +- (void)setSize:(CGSize)size { + _metalView.drawableSize = size; + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate videoView:self didChangeVideoSize:size]; + }); + [_metalView draw]; +} + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (frame == nil) { + return; + } + self.videoFrame = [frame newI420VideoFrame]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h new file mode 100644 index 0000000000..866b7ea17e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h @@ -0,0 +1,18 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLNV12Renderer : RTCMTLRenderer + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm new file mode 100644 index 0000000000..7b037c6dbc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm @@ -0,0 +1,164 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLNV12Renderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "RTCMTLRenderer+Private.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#include "rtc_base/checks.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } Varyings; + + vertex Varyings vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + unsigned int vid[[vertex_id]]) { + Varyings out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + return out; + } + + // Receiving YCrCb textures. + fragment half4 fragmentColorConversion( + Varyings in[[stage_in]], + texture2d<float, access::sample> textureY[[texture(0)]], + texture2d<float, access::sample> textureCbCr[[texture(1)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + float y; + float2 uv; + y = textureY.sample(s, in.texcoord).r; + uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5); + + // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php + float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0); + + return half4(out); + }); + +@implementation RTCMTLNV12Renderer { + // Textures. + CVMetalTextureCacheRef _textureCache; + id<MTLTexture> _yTexture; + id<MTLTexture> _CrCbTexture; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + if ([super addRenderingDestination:view]) { + return [self initializeTextureCache]; + } + return NO; +} + +- (BOOL)initializeTextureCache { + CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice], + nil, &_textureCache); + if (status != kCVReturnSuccess) { + RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); + return NO; + } + + return YES; +} + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + *width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer); + *height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer); + *cropWidth = pixelBuffer.cropWidth; + *cropHeight = pixelBuffer.cropHeight; + *cropX = pixelBuffer.cropX; + *cropY = pixelBuffer.cropY; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + if (![super setupTexturesForFrame:frame]) { + return NO; + } + CVPixelBufferRef pixelBuffer = ((RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer).pixelBuffer; + + id<MTLTexture> lumaTexture = nil; + id<MTLTexture> chromaTexture = nil; + CVMetalTextureRef outTexture = nullptr; + + // Luma (y) texture. + int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); + int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); + + int indexPlane = 0; + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth, + lumaHeight, indexPlane, &outTexture); + + if (result == kCVReturnSuccess) { + lumaTexture = CVMetalTextureGetTexture(outTexture); + } + + // Same as CFRelease except it can be passed NULL without crashing. + CVBufferRelease(outTexture); + outTexture = nullptr; + + // Chroma (CrCb) texture. + indexPlane = 1; + result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, lumaWidth / 2, + lumaHeight / 2, indexPlane, &outTexture); + if (result == kCVReturnSuccess) { + chromaTexture = CVMetalTextureGetTexture(outTexture); + } + CVBufferRelease(outTexture); + + if (lumaTexture != nil && chromaTexture != nil) { + _yTexture = lumaTexture; + _CrCbTexture = chromaTexture; + return YES; + } + return NO; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_yTexture atIndex:0]; + [renderEncoder setFragmentTexture:_CrCbTexture atIndex:1]; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + } +} +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h new file mode 100644 index 0000000000..9db422cd22 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +/** @abstract RGB/BGR renderer. + * @discussion This renderer handles both kCVPixelFormatType_32BGRA and + * kCVPixelFormatType_32ARGB. + */ +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLRGBRenderer : RTCMTLRenderer + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm new file mode 100644 index 0000000000..e5dc4ef80a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm @@ -0,0 +1,164 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLRGBRenderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "RTCMTLRenderer+Private.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#include "rtc_base/checks.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } VertexIO; + + vertex VertexIO vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + uint vid[[vertex_id]]) { + VertexIO out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + return out; + } + + fragment half4 fragmentColorConversion(VertexIO in[[stage_in]], + texture2d<half, access::sample> texture[[texture(0)]], + constant bool &isARGB[[buffer(0)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + + half4 out = texture.sample(s, in.texcoord); + if (isARGB) { + out = half4(out.g, out.b, out.a, out.r); + } + + return out; + }); + +@implementation RTCMTLRGBRenderer { + // Textures. + CVMetalTextureCacheRef _textureCache; + id<MTLTexture> _texture; + + // Uniforms. + id<MTLBuffer> _uniformsBuffer; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + if ([super addRenderingDestination:view]) { + return [self initializeTextureCache]; + } + return NO; +} + +- (BOOL)initializeTextureCache { + CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice], + nil, &_textureCache); + if (status != kCVReturnSuccess) { + RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); + return NO; + } + + return YES; +} + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + *width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer); + *height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer); + *cropWidth = pixelBuffer.cropWidth; + *cropHeight = pixelBuffer.cropHeight; + *cropX = pixelBuffer.cropX; + *cropY = pixelBuffer.cropY; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + if (![super setupTexturesForFrame:frame]) { + return NO; + } + CVPixelBufferRef pixelBuffer = ((RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer).pixelBuffer; + + id<MTLTexture> gpuTexture = nil; + CVMetalTextureRef textureOut = nullptr; + bool isARGB; + + int width = CVPixelBufferGetWidth(pixelBuffer); + int height = CVPixelBufferGetHeight(pixelBuffer); + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + + MTLPixelFormat mtlPixelFormat; + if (pixelFormat == kCVPixelFormatType_32BGRA) { + mtlPixelFormat = MTLPixelFormatBGRA8Unorm; + isARGB = false; + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + mtlPixelFormat = MTLPixelFormatRGBA8Unorm; + isARGB = true; + } else { + RTC_DCHECK_NOTREACHED(); + return NO; + } + + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, mtlPixelFormat, + width, height, 0, &textureOut); + if (result == kCVReturnSuccess) { + gpuTexture = CVMetalTextureGetTexture(textureOut); + } + CVBufferRelease(textureOut); + + if (gpuTexture != nil) { + _texture = gpuTexture; + _uniformsBuffer = + [[self currentMetalDevice] newBufferWithBytes:&isARGB + length:sizeof(isARGB) + options:MTLResourceCPUCacheModeDefaultCache]; + return YES; + } + + return NO; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_texture atIndex:0]; + [renderEncoder setFragmentBuffer:_uniformsBuffer offset:0 atIndex:0]; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h new file mode 100644 index 0000000000..916d4d4430 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Metal/Metal.h> + +#import "RTCMTLRenderer.h" + +#define MTL_STRINGIFY(s) @ #s + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCMTLRenderer (Private) +- (nullable id<MTLDevice>)currentMetalDevice; +- (NSString *)shaderSource; +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder; +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h new file mode 100644 index 0000000000..aa31545973 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h @@ -0,0 +1,61 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#if TARGET_OS_IPHONE +#import <UIKit/UIKit.h> +#else +#import <AppKit/AppKit.h> +#endif + +#import "base/RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN +/** + * Protocol defining ability to render RTCVideoFrame in Metal enabled views. + */ +@protocol RTCMTLRenderer <NSObject> + +/** + * Method to be implemented to perform actual rendering of the provided frame. + * + * @param frame The frame to be rendered. + */ +- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +/** + * Sets the provided view as rendering destination if possible. + * + * If not possible method returns NO and callers of the method are responisble for performing + * cleanups. + */ + +#if TARGET_OS_IOS +- (BOOL)addRenderingDestination:(__kindof UIView *)view; +#else +- (BOOL)addRenderingDestination:(__kindof NSView *)view; +#endif + +@end + +/** + * Implementation of RTCMTLRenderer protocol. + */ +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLRenderer : NSObject <RTCMTLRenderer> + +/** @abstract A wrapped RTCVideoRotation, or nil. + @discussion When not nil, the rotation of the actual frame is ignored when rendering. + */ +@property(atomic, nullable) NSValue *rotationOverride; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm new file mode 100644 index 0000000000..410590a7b1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm @@ -0,0 +1,328 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLRenderer+Private.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" + +#include "api/video/video_rotation.h" +#include "rtc_base/checks.h" + +// As defined in shaderSource. +static NSString *const vertexFunctionName = @"vertexPassthrough"; +static NSString *const fragmentFunctionName = @"fragmentColorConversion"; + +static NSString *const pipelineDescriptorLabel = @"RTCPipeline"; +static NSString *const commandBufferLabel = @"RTCCommandBuffer"; +static NSString *const renderEncoderLabel = @"RTCEncoder"; +static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame"; + +// Computes the texture coordinates given rotation and cropping. +static inline void getCubeVertexData(int cropX, + int cropY, + int cropWidth, + int cropHeight, + size_t frameWidth, + size_t frameHeight, + RTCVideoRotation rotation, + float *buffer) { + // The computed values are the adjusted texture coordinates, in [0..1]. + // For the left and top, 0.0 means no cropping and e.g. 0.2 means we're skipping 20% of the + // left/top edge. + // For the right and bottom, 1.0 means no cropping and e.g. 0.8 means we're skipping 20% of the + // right/bottom edge (i.e. render up to 80% of the width/height). + float cropLeft = cropX / (float)frameWidth; + float cropRight = (cropX + cropWidth) / (float)frameWidth; + float cropTop = cropY / (float)frameHeight; + float cropBottom = (cropY + cropHeight) / (float)frameHeight; + + // These arrays map the view coordinates to texture coordinates, taking cropping and rotation + // into account. The first two columns are view coordinates, the last two are texture coordinates. + switch (rotation) { + case RTCVideoRotation_0: { + float values[16] = {-1.0, -1.0, cropLeft, cropBottom, + 1.0, -1.0, cropRight, cropBottom, + -1.0, 1.0, cropLeft, cropTop, + 1.0, 1.0, cropRight, cropTop}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_90: { + float values[16] = {-1.0, -1.0, cropRight, cropBottom, + 1.0, -1.0, cropRight, cropTop, + -1.0, 1.0, cropLeft, cropBottom, + 1.0, 1.0, cropLeft, cropTop}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_180: { + float values[16] = {-1.0, -1.0, cropRight, cropTop, + 1.0, -1.0, cropLeft, cropTop, + -1.0, 1.0, cropRight, cropBottom, + 1.0, 1.0, cropLeft, cropBottom}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_270: { + float values[16] = {-1.0, -1.0, cropLeft, cropTop, + 1.0, -1.0, cropLeft, cropBottom, + -1.0, 1.0, cropRight, cropTop, + 1.0, 1.0, cropRight, cropBottom}; + memcpy(buffer, &values, sizeof(values)); + } break; + } +} + +// The max number of command buffers in flight (submitted to GPU). +// For now setting it up to 1. +// In future we might use triple buffering method if it improves performance. +static const NSInteger kMaxInflightBuffers = 1; + +@implementation RTCMTLRenderer { + __kindof MTKView *_view; + + // Controller. + dispatch_semaphore_t _inflight_semaphore; + + // Renderer. + id<MTLDevice> _device; + id<MTLCommandQueue> _commandQueue; + id<MTLLibrary> _defaultLibrary; + id<MTLRenderPipelineState> _pipelineState; + + // Buffers. + id<MTLBuffer> _vertexBuffer; + + // Values affecting the vertex buffer. Stored for comparison to avoid unnecessary recreation. + int _oldFrameWidth; + int _oldFrameHeight; + int _oldCropWidth; + int _oldCropHeight; + int _oldCropX; + int _oldCropY; + RTCVideoRotation _oldRotation; +} + +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)init { + if (self = [super init]) { + _inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers); + } + + return self; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + return [self setupWithView:view]; +} + +#pragma mark - Private + +- (BOOL)setupWithView:(__kindof MTKView *)view { + BOOL success = NO; + if ([self setupMetal]) { + _view = view; + view.device = _device; + view.preferredFramesPerSecond = 30; + view.autoResizeDrawable = NO; + + [self loadAssets]; + + float vertexBufferArray[16] = {0}; + _vertexBuffer = [_device newBufferWithBytes:vertexBufferArray + length:sizeof(vertexBufferArray) + options:MTLResourceCPUCacheModeWriteCombined]; + success = YES; + } + return success; +} +#pragma mark - Inheritance + +- (id<MTLDevice>)currentMetalDevice { + return _device; +} + +- (NSString *)shaderSource { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; + return nil; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; +} + +- (void)getWidth:(int *)width + height:(int *)height + cropWidth:(int *)cropWidth + cropHeight:(int *)cropHeight + cropX:(int *)cropX + cropY:(int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + // Apply rotation override if set. + RTCVideoRotation rotation; + NSValue *rotationOverride = self.rotationOverride; + if (rotationOverride) { +#if defined(__IPHONE_11_0) && defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && \ + (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_11_0) + if (@available(iOS 11, *)) { + [rotationOverride getValue:&rotation size:sizeof(rotation)]; + } else +#endif + { + [rotationOverride getValue:&rotation]; + } + } else { + rotation = frame.rotation; + } + + int frameWidth, frameHeight, cropWidth, cropHeight, cropX, cropY; + [self getWidth:&frameWidth + height:&frameHeight + cropWidth:&cropWidth + cropHeight:&cropHeight + cropX:&cropX + cropY:&cropY + ofFrame:frame]; + + // Recompute the texture cropping and recreate vertexBuffer if necessary. + if (cropX != _oldCropX || cropY != _oldCropY || cropWidth != _oldCropWidth || + cropHeight != _oldCropHeight || rotation != _oldRotation || frameWidth != _oldFrameWidth || + frameHeight != _oldFrameHeight) { + getCubeVertexData(cropX, + cropY, + cropWidth, + cropHeight, + frameWidth, + frameHeight, + rotation, + (float *)_vertexBuffer.contents); + _oldCropX = cropX; + _oldCropY = cropY; + _oldCropWidth = cropWidth; + _oldCropHeight = cropHeight; + _oldRotation = rotation; + _oldFrameWidth = frameWidth; + _oldFrameHeight = frameHeight; + } + + return YES; +} + +#pragma mark - GPU methods + +- (BOOL)setupMetal { + // Set the view to use the default device. + _device = MTLCreateSystemDefaultDevice(); + if (!_device) { + return NO; + } + + // Create a new command queue. + _commandQueue = [_device newCommandQueue]; + + // Load metal library from source. + NSError *libraryError = nil; + NSString *shaderSource = [self shaderSource]; + + id<MTLLibrary> sourceLibrary = + [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError]; + + if (libraryError) { + RTCLogError(@"Metal: Library with source failed\n%@", libraryError); + return NO; + } + + if (!sourceLibrary) { + RTCLogError(@"Metal: Failed to load library. %@", libraryError); + return NO; + } + _defaultLibrary = sourceLibrary; + + return YES; +} + +- (void)loadAssets { + id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName]; + id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName]; + + MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init]; + pipelineDescriptor.label = pipelineDescriptorLabel; + pipelineDescriptor.vertexFunction = vertexFunction; + pipelineDescriptor.fragmentFunction = fragmentFunction; + pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat; + pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid; + NSError *error = nil; + _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error]; + + if (!_pipelineState) { + RTCLogError(@"Metal: Failed to create pipeline state. %@", error); + } +} + +- (void)render { + id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer]; + commandBuffer.label = commandBufferLabel; + + __block dispatch_semaphore_t block_semaphore = _inflight_semaphore; + [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) { + // GPU work completed. + dispatch_semaphore_signal(block_semaphore); + }]; + + MTLRenderPassDescriptor *renderPassDescriptor = _view.currentRenderPassDescriptor; + if (renderPassDescriptor) { // Valid drawable. + id<MTLRenderCommandEncoder> renderEncoder = + [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor]; + renderEncoder.label = renderEncoderLabel; + + // Set context state. + [renderEncoder pushDebugGroup:renderEncoderDebugGroup]; + [renderEncoder setRenderPipelineState:_pipelineState]; + [renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0]; + [self uploadTexturesToRenderEncoder:renderEncoder]; + + [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip + vertexStart:0 + vertexCount:4 + instanceCount:1]; + [renderEncoder popDebugGroup]; + [renderEncoder endEncoding]; + + [commandBuffer presentDrawable:_view.currentDrawable]; + } + + // CPU work is completed, GPU work can be started. + [commandBuffer commit]; +} + +#pragma mark - RTCMTLRenderer + +- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + @autoreleasepool { + // Wait until the inflight (curently sent to GPU) command buffer + // has completed the GPU work. + dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER); + + if ([self setupTexturesForFrame:frame]) { + [self render]; + } else { + dispatch_semaphore_signal(_inflight_semaphore); + } + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h new file mode 100644 index 0000000000..3320d12076 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h @@ -0,0 +1,44 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrame.h" +#import "RTCVideoRenderer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * RTCMTLVideoView is thin wrapper around MTKView. + * + * It has id<RTCVideoRenderer> property that renders video frames in the view's + * bounds using Metal. + */ +NS_CLASS_AVAILABLE_IOS(9) + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMTLVideoView) : UIView<RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + +@property(nonatomic) UIViewContentMode videoContentMode; + +/** @abstract Enables/disables rendering. + */ +@property(nonatomic, getter=isEnabled) BOOL enabled; + +/** @abstract Wrapped RTCVideoRotation, or nil. + */ +@property(nonatomic, nullable) NSValue* rotationOverride; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m new file mode 100644 index 0000000000..c5d9e4385f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m @@ -0,0 +1,265 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLVideoView.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#import "RTCMTLI420Renderer.h" +#import "RTCMTLNV12Renderer.h" +#import "RTCMTLRGBRenderer.h" + +// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime. +// Linking errors occur when compiling for architectures that don't support Metal. +#define MTKViewClass NSClassFromString(@"MTKView") +#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer") +#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer") +#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer") + +@interface RTC_OBJC_TYPE (RTCMTLVideoView) +()<MTKViewDelegate> @property(nonatomic) RTCMTLI420Renderer *rendererI420; +@property(nonatomic) RTCMTLNV12Renderer *rendererNV12; +@property(nonatomic) RTCMTLRGBRenderer *rendererRGB; +@property(nonatomic) MTKView *metalView; +@property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@property(nonatomic) CGSize videoFrameSize; +@property(nonatomic) int64_t lastFrameTimeNs; +@end + +@implementation RTC_OBJC_TYPE (RTCMTLVideoView) + +@synthesize delegate = _delegate; +@synthesize rendererI420 = _rendererI420; +@synthesize rendererNV12 = _rendererNV12; +@synthesize rendererRGB = _rendererRGB; +@synthesize metalView = _metalView; +@synthesize videoFrame = _videoFrame; +@synthesize videoFrameSize = _videoFrameSize; +@synthesize lastFrameTimeNs = _lastFrameTimeNs; +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)initWithFrame:(CGRect)frameRect { + self = [super initWithFrame:frameRect]; + if (self) { + [self configure]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aCoder { + self = [super initWithCoder:aCoder]; + if (self) { + [self configure]; + } + return self; +} + +- (BOOL)isEnabled { + return !self.metalView.paused; +} + +- (void)setEnabled:(BOOL)enabled { + self.metalView.paused = !enabled; +} + +- (UIViewContentMode)videoContentMode { + return self.metalView.contentMode; +} + +- (void)setVideoContentMode:(UIViewContentMode)mode { + self.metalView.contentMode = mode; +} + +#pragma mark - Private + ++ (BOOL)isMetalAvailable { + return MTLCreateSystemDefaultDevice() != nil; +} + ++ (MTKView *)createMetalView:(CGRect)frame { + return [[MTKViewClass alloc] initWithFrame:frame]; +} + ++ (RTCMTLNV12Renderer *)createNV12Renderer { + return [[RTCMTLNV12RendererClass alloc] init]; +} + ++ (RTCMTLI420Renderer *)createI420Renderer { + return [[RTCMTLI420RendererClass alloc] init]; +} + ++ (RTCMTLRGBRenderer *)createRGBRenderer { + return [[RTCMTLRGBRenderer alloc] init]; +} + +- (void)configure { + NSAssert([RTC_OBJC_TYPE(RTCMTLVideoView) isMetalAvailable], + @"Metal not availiable on this device"); + + self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds]; + self.metalView.delegate = self; + self.metalView.contentMode = UIViewContentModeScaleAspectFill; + [self addSubview:self.metalView]; + self.videoFrameSize = CGSizeZero; +} + +- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { + [super setMultipleTouchEnabled:multipleTouchEnabled]; + self.metalView.multipleTouchEnabled = multipleTouchEnabled; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + + CGRect bounds = self.bounds; + self.metalView.frame = bounds; + if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) { + self.metalView.drawableSize = [self drawableSize]; + } else { + self.metalView.drawableSize = bounds.size; + } +} + +#pragma mark - MTKViewDelegate methods + +- (void)drawInMTKView:(nonnull MTKView *)view { + NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance."); + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = self.videoFrame; + // Skip rendering if we've already rendered this frame. + if (!videoFrame || videoFrame.width <= 0 || videoFrame.height <= 0 || + videoFrame.timeStampNs == self.lastFrameTimeNs) { + return; + } + + if (CGRectIsEmpty(view.bounds)) { + return; + } + + RTCMTLRenderer *renderer; + if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer; + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); + if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) { + if (!self.rendererRGB) { + self.rendererRGB = [RTC_OBJC_TYPE(RTCMTLVideoView) createRGBRenderer]; + if (![self.rendererRGB addRenderingDestination:self.metalView]) { + self.rendererRGB = nil; + RTCLogError(@"Failed to create RGB renderer"); + return; + } + } + renderer = self.rendererRGB; + } else { + if (!self.rendererNV12) { + self.rendererNV12 = [RTC_OBJC_TYPE(RTCMTLVideoView) createNV12Renderer]; + if (![self.rendererNV12 addRenderingDestination:self.metalView]) { + self.rendererNV12 = nil; + RTCLogError(@"Failed to create NV12 renderer"); + return; + } + } + renderer = self.rendererNV12; + } + } else { + if (!self.rendererI420) { + self.rendererI420 = [RTC_OBJC_TYPE(RTCMTLVideoView) createI420Renderer]; + if (![self.rendererI420 addRenderingDestination:self.metalView]) { + self.rendererI420 = nil; + RTCLogError(@"Failed to create I420 renderer"); + return; + } + } + renderer = self.rendererI420; + } + + renderer.rotationOverride = self.rotationOverride; + + [renderer drawFrame:videoFrame]; + self.lastFrameTimeNs = videoFrame.timeStampNs; +} + +- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { +} + +#pragma mark - + +- (void)setRotationOverride:(NSValue *)rotationOverride { + _rotationOverride = rotationOverride; + + self.metalView.drawableSize = [self drawableSize]; + [self setNeedsLayout]; +} + +- (RTCVideoRotation)frameRotation { + if (self.rotationOverride) { + RTCVideoRotation rotation; + if (@available(iOS 11, *)) { + [self.rotationOverride getValue:&rotation size:sizeof(rotation)]; + } else { + [self.rotationOverride getValue:&rotation]; + } + return rotation; + } + + return self.videoFrame.rotation; +} + +- (CGSize)drawableSize { + // Flip width/height if the rotations are not the same. + CGSize videoFrameSize = self.videoFrameSize; + RTCVideoRotation frameRotation = [self frameRotation]; + + BOOL useLandscape = + (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180); + BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) || + (self.videoFrame.rotation == RTCVideoRotation_180); + + if (useLandscape == sizeIsLandscape) { + return videoFrameSize; + } else { + return CGSizeMake(videoFrameSize.height, videoFrameSize.width); + } +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +- (void)setSize:(CGSize)size { + __weak RTC_OBJC_TYPE(RTCMTLVideoView) *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCMTLVideoView) *strongSelf = weakSelf; + + strongSelf.videoFrameSize = size; + CGSize drawableSize = [strongSelf drawableSize]; + + strongSelf.metalView.drawableSize = drawableSize; + [strongSelf setNeedsLayout]; + [strongSelf.delegate videoView:self didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (!self.isEnabled) { + return; + } + + if (frame == nil) { + RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); + return; + } + self.videoFrame = frame; +} + +@end |