diff options
Diffstat (limited to 'third_party/libwebrtc/sdk/objc/components/renderer/opengl')
16 files changed, 1508 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h new file mode 100644 index 0000000000..71a073ab21 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Default RTCVideoViewShading that will be used in RTCNSGLVideoView + * and RTCEAGLVideoView if no external shader is specified. This shader will render + * the video in a rectangle without any color or geometric transformations. + */ +@interface RTCDefaultShader : NSObject <RTC_OBJC_TYPE (RTCVideoViewShading)> + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm new file mode 100644 index 0000000000..51dca3223d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm @@ -0,0 +1,207 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDefaultShader.h" + +#if TARGET_OS_IPHONE +#import <OpenGLES/ES3/gl.h> +#else +#import <OpenGL/gl3.h> +#endif + +#import "RTCOpenGLDefines.h" +#import "RTCShader.h" +#import "base/RTCLogging.h" + +#include "absl/types/optional.h" + +static const int kYTextureUnit = 0; +static const int kUTextureUnit = 1; +static const int kVTextureUnit = 2; +static const int kUvTextureUnit = 1; + +// Fragment shader converts YUV values from input textures into a final RGB +// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. +static const char kI420FragmentShaderSource[] = + SHADER_VERSION + "precision highp float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureU;\n" + "uniform lowp sampler2D s_textureV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " float y, u, v, r, g, b;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" + " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" + " u = u - 0.5;\n" + " v = v - 0.5;\n" + " r = y + 1.403 * v;\n" + " g = y - 0.344 * u - 0.714 * v;\n" + " b = y + 1.770 * u;\n" + " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" + " }\n"; + +static const char kNV12FragmentShaderSource[] = + SHADER_VERSION + "precision mediump float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureUV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " mediump float y;\n" + " mediump vec2 uv;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " uv = " FRAGMENT_SHADER_TEXTURE "(s_textureUV, v_texcoord).ra -\n" + " vec2(0.5, 0.5);\n" + " " FRAGMENT_SHADER_COLOR " = vec4(y + 1.403 * uv.y,\n" + " y - 0.344 * uv.x - 0.714 * uv.y,\n" + " y + 1.770 * uv.x,\n" + " 1.0);\n" + " }\n"; + +@implementation RTCDefaultShader { + GLuint _vertexBuffer; + GLuint _vertexArray; + // Store current rotation and only upload new vertex data when rotation changes. + absl::optional<RTCVideoRotation> _currentRotation; + + GLuint _i420Program; + GLuint _nv12Program; +} + +- (void)dealloc { + glDeleteProgram(_i420Program); + glDeleteProgram(_nv12Program); + glDeleteBuffers(1, &_vertexBuffer); + glDeleteVertexArrays(1, &_vertexArray); +} + +- (BOOL)createAndSetupI420Program { + NSAssert(!_i420Program, @"I420 program already created"); + _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource); + if (!_i420Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_i420Program, "s_textureY"); + GLint uSampler = glGetUniformLocation(_i420Program, "s_textureU"); + GLint vSampler = glGetUniformLocation(_i420Program, "s_textureV"); + + if (ySampler < 0 || uSampler < 0 || vSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in I420 shader"); + glDeleteProgram(_i420Program); + _i420Program = 0; + return NO; + } + + glUseProgram(_i420Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uSampler, kUTextureUnit); + glUniform1i(vSampler, kVTextureUnit); + + return YES; +} + +- (BOOL)createAndSetupNV12Program { + NSAssert(!_nv12Program, @"NV12 program already created"); + _nv12Program = RTCCreateProgramFromFragmentSource(kNV12FragmentShaderSource); + if (!_nv12Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_nv12Program, "s_textureY"); + GLint uvSampler = glGetUniformLocation(_nv12Program, "s_textureUV"); + + if (ySampler < 0 || uvSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in NV12 shader"); + glDeleteProgram(_nv12Program); + _nv12Program = 0; + return NO; + } + + glUseProgram(_nv12Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uvSampler, kUvTextureUnit); + + return YES; +} + +- (BOOL)prepareVertexBufferWithRotation:(RTCVideoRotation)rotation { + if (!_vertexBuffer && !RTCCreateVertexBuffer(&_vertexBuffer, &_vertexArray)) { + RTCLog(@"Failed to setup vertex buffer"); + return NO; + } +#if !TARGET_OS_IPHONE + glBindVertexArray(_vertexArray); +#endif + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + if (!_currentRotation || rotation != *_currentRotation) { + _currentRotation = absl::optional<RTCVideoRotation>(rotation); + RTCSetVertexData(*_currentRotation); + } + return YES; +} + +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uPlane:(GLuint)uPlane + vPlane:(GLuint)vPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_i420Program && ![self createAndSetupI420Program]) { + RTCLog(@"Failed to setup I420 program"); + return; + } + + glUseProgram(_i420Program); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kUTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kVTextureUnit)); + glBindTexture(GL_TEXTURE_2D, vPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_nv12Program && ![self createAndSetupNV12Program]) { + RTCLog(@"Failed to setup NV12 shader"); + return; + } + + glUseProgram(_nv12Program); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kUvTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uvPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h new file mode 100644 index 0000000000..b78501e9e6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +// RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen +// refreshes, which should be 30fps. We wrap the display link in order to avoid +// a retain cycle since CADisplayLink takes a strong reference onto its target. +// The timer is paused by default. +@interface RTCDisplayLinkTimer : NSObject + +@property(nonatomic) BOOL isPaused; + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler; +- (void)invalidate; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m new file mode 100644 index 0000000000..906bb898d6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m @@ -0,0 +1,59 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDisplayLinkTimer.h" + +#import <UIKit/UIKit.h> + +@implementation RTCDisplayLinkTimer { + CADisplayLink *_displayLink; + void (^_timerHandler)(void); +} + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler { + NSParameterAssert(timerHandler); + if (self = [super init]) { + _timerHandler = timerHandler; + _displayLink = + [CADisplayLink displayLinkWithTarget:self + selector:@selector(displayLinkDidFire:)]; + _displayLink.paused = YES; +#if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_10_0 + _displayLink.preferredFramesPerSecond = 30; +#else + [_displayLink setFrameInterval:2]; +#endif + [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] + forMode:NSRunLoopCommonModes]; + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)isPaused { + return _displayLink.paused; +} + +- (void)setIsPaused:(BOOL)isPaused { + _displayLink.paused = isPaused; +} + +- (void)invalidate { + [_displayLink invalidate]; +} + +- (void)displayLinkDidFire:(CADisplayLink *)displayLink { + _timerHandler(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h new file mode 100644 index 0000000000..24b26cd602 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h @@ -0,0 +1,45 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <UIKit/UIKit.h> + +#import "RTCMacros.h" +#import "RTCVideoRenderer.h" +#import "RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCEAGLVideoView); + +/** + * RTCEAGLVideoView is an RTCVideoRenderer which renders video frames + * in its bounds using OpenGLES 2.0 or OpenGLES 3.0. + */ +RTC_OBJC_EXPORT +NS_EXTENSION_UNAVAILABLE_IOS("Rendering not available in app extensions.") +@interface RTC_OBJC_TYPE (RTCEAGLVideoView) : UIView <RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + +- (instancetype)initWithFrame:(CGRect)frame + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader + NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithCoder:(NSCoder *)aDecoder + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader + NS_DESIGNATED_INITIALIZER; + +/** @abstract Wrapped RTCVideoRotation, or nil. + */ +@property(nonatomic, nullable) NSValue *rotationOverride; +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m new file mode 100644 index 0000000000..89e62d2ce7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m @@ -0,0 +1,295 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCEAGLVideoView.h" + +#import <GLKit/GLKit.h> + +#import "RTCDefaultShader.h" +#import "RTCDisplayLinkTimer.h" +#import "RTCI420TextureCache.h" +#import "RTCNV12TextureCache.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +// RTC_OBJC_TYPE(RTCEAGLVideoView) wraps a GLKView which is setup with +// enableSetNeedsDisplay = NO for the purpose of gaining control of +// exactly when to call -[GLKView display]. This need for extra +// control is required to avoid triggering method calls on GLKView +// that results in attempting to bind the underlying render buffer +// when the drawable size would be empty which would result in the +// error GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT. -[GLKView display] is +// the method that will trigger the binding of the render +// buffer. Because the standard behaviour of -[UIView setNeedsDisplay] +// is disabled for the reasons above, the RTC_OBJC_TYPE(RTCEAGLVideoView) maintains +// its own `isDirty` flag. + +@interface RTC_OBJC_TYPE (RTCEAGLVideoView) +()<GLKViewDelegate> + // `videoFrame` is set when we receive a frame from a worker thread and is read + // from the display link callback so atomicity is required. + @property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@property(nonatomic, readonly) GLKView *glkView; +@end + +@implementation RTC_OBJC_TYPE (RTCEAGLVideoView) { + RTCDisplayLinkTimer *_timer; + EAGLContext *_glContext; + // This flag should only be set and read on the main thread (e.g. by + // setNeedsDisplay) + BOOL _isDirty; + id<RTC_OBJC_TYPE(RTCVideoViewShading)> _shader; + RTCNV12TextureCache *_nv12TextureCache; + RTCI420TextureCache *_i420TextureCache; + // As timestamps should be unique between frames, will store last + // drawn frame timestamp instead of the whole frame to reduce memory usage. + int64_t _lastDrawnFrameTimeStampNs; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize glkView = _glkView; +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)initWithFrame:(CGRect)frame { + return [self initWithFrame:frame shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder { + return [self initWithCoder:aDecoder shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithFrame:(CGRect)frame shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader { + if (self = [super initWithFrame:frame]) { + _shader = shader; + if (![self configure]) { + return nil; + } + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader { + if (self = [super initWithCoder:aDecoder]) { + _shader = shader; + if (![self configure]) { + return nil; + } + } + return self; +} + +- (BOOL)configure { + EAGLContext *glContext = + [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; + if (!glContext) { + glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; + } + if (!glContext) { + RTCLogError(@"Failed to create EAGLContext"); + return NO; + } + _glContext = glContext; + + // GLKView manages a framebuffer for us. + _glkView = [[GLKView alloc] initWithFrame:CGRectZero + context:_glContext]; + _glkView.drawableColorFormat = GLKViewDrawableColorFormatRGBA8888; + _glkView.drawableDepthFormat = GLKViewDrawableDepthFormatNone; + _glkView.drawableStencilFormat = GLKViewDrawableStencilFormatNone; + _glkView.drawableMultisample = GLKViewDrawableMultisampleNone; + _glkView.delegate = self; + _glkView.layer.masksToBounds = YES; + _glkView.enableSetNeedsDisplay = NO; + [self addSubview:_glkView]; + + // Listen to application state in order to clean up OpenGL before app goes + // away. + NSNotificationCenter *notificationCenter = + [NSNotificationCenter defaultCenter]; + [notificationCenter addObserver:self + selector:@selector(willResignActive) + name:UIApplicationWillResignActiveNotification + object:nil]; + [notificationCenter addObserver:self + selector:@selector(didBecomeActive) + name:UIApplicationDidBecomeActiveNotification + object:nil]; + + // Frames are received on a separate thread, so we poll for current frame + // using a refresh rate proportional to screen refresh frequency. This + // occurs on the main thread. + __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self; + _timer = [[RTCDisplayLinkTimer alloc] initWithTimerHandler:^{ + RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf; + [strongSelf displayLinkTimerDidFire]; + }]; + if ([[UIApplication sharedApplication] applicationState] == UIApplicationStateActive) { + [self setupGL]; + } + return YES; +} + +- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { + [super setMultipleTouchEnabled:multipleTouchEnabled]; + _glkView.multipleTouchEnabled = multipleTouchEnabled; +} + +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; + UIApplicationState appState = + [UIApplication sharedApplication].applicationState; + if (appState == UIApplicationStateActive) { + [self teardownGL]; + } + [_timer invalidate]; + [self ensureGLContext]; + _shader = nil; + if (_glContext && [EAGLContext currentContext] == _glContext) { + [EAGLContext setCurrentContext:nil]; + } +} + +#pragma mark - UIView + +- (void)setNeedsDisplay { + [super setNeedsDisplay]; + _isDirty = YES; +} + +- (void)setNeedsDisplayInRect:(CGRect)rect { + [super setNeedsDisplayInRect:rect]; + _isDirty = YES; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + _glkView.frame = self.bounds; +} + +#pragma mark - GLKViewDelegate + +// This method is called when the GLKView's content is dirty and needs to be +// redrawn. This occurs on main thread. +- (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { + // The renderer will draw the frame to the framebuffer corresponding to the + // one used by `view`. + RTC_OBJC_TYPE(RTCVideoFrame) *frame = self.videoFrame; + if (!frame || frame.timeStampNs == _lastDrawnFrameTimeStampNs) { + return; + } + RTCVideoRotation rotation = frame.rotation; + if(_rotationOverride != nil) { + [_rotationOverride getValue: &rotation]; + } + [self ensureGLContext]; + glClear(GL_COLOR_BUFFER_BIT); + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + if (!_nv12TextureCache) { + _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; + } + if (_nv12TextureCache) { + [_nv12TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:rotation + yPlane:_nv12TextureCache.yTexture + uvPlane:_nv12TextureCache.uvTexture]; + [_nv12TextureCache releaseTextures]; + + _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs; + } + } else { + if (!_i420TextureCache) { + _i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:_glContext]; + } + [_i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:rotation + yPlane:_i420TextureCache.yTexture + uPlane:_i420TextureCache.uTexture + vPlane:_i420TextureCache.vTexture]; + + _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs; + } +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf; + [strongSelf.delegate videoView:strongSelf didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)displayLinkTimerDidFire { + // Don't render unless video frame have changed or the view content + // has explicitly been marked dirty. + if (!_isDirty && _lastDrawnFrameTimeStampNs == self.videoFrame.timeStampNs) { + return; + } + + // Always reset isDirty at this point, even if -[GLKView display] + // won't be called in the case the drawable size is empty. + _isDirty = NO; + + // Only call -[GLKView display] if the drawable size is + // non-empty. Calling display will make the GLKView setup its + // render buffer if necessary, but that will fail with error + // GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT if size is empty. + if (self.bounds.size.width > 0 && self.bounds.size.height > 0) { + [_glkView display]; + } +} + +- (void)setupGL { + [self ensureGLContext]; + glDisable(GL_DITHER); + _timer.isPaused = NO; +} + +- (void)teardownGL { + self.videoFrame = nil; + _timer.isPaused = YES; + [_glkView deleteDrawable]; + [self ensureGLContext]; + _nv12TextureCache = nil; + _i420TextureCache = nil; +} + +- (void)didBecomeActive { + [self setupGL]; +} + +- (void)willResignActive { + [self teardownGL]; +} + +- (void)ensureGLContext { + NSAssert(_glContext, @"context shouldn't be nil"); + if ([EAGLContext currentContext] != _glContext) { + [EAGLContext setCurrentContext:_glContext]; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h new file mode 100644 index 0000000000..9fdcc5a695 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCOpenGLDefines.h" +#import "base/RTCVideoFrame.h" + +@interface RTCI420TextureCache : NSObject + +@property(nonatomic, readonly) GLuint yTexture; +@property(nonatomic, readonly) GLuint uTexture; +@property(nonatomic, readonly) GLuint vTexture; + +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWithContext:(GlContextType *)context NS_DESIGNATED_INITIALIZER; + +- (void)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm new file mode 100644 index 0000000000..5dccd4bf6a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm @@ -0,0 +1,157 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCI420TextureCache.h" + +#if TARGET_OS_IPHONE +#import <OpenGLES/ES3/gl.h> +#else +#import <OpenGL/gl3.h> +#endif + +#import "base/RTCI420Buffer.h" +#import "base/RTCVideoFrameBuffer.h" + +#include <vector> + +// Two sets of 3 textures are used here, one for each of the Y, U and V planes. Having two sets +// alleviates CPU blockage in the event that the GPU is asked to render to a texture that is already +// in use. +static const GLsizei kNumTextureSets = 2; +static const GLsizei kNumTexturesPerSet = 3; +static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; + +@implementation RTCI420TextureCache { + BOOL _hasUnpackRowLength; + GLint _currentTextureSet; + // Handles for OpenGL constructs. + GLuint _textures[kNumTextures]; + // Used to create a non-padded plane for GPU upload when we receive padded frames. + std::vector<uint8_t> _planeBuffer; +} + +- (GLuint)yTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet]; +} + +- (GLuint)uTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 1]; +} + +- (GLuint)vTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 2]; +} + +- (instancetype)initWithContext:(GlContextType *)context { + if (self = [super init]) { +#if TARGET_OS_IPHONE + _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3); +#else + _hasUnpackRowLength = YES; +#endif + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + [self setupTextures]; + } + return self; +} + +- (void)dealloc { + glDeleteTextures(kNumTextures, _textures); +} + +- (void)setupTextures { + glGenTextures(kNumTextures, _textures); + // Set parameters for each of the textures we created. + for (GLsizei i = 0; i < kNumTextures; i++) { + glBindTexture(GL_TEXTURE_2D, _textures[i]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } +} + +- (void)uploadPlane:(const uint8_t *)plane + texture:(GLuint)texture + width:(size_t)width + height:(size_t)height + stride:(int32_t)stride { + glBindTexture(GL_TEXTURE_2D, texture); + + const uint8_t *uploadPlane = plane; + if ((size_t)stride != width) { + if (_hasUnpackRowLength) { + // GLES3 allows us to specify stride. + glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast<GLsizei>(width), + static_cast<GLsizei>(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + return; + } else { + // Make an unpadded copy and upload that instead. Quick profiling showed + // that this is faster than uploading row by row using glTexSubImage2D. + uint8_t *unpaddedPlane = _planeBuffer.data(); + for (size_t y = 0; y < height; ++y) { + memcpy(unpaddedPlane + y * width, plane + y * stride, width); + } + uploadPlane = unpaddedPlane; + } + } + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast<GLsizei>(width), + static_cast<GLsizei>(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); +} + +- (void)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; + + id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420]; + + const int chromaWidth = buffer.chromaWidth; + const int chromaHeight = buffer.chromaHeight; + if (buffer.strideY != frame.width || buffer.strideU != chromaWidth || + buffer.strideV != chromaWidth) { + _planeBuffer.resize(buffer.width * buffer.height); + } + + [self uploadPlane:buffer.dataY + texture:self.yTexture + width:buffer.width + height:buffer.height + stride:buffer.strideY]; + + [self uploadPlane:buffer.dataU + texture:self.uTexture + width:chromaWidth + height:chromaHeight + stride:buffer.strideU]; + + [self uploadPlane:buffer.dataV + texture:self.vTexture + width:chromaWidth + height:chromaHeight + stride:buffer.strideV]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.h new file mode 100644 index 0000000000..c9ee986f88 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.h @@ -0,0 +1,42 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#if !TARGET_OS_IPHONE + +#import <AppKit/NSOpenGLView.h> + +#import "RTCVideoRenderer.h" +#import "RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCNSGLVideoView); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCNSGLVideoViewDelegate)<RTC_OBJC_TYPE(RTCVideoViewDelegate)> @end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCNSGLVideoView) : NSOpenGLView <RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + +- (instancetype)initWithFrame:(NSRect)frameRect + pixelFormat:(NSOpenGLPixelFormat *)format + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END + +#endif diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m new file mode 100644 index 0000000000..168c73126f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNSGLVideoView.m @@ -0,0 +1,199 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#if !TARGET_OS_IPHONE + +#import "RTCNSGLVideoView.h" + +#import <AppKit/NSOpenGL.h> +#import <CoreVideo/CVDisplayLink.h> +#import <OpenGL/gl3.h> + +#import "RTCDefaultShader.h" +#import "RTCI420TextureCache.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" + +@interface RTC_OBJC_TYPE (RTCNSGLVideoView) +() + // `videoFrame` is set when we receive a frame from a worker thread and is read + // from the display link callback so atomicity is required. + @property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * + videoFrame; +@property(atomic, strong) RTCI420TextureCache *i420TextureCache; + +- (void)drawFrame; +@end + +static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, + const CVTimeStamp *now, + const CVTimeStamp *outputTime, + CVOptionFlags flagsIn, + CVOptionFlags *flagsOut, + void *displayLinkContext) { + RTC_OBJC_TYPE(RTCNSGLVideoView) *view = + (__bridge RTC_OBJC_TYPE(RTCNSGLVideoView) *)displayLinkContext; + [view drawFrame]; + return kCVReturnSuccess; +} + +@implementation RTC_OBJC_TYPE (RTCNSGLVideoView) { + CVDisplayLinkRef _displayLink; + RTC_OBJC_TYPE(RTCVideoFrame) * _lastDrawnFrame; + id<RTC_OBJC_TYPE(RTCVideoViewShading)> _shader; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize i420TextureCache = _i420TextureCache; + +- (instancetype)initWithFrame:(NSRect)frame pixelFormat:(NSOpenGLPixelFormat *)format { + return [self initWithFrame:frame pixelFormat:format shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithFrame:(NSRect)frame + pixelFormat:(NSOpenGLPixelFormat *)format + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader { + if (self = [super initWithFrame:frame pixelFormat:format]) { + _shader = shader; + } + return self; +} + +- (void)dealloc { + [self teardownDisplayLink]; +} + +- (void)drawRect:(NSRect)rect { + [self drawFrame]; +} + +- (void)reshape { + [super reshape]; + NSRect frame = [self frame]; + [self ensureGLContext]; + CGLLockContext([[self openGLContext] CGLContextObj]); + glViewport(0, 0, frame.size.width, frame.size.height); + CGLUnlockContext([[self openGLContext] CGLContextObj]); +} + +- (void)lockFocus { + NSOpenGLContext *context = [self openGLContext]; + [super lockFocus]; + if ([context view] != self) { + [context setView:self]; + } + [context makeCurrentContext]; +} + +- (void)prepareOpenGL { + [super prepareOpenGL]; + [self ensureGLContext]; + glDisable(GL_DITHER); + [self setupDisplayLink]; +} + +- (void)clearGLContext { + [self ensureGLContext]; + self.i420TextureCache = nil; + [super clearGLContext]; +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate videoView:self didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)drawFrame { + RTC_OBJC_TYPE(RTCVideoFrame) *frame = self.videoFrame; + if (!frame || frame == _lastDrawnFrame) { + return; + } + // This method may be called from CVDisplayLink callback which isn't on the + // main thread so we have to lock the GL context before drawing. + NSOpenGLContext *context = [self openGLContext]; + CGLLockContext([context CGLContextObj]); + + [self ensureGLContext]; + glClear(GL_COLOR_BUFFER_BIT); + + // Rendering native CVPixelBuffer is not supported on OS X. + // TODO(magjed): Add support for NV12 texture cache on OS X. + frame = [frame newI420VideoFrame]; + if (!self.i420TextureCache) { + self.i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:context]; + } + RTCI420TextureCache *i420TextureCache = self.i420TextureCache; + if (i420TextureCache) { + [i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:frame.rotation + yPlane:i420TextureCache.yTexture + uPlane:i420TextureCache.uTexture + vPlane:i420TextureCache.vTexture]; + [context flushBuffer]; + _lastDrawnFrame = frame; + } + CGLUnlockContext([context CGLContextObj]); +} + +- (void)setupDisplayLink { + if (_displayLink) { + return; + } + // Synchronize buffer swaps with vertical refresh rate. + GLint swapInt = 1; + [[self openGLContext] setValues:&swapInt forParameter:NSOpenGLCPSwapInterval]; + + // Create display link. + CVDisplayLinkCreateWithActiveCGDisplays(&_displayLink); + CVDisplayLinkSetOutputCallback(_displayLink, + &OnDisplayLinkFired, + (__bridge void *)self); + // Set the display link for the current renderer. + CGLContextObj cglContext = [[self openGLContext] CGLContextObj]; + CGLPixelFormatObj cglPixelFormat = [[self pixelFormat] CGLPixelFormatObj]; + CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext( + _displayLink, cglContext, cglPixelFormat); + CVDisplayLinkStart(_displayLink); +} + +- (void)teardownDisplayLink { + if (!_displayLink) { + return; + } + CVDisplayLinkRelease(_displayLink); + _displayLink = NULL; +} + +- (void)ensureGLContext { + NSOpenGLContext* context = [self openGLContext]; + NSAssert(context, @"context shouldn't be nil"); + if ([NSOpenGLContext currentContext] != context) { + [context makeCurrentContext]; + } +} + +@end + +#endif // !TARGET_OS_IPHONE diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h new file mode 100644 index 0000000000..f202b836b5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <GLKit/GLKit.h> + +#import "base/RTCMacros.h" + +@class RTC_OBJC_TYPE(RTCVideoFrame); + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCNV12TextureCache : NSObject + +@property(nonatomic, readonly) GLuint yTexture; +@property(nonatomic, readonly) GLuint uvTexture; + +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithContext:(EAGLContext *)context NS_DESIGNATED_INITIALIZER; + +- (BOOL)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +- (void)releaseTextures; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m new file mode 100644 index 0000000000..a520ac45b4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m @@ -0,0 +1,113 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNV12TextureCache.h" + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +@implementation RTCNV12TextureCache { + CVOpenGLESTextureCacheRef _textureCache; + CVOpenGLESTextureRef _yTextureRef; + CVOpenGLESTextureRef _uvTextureRef; +} + +- (GLuint)yTexture { + return CVOpenGLESTextureGetName(_yTextureRef); +} + +- (GLuint)uvTexture { + return CVOpenGLESTextureGetName(_uvTextureRef); +} + +- (instancetype)initWithContext:(EAGLContext *)context { + if (self = [super init]) { + CVReturn ret = CVOpenGLESTextureCacheCreate( + kCFAllocatorDefault, NULL, +#if COREVIDEO_USE_EAGLCONTEXT_CLASS_IN_API + context, +#else + (__bridge void *)context, +#endif + NULL, &_textureCache); + if (ret != kCVReturnSuccess) { + self = nil; + } + } + return self; +} + +- (BOOL)loadTexture:(CVOpenGLESTextureRef *)textureOut + pixelBuffer:(CVPixelBufferRef)pixelBuffer + planeIndex:(int)planeIndex + pixelFormat:(GLenum)pixelFormat { + const int width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex); + const int height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex); + + if (*textureOut) { + CFRelease(*textureOut); + *textureOut = nil; + } + CVReturn ret = CVOpenGLESTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, GL_TEXTURE_2D, pixelFormat, width, + height, pixelFormat, GL_UNSIGNED_BYTE, planeIndex, textureOut); + if (ret != kCVReturnSuccess) { + if (*textureOut) { + CFRelease(*textureOut); + *textureOut = nil; + } + return NO; + } + NSAssert(CVOpenGLESTextureGetTarget(*textureOut) == GL_TEXTURE_2D, + @"Unexpected GLES texture target"); + glBindTexture(GL_TEXTURE_2D, CVOpenGLESTextureGetName(*textureOut)); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + return YES; +} + +- (BOOL)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + NSAssert([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]], + @"frame must be CVPixelBuffer backed"); + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + CVPixelBufferRef pixelBuffer = rtcPixelBuffer.pixelBuffer; + return [self loadTexture:&_yTextureRef + pixelBuffer:pixelBuffer + planeIndex:0 + pixelFormat:GL_LUMINANCE] && + [self loadTexture:&_uvTextureRef + pixelBuffer:pixelBuffer + planeIndex:1 + pixelFormat:GL_LUMINANCE_ALPHA]; +} + +- (void)releaseTextures { + if (_uvTextureRef) { + CFRelease(_uvTextureRef); + _uvTextureRef = nil; + } + if (_yTextureRef) { + CFRelease(_yTextureRef); + _yTextureRef = nil; + } +} + +- (void)dealloc { + [self releaseTextures]; + if (_textureCache) { + CFRelease(_textureCache); + _textureCache = nil; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h new file mode 100644 index 0000000000..4088535861 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h @@ -0,0 +1,37 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#if TARGET_OS_IPHONE +#define RTC_PIXEL_FORMAT GL_LUMINANCE +#define SHADER_VERSION +#define VERTEX_SHADER_IN "attribute" +#define VERTEX_SHADER_OUT "varying" +#define FRAGMENT_SHADER_IN "varying" +#define FRAGMENT_SHADER_OUT +#define FRAGMENT_SHADER_COLOR "gl_FragColor" +#define FRAGMENT_SHADER_TEXTURE "texture2D" + +@class EAGLContext; +typedef EAGLContext GlContextType; +#else +#define RTC_PIXEL_FORMAT GL_RED +#define SHADER_VERSION "#version 150\n" +#define VERTEX_SHADER_IN "in" +#define VERTEX_SHADER_OUT "out" +#define FRAGMENT_SHADER_IN "in" +#define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" +#define FRAGMENT_SHADER_COLOR "fragColor" +#define FRAGMENT_SHADER_TEXTURE "texture" + +@class NSOpenGLContext; +typedef NSOpenGLContext GlContextType; +#endif diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h new file mode 100644 index 0000000000..d1b91fb643 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h @@ -0,0 +1,21 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "base/RTCVideoFrame.h" + +RTC_EXTERN const char kRTCVertexShaderSource[]; + +RTC_EXTERN GLuint RTCCreateShader(GLenum type, const GLchar* source); +RTC_EXTERN GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader); +RTC_EXTERN GLuint +RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]); +RTC_EXTERN BOOL RTCCreateVertexBuffer(GLuint* vertexBuffer, + GLuint* vertexArray); +RTC_EXTERN void RTCSetVertexData(RTCVideoRotation rotation); diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm new file mode 100644 index 0000000000..8eccd7fbec --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm @@ -0,0 +1,189 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCShader.h" + +#if TARGET_OS_IPHONE +#import <OpenGLES/ES3/gl.h> +#else +#import <OpenGL/gl3.h> +#endif + +#include <algorithm> +#include <array> +#include <memory> + +#import "RTCOpenGLDefines.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +// Vertex shader doesn't do anything except pass coordinates through. +const char kRTCVertexShaderSource[] = + SHADER_VERSION + VERTEX_SHADER_IN " vec2 position;\n" + VERTEX_SHADER_IN " vec2 texcoord;\n" + VERTEX_SHADER_OUT " vec2 v_texcoord;\n" + "void main() {\n" + " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" + " v_texcoord = texcoord;\n" + "}\n"; + +// Compiles a shader of the given `type` with GLSL source `source` and returns +// the shader handle or 0 on error. +GLuint RTCCreateShader(GLenum type, const GLchar *source) { + GLuint shader = glCreateShader(type); + if (!shader) { + return 0; + } + glShaderSource(shader, 1, &source, NULL); + glCompileShader(shader); + GLint compileStatus = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); + if (compileStatus == GL_FALSE) { + GLint logLength = 0; + // The null termination character is included in the returned log length. + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &logLength); + if (logLength > 0) { + std::unique_ptr<char[]> compileLog(new char[logLength]); + // The returned string is null terminated. + glGetShaderInfoLog(shader, logLength, NULL, compileLog.get()); + RTC_LOG(LS_ERROR) << "Shader compile error: " << compileLog.get(); + } + glDeleteShader(shader); + shader = 0; + } + return shader; +} + +// Links a shader program with the given vertex and fragment shaders and +// returns the program handle or 0 on error. +GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader) { + if (vertexShader == 0 || fragmentShader == 0) { + return 0; + } + GLuint program = glCreateProgram(); + if (!program) { + return 0; + } + glAttachShader(program, vertexShader); + glAttachShader(program, fragmentShader); + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (linkStatus == GL_FALSE) { + glDeleteProgram(program); + program = 0; + } + return program; +} + +// Creates and links a shader program with the given fragment shader source and +// a plain vertex shader. Returns the program handle or 0 on error. +GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]) { + GLuint vertexShader = RTCCreateShader(GL_VERTEX_SHADER, kRTCVertexShaderSource); + RTC_CHECK(vertexShader) << "failed to create vertex shader"; + GLuint fragmentShader = + RTCCreateShader(GL_FRAGMENT_SHADER, fragmentShaderSource); + RTC_CHECK(fragmentShader) << "failed to create fragment shader"; + GLuint program = RTCCreateProgram(vertexShader, fragmentShader); + // Shaders are created only to generate program. + if (vertexShader) { + glDeleteShader(vertexShader); + } + if (fragmentShader) { + glDeleteShader(fragmentShader); + } + + // Set vertex shader variables 'position' and 'texcoord' in program. + GLint position = glGetAttribLocation(program, "position"); + GLint texcoord = glGetAttribLocation(program, "texcoord"); + if (position < 0 || texcoord < 0) { + glDeleteProgram(program); + return 0; + } + + // Read position attribute with size of 2 and stride of 4 beginning at the start of the array. The + // last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); + glEnableVertexAttribArray(position); + + // Read texcoord attribute with size of 2 and stride of 4 beginning at the first texcoord in the + // array. The last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer( + texcoord, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)(2 * sizeof(GLfloat))); + glEnableVertexAttribArray(texcoord); + + return program; +} + +BOOL RTCCreateVertexBuffer(GLuint *vertexBuffer, GLuint *vertexArray) { +#if !TARGET_OS_IPHONE + glGenVertexArrays(1, vertexArray); + if (*vertexArray == 0) { + return NO; + } + glBindVertexArray(*vertexArray); +#endif + glGenBuffers(1, vertexBuffer); + if (*vertexBuffer == 0) { + glDeleteVertexArrays(1, vertexArray); + return NO; + } + glBindBuffer(GL_ARRAY_BUFFER, *vertexBuffer); + glBufferData(GL_ARRAY_BUFFER, 4 * 4 * sizeof(GLfloat), NULL, GL_DYNAMIC_DRAW); + return YES; +} + +// Set vertex data to the currently bound vertex buffer. +void RTCSetVertexData(RTCVideoRotation rotation) { + // When modelview and projection matrices are identity (default) the world is + // contained in the square around origin with unit size 2. Drawing to these + // coordinates is equivalent to drawing to the entire screen. The texture is + // stretched over that square using texture coordinates (u, v) that range + // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically + // here because the incoming frame has origin in upper left hand corner but + // OpenGL expects origin in bottom left corner. + std::array<std::array<GLfloat, 2>, 4> UVCoords = {{ + {{0, 1}}, // Lower left. + {{1, 1}}, // Lower right. + {{1, 0}}, // Upper right. + {{0, 0}}, // Upper left. + }}; + + // Rotate the UV coordinates. + int rotation_offset; + switch (rotation) { + case RTCVideoRotation_0: + rotation_offset = 0; + break; + case RTCVideoRotation_90: + rotation_offset = 1; + break; + case RTCVideoRotation_180: + rotation_offset = 2; + break; + case RTCVideoRotation_270: + rotation_offset = 3; + break; + } + std::rotate(UVCoords.begin(), UVCoords.begin() + rotation_offset, + UVCoords.end()); + + const GLfloat gVertices[] = { + // X, Y, U, V. + -1, -1, UVCoords[0][0], UVCoords[0][1], + 1, -1, UVCoords[1][0], UVCoords[1][1], + 1, 1, UVCoords[2][0], UVCoords[2][1], + -1, 1, UVCoords[3][0], UVCoords[3][1], + }; + + glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(gVertices), gVertices); +} diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h new file mode 100644 index 0000000000..9df30a8fa0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h @@ -0,0 +1,39 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * RTCVideoViewShading provides a way for apps to customize the OpenGL(ES shaders + * used in rendering for the RTCEAGLVideoView/RTCNSGLVideoView. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoViewShading)<NSObject> + + /** Callback for I420 frames. Each plane is given as a texture. */ + - (void)applyShadingForFrameWithWidth : (int)width height : (int)height rotation + : (RTCVideoRotation)rotation yPlane : (GLuint)yPlane uPlane : (GLuint)uPlane vPlane + : (GLuint)vPlane; + +/** Callback for NV12 frames. Each plane is given as a texture. */ +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane; + +@end + +NS_ASSUME_NONNULL_END |