diff options
Diffstat (limited to '')
502 files changed, 53525 insertions, 0 deletions
diff --git a/third_party/libwebrtc/sdk/BUILD.gn b/third_party/libwebrtc/sdk/BUILD.gn new file mode 100644 index 0000000000..eea26dc31d --- /dev/null +++ b/third_party/libwebrtc/sdk/BUILD.gn @@ -0,0 +1,1739 @@ +# Copyright 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("//third_party/libaom/options.gni") +import("../webrtc.gni") +if (is_ios) { + import("//build/config/ios/ios_sdk.gni") + import("//build/config/ios/rules.gni") +} +if (is_mac) { + import("//build/config/mac/rules.gni") +} + +group("sdk") { + public_deps = [] + if (!build_with_chromium) { + if (is_android) { + public_deps += [ "android" ] + } + if (is_ios) { + public_deps += [ ":framework_objc" ] + } + } +} + +rtc_library("media_constraints") { + sources = [ + "media_constraints.cc", + "media_constraints.h", + ] + deps = [ + "../api:audio_options_api", + "../api:libjingle_peerconnection_api", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("sdk_tests") { + testonly = true + sources = [ "media_constraints_unittest.cc" ] + deps = [ + ":media_constraints", + "../test:test_support", + ] +} + +if (is_ios || is_mac) { + config("common_config_objc") { + include_dirs = [ + "objc", + + # This is needed so that framework headers can include base headers + # without pathname (so it works from within the framework module). + "objc/base", + ] + cflags = [ + "-Wimplicit-retain-self", + "-Wstrict-overflow", + "-Wmissing-field-initializers", + ] + + if (use_clang_coverage) { + configs = [ "//build/config/coverage:default_coverage" ] + } + } + + config("used_from_extension") { + if (is_ios && rtc_apprtcmobile_broadcast_extension) { + cflags = [ "-fapplication-extension" ] + } + } + + # TODO(bugs.webrtc.org/9627): Remove this when unused. Targets should depend on base_objc + # or helpers_objc directly instead. + rtc_library("common_objc") { + visibility = [ "*" ] + + sources = [ "objc/helpers/noop.mm" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ] + } + + rtc_library("base_objc") { + visibility = [ "*" ] + sources = [ + "objc/base/RTCCodecSpecificInfo.h", + "objc/base/RTCEncodedImage.h", + "objc/base/RTCEncodedImage.m", + "objc/base/RTCI420Buffer.h", + "objc/base/RTCLogging.h", + "objc/base/RTCLogging.mm", + "objc/base/RTCMacros.h", + "objc/base/RTCMutableI420Buffer.h", + "objc/base/RTCMutableYUVPlanarBuffer.h", + "objc/base/RTCSSLCertificateVerifier.h", + "objc/base/RTCVideoCapturer.h", + "objc/base/RTCVideoCapturer.m", + "objc/base/RTCVideoCodecInfo.h", + "objc/base/RTCVideoCodecInfo.m", + "objc/base/RTCVideoDecoder.h", + "objc/base/RTCVideoDecoderFactory.h", + "objc/base/RTCVideoEncoder.h", + "objc/base/RTCVideoEncoderFactory.h", + "objc/base/RTCVideoEncoderQpThresholds.h", + "objc/base/RTCVideoEncoderQpThresholds.m", + "objc/base/RTCVideoEncoderSettings.h", + "objc/base/RTCVideoEncoderSettings.m", + "objc/base/RTCVideoFrame.h", + "objc/base/RTCVideoFrame.mm", + "objc/base/RTCVideoFrameBuffer.h", + "objc/base/RTCVideoRenderer.h", + "objc/base/RTCYUVPlanarBuffer.h", + ] + + deps = [ + "../rtc_base:checks", + "../rtc_base:logging", + ] + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] + } + + rtc_library("helpers_objc") { + sources = [ + "objc/helpers/AVCaptureSession+DevicePosition.h", + "objc/helpers/AVCaptureSession+DevicePosition.mm", + "objc/helpers/NSString+StdString.h", + "objc/helpers/NSString+StdString.mm", + "objc/helpers/RTCDispatcher+Private.h", + "objc/helpers/RTCDispatcher.h", + "objc/helpers/RTCDispatcher.m", + "objc/helpers/scoped_cftyperef.h", + ] + + deps = [ + ":base_objc", + "../rtc_base:checks", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + + frameworks = [ + "AVFoundation.framework", + "CoreMedia.framework", + "Foundation.framework", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] + + if (is_ios) { + sources += [ + "objc/helpers/RTCCameraPreviewView.h", + "objc/helpers/RTCCameraPreviewView.m", + "objc/helpers/UIDevice+RTCDevice.h", + "objc/helpers/UIDevice+RTCDevice.mm", + ] + frameworks += [ "UIKit.framework" ] + } + } + + if (!build_with_chromium) { + rtc_library("callback_logger_objc") { + sources = [ + "objc/api/logging/RTCCallbackLogger.h", + "objc/api/logging/RTCCallbackLogger.mm", + ] + + deps = [ + ":base_objc", + ":helpers_objc", + "../rtc_base:checks", + "../rtc_base:log_sinks", + "../rtc_base:logging", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("file_logger_objc") { + sources = [ + "objc/api/peerconnection/RTCFileLogger.h", + "objc/api/peerconnection/RTCFileLogger.mm", + ] + + deps = [ + ":base_objc", + "../rtc_base:checks", + "../rtc_base:file_rotating_stream", + "../rtc_base:log_sinks", + "../rtc_base:logging", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + } + } + + if (!build_with_chromium) { + if (is_ios) { + rtc_library("native_api_audio_device_module") { + visibility = [ "*" ] + + sources = [ + "objc/native/api/audio_device_module.h", + "objc/native/api/audio_device_module.mm", + ] + + deps = [ + ":audio_device", + "../api:make_ref_counted", + "../modules/audio_device:audio_device_api", + "../modules/audio_device:audio_device_generic", + "../rtc_base:checks", + "../rtc_base:logging", + "../system_wrappers", + ] + } + + rtc_source_set("audio_session_observer") { + visibility = [ ":*" ] + + sources = [ "objc/native/src/audio/audio_session_observer.h" ] + + deps = [ "../rtc_base:threading" ] + } + + rtc_library("opengl_ui_objc") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", # TODO(bugs.webrtc.org/8396): Remove. + "default_task_queue", + ] + sources = [ + "objc/components/renderer/opengl/RTCDisplayLinkTimer.h", + "objc/components/renderer/opengl/RTCDisplayLinkTimer.m", + "objc/components/renderer/opengl/RTCEAGLVideoView.h", + "objc/components/renderer/opengl/RTCEAGLVideoView.m", + ] + + # TODO(bugs.webrtc.org/12937): Remove OpenGL deprecation warning + # workaround. + defines = [ "GLES_SILENCE_DEPRECATION" ] + configs += [ "..:common_objc" ] + deps = [ + ":base_objc", + ":helpers_objc", + ":metal_objc", + ":opengl_objc", + ":videocapture_objc", + ":videoframebuffer_objc", + ] + } + + rtc_library("audio_device") { + visibility = [ "*" ] + + sources = [ + "objc/native/src/audio/audio_device_ios.h", + "objc/native/src/audio/audio_device_ios.mm", + "objc/native/src/audio/audio_device_module_ios.h", + "objc/native/src/audio/audio_device_module_ios.mm", + "objc/native/src/audio/helpers.h", + "objc/native/src/audio/helpers.mm", + "objc/native/src/audio/voice_processing_audio_unit.h", + "objc/native/src/audio/voice_processing_audio_unit.mm", + ] + + deps = [ + ":audio_objc", + ":audio_session_observer", + ":base_objc", + "../api:array_view", + "../api:scoped_refptr", + "../api:sequence_checker", + "../api/task_queue", + "../api/task_queue:default_task_queue_factory", + "../api/task_queue:pending_task_safety_flag", + "../modules/audio_device:audio_device_api", + "../modules/audio_device:audio_device_buffer", + "../modules/audio_device:audio_device_config", + "../modules/audio_device:audio_device_generic", + "../rtc_base:buffer", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:macromagic", + "../rtc_base:refcount", + "../rtc_base:threading", + "../rtc_base:timeutils", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] + + frameworks = [ "AudioToolbox.framework" ] + if (is_mac) { + frameworks += [ "AudioUnit.framework" ] + } + } + + # This target exists to expose :audio_session_objc and + # :audio_session_delegate_adapter_objc for backward compatibility, + # and should be deprecated. + group("audio_objc") { + public_deps = [ # no-presubmit-check TODO(webrtc:11238) + ":audio_session_delegate_adapter_objc", + ":audio_session_objc", + ] + } + + rtc_library("audio_session_delegate_adapter_objc") { + sources = [ + "objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h", + "objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":audio_session_objc", + ":audio_session_observer", + ":base_objc", + ] + } + + rtc_library("audio_session_objc") { + visibility = [ "*" ] + + sources = [ + "objc/components/audio/RTCAudioSession+Configuration.mm", + "objc/components/audio/RTCAudioSession+Private.h", + "objc/components/audio/RTCAudioSession.h", + "objc/components/audio/RTCAudioSession.mm", + "objc/components/audio/RTCAudioSessionConfiguration.h", + "objc/components/audio/RTCAudioSessionConfiguration.m", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] + + frameworks = [ "AVFoundation.framework" ] + + deps = [ + ":base_objc", + ":helpers_objc", + "../rtc_base:checks", + "../rtc_base/synchronization:mutex", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] + } + + rtc_source_set("network_monitor_observer") { + visibility = [ ":*" ] + + sources = [ "objc/native/src/network_monitor_observer.h" ] + + deps = [ + "../rtc_base:network_constants", + "../rtc_base:stringutils", + "../rtc_base:threading", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("network_monitor_objc") { + visibility = [ "*" ] + + sources = [ + "objc/components/network/RTCNetworkMonitor+Private.h", + "objc/components/network/RTCNetworkMonitor.h", + "objc/components/network/RTCNetworkMonitor.mm", + ] + + configs += [ ":used_from_extension" ] + + frameworks = [ "Network.framework" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ":network_monitor_observer", + "../rtc_base:stringutils", + "../rtc_base/system:gcd_helpers", + ] + } + + rtc_library("opengl_objc") { + sources = [ + "objc/components/renderer/opengl/RTCDefaultShader.h", + "objc/components/renderer/opengl/RTCDefaultShader.mm", + "objc/components/renderer/opengl/RTCI420TextureCache.h", + "objc/components/renderer/opengl/RTCI420TextureCache.mm", + "objc/components/renderer/opengl/RTCNV12TextureCache.h", + "objc/components/renderer/opengl/RTCNV12TextureCache.m", + "objc/components/renderer/opengl/RTCOpenGLDefines.h", + "objc/components/renderer/opengl/RTCShader.h", + "objc/components/renderer/opengl/RTCShader.mm", + "objc/components/renderer/opengl/RTCVideoViewShading.h", + ] + frameworks = [ + "CoreVideo.framework", + "GLKit.framework", + "OpenGLES.framework", + "QuartzCore.framework", + ] + + # TODO(bugs.webrtc.org/12937): Remove OpenGL deprecation warning + # workaround. + defines = [ "GLES_SILENCE_DEPRECATION" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ":mediaconstraints_objc", + ":native_video", + ":videoframebuffer_objc", + ":videosource_objc", + "../api:libjingle_peerconnection_api", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../common_video", + "../media:rtc_media_base", + "../rtc_base:checks", + "../rtc_base:logging", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + } + } + + rtc_source_set("audio_device_api_objc") { + visibility = [ "*" ] + + sources = [ "objc/components/audio/RTCAudioDevice.h" ] + + public_configs = [ ":common_config_objc" ] + + frameworks = [ "AudioToolbox.framework" ] + + deps = [ ":base_objc" ] + } + + rtc_library("audio_device_objc") { + visibility = [ "*" ] + allow_poison = [ "default_task_queue" ] + sources = [ + "objc/native/src/objc_audio_device.h", + "objc/native/src/objc_audio_device.mm", + "objc/native/src/objc_audio_device_delegate.h", + "objc/native/src/objc_audio_device_delegate.mm", + ] + + deps = [ + ":audio_device_api_objc", + "../api:array_view", + "../api:make_ref_counted", + "../api:refcountedbase", + "../api:scoped_refptr", + "../api:sequence_checker", + "../api/task_queue", + "../api/task_queue:default_task_queue_factory", + "../modules/audio_device:audio_device_api", + "../modules/audio_device:audio_device_buffer", + "../rtc_base:buffer", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:safe_minmax", + "../rtc_base:threading", + "../rtc_base:timeutils", + ] + if (is_mac) { + frameworks = [ "AudioUnit.framework" ] + } + } + + rtc_library("objc_audio_device_module") { + visibility = [ "*" ] + allow_poison = [ "default_task_queue" ] + sources = [ + "objc/native/api/objc_audio_device_module.h", + "objc/native/api/objc_audio_device_module.mm", + ] + + deps = [ + ":audio_device_api_objc", + ":audio_device_objc", + "../api:make_ref_counted", + "../modules/audio_device:audio_device_api", + "../rtc_base:logging", + ] + if (is_mac) { + frameworks = [ "AudioUnit.framework" ] + } + } + + if (!build_with_mozilla) { + rtc_library("videosource_objc") { + sources = [ + "objc/api/peerconnection/RTCVideoSource+Private.h", + "objc/api/peerconnection/RTCVideoSource.h", + "objc/api/peerconnection/RTCVideoSource.mm", + ] + + deps = [ + ":base_objc", + ":mediasource_objc", + ":native_video", + ":videoframebuffer_objc", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../common_video", + "../media:rtc_media_base", + "../pc:video_track_source_proxy", + "../rtc_base:checks", + "../rtc_base:threading", + "//third_party/libyuv", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + } + } + + rtc_library("videoframebuffer_objc") { + visibility = [ "*" ] + sources = [ + "objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h", + "objc/api/video_frame_buffer/RTCNativeI420Buffer.h", + "objc/api/video_frame_buffer/RTCNativeI420Buffer.mm", + "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h", + "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.mm", + "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", + "objc/components/video_frame_buffer/RTCCVPixelBuffer.mm", + ] + deps = [ + ":base_objc", + "../rtc_base:logging", + "//api/video:video_frame", + "//api/video:video_rtp_headers", + "//common_video", + "//rtc_base:checks", + "//third_party/libyuv", + ] + configs += [ + "..:common_objc", + ":used_from_extension", + ] + frameworks = [ + "VideoToolbox.framework", + "CoreGraphics.framework", + "CoreVideo.framework", + ] + } + + if (!build_with_mozilla) { + rtc_library("metal_objc") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", # TODO(bugs.webrtc.org/8396): Remove. + "default_task_queue", + ] + sources = [ + "objc/components/renderer/metal/RTCMTLI420Renderer.h", + "objc/components/renderer/metal/RTCMTLI420Renderer.mm", + "objc/components/renderer/metal/RTCMTLNV12Renderer.h", + "objc/components/renderer/metal/RTCMTLNV12Renderer.mm", + "objc/components/renderer/metal/RTCMTLRGBRenderer.h", + "objc/components/renderer/metal/RTCMTLRGBRenderer.mm", + "objc/components/renderer/metal/RTCMTLRenderer+Private.h", + "objc/components/renderer/metal/RTCMTLRenderer.h", + "objc/components/renderer/metal/RTCMTLRenderer.mm", + ] + frameworks = [ + "CoreVideo.framework", + "Metal.framework", + "MetalKit.framework", + ] + if (is_ios) { + sources += [ + "objc/components/renderer/metal/RTCMTLVideoView.h", + "objc/components/renderer/metal/RTCMTLVideoView.m", + ] + } + if (is_mac) { + sources += [ + "objc/components/renderer/metal/RTCMTLNSVideoView.h", + "objc/components/renderer/metal/RTCMTLNSVideoView.m", + ] + frameworks += [ "AppKit.framework" ] + } + deps = [ + ":base_objc", + ":peerconnectionfactory_base_objc", + ":videoframebuffer_objc", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../rtc_base:checks", + ] + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + } + + # TODO(bugs.webrtc.org/9627): Remove this target. + rtc_library("videocapturebase_objc") { + visibility = [ "*" ] + sources = [ "objc/helpers/noop.mm" ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":videoframebuffer_objc", + ] + } + } + + rtc_library("videocapture_objc") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + sources = [ + "objc/components/capturer/RTCCameraVideoCapturer.h", + "objc/components/capturer/RTCCameraVideoCapturer.m", + "objc/components/capturer/RTCFileVideoCapturer.h", + "objc/components/capturer/RTCFileVideoCapturer.m", + ] + frameworks = [ + "AVFoundation.framework", + "CoreVideo.framework", + "QuartzCore.framework", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + ":videoframebuffer_objc", + "../rtc_base/system:gcd_helpers", + ] + } + + if (!build_with_mozilla) { + rtc_library("videocodec_objc") { + visibility = [ "*" ] + configs += [ "..:no_global_constructors" ] + sources = [ + "objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h", + "objc/components/video_codec/RTCCodecSpecificInfoH264.h", + "objc/components/video_codec/RTCCodecSpecificInfoH264.mm", + "objc/components/video_codec/RTCH264ProfileLevelId.h", + "objc/components/video_codec/RTCH264ProfileLevelId.mm", + ] + if (is_ios) { + sources += [ + "objc/components/video_codec/UIDevice+H264Profile.h", + "objc/components/video_codec/UIDevice+H264Profile.mm", + ] + } + + public_configs = [ ":common_config_objc" ] + deps = [ + ":base_objc", + ":helpers_objc", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../media:media_constants", + "../media:rtc_media_base", + "../modules/video_coding:video_codec_interface", + "../rtc_base:checks", + ] + } + + rtc_library("default_codec_factory_objc") { + sources = [ + "objc/components/video_codec/RTCDefaultVideoDecoderFactory.h", + "objc/components/video_codec/RTCDefaultVideoDecoderFactory.m", + "objc/components/video_codec/RTCDefaultVideoEncoderFactory.h", + "objc/components/video_codec/RTCDefaultVideoEncoderFactory.m", + ] + + deps = [ + ":base_objc", + ":native_video", + ":videocodec_objc", + ":videotoolbox_objc", + ":vp8", + ":vp9", + ":vpx_codec_constants", + ] + + defines = [] + if (enable_libaom) { + defines += [ "RTC_USE_LIBAOM_AV1_ENCODER" ] + deps += [ ":libaom_av1_encoder" ] + } + + if (rtc_include_dav1d_in_internal_decoder_factory) { + deps += [ ":dav1d_decoder" ] + } + } + + rtc_library("vpx_codec_constants") { + configs += [ "..:no_global_constructors" ] + sources = [ + "objc/api/video_codec/RTCVideoCodecConstants.h", + "objc/api/video_codec/RTCVideoCodecConstants.mm", + ] + + deps = [ + ":base_objc", + "../media:media_constants", + "../media:rtc_media_base", + ] + } + + rtc_library("vp8") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "objc/api/video_codec/RTCVideoDecoderVP8.h", + "objc/api/video_codec/RTCVideoDecoderVP8.mm", + "objc/api/video_codec/RTCVideoEncoderVP8.h", + "objc/api/video_codec/RTCVideoEncoderVP8.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../modules/video_coding:webrtc_vp8", + ] + } + + rtc_library("vp9") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "objc/api/video_codec/RTCVideoDecoderVP9.h", + "objc/api/video_codec/RTCVideoDecoderVP9.mm", + "objc/api/video_codec/RTCVideoEncoderVP9.h", + "objc/api/video_codec/RTCVideoEncoderVP9.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../media:rtc_media_base", + "../modules/video_coding:webrtc_vp9", + ] + } + + rtc_library("dav1d_decoder") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "objc/api/video_codec/RTCVideoDecoderAV1.h", + "objc/api/video_codec/RTCVideoDecoderAV1.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../media:rtc_media_base", + "../modules/video_coding/codecs/av1:dav1d_decoder", + ] + } + + rtc_library("libaom_av1_encoder") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "objc/api/video_codec/RTCVideoEncoderAV1.h", + "objc/api/video_codec/RTCVideoEncoderAV1.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../media:rtc_media_base", + "../modules/video_coding/codecs/av1:libaom_av1_encoder", + ] + } + + rtc_library("mediaconstraints_objc") { + configs += [ "..:no_global_constructors" ] + sources = [ + "objc/api/peerconnection/RTCMediaConstraints+Private.h", + "objc/api/peerconnection/RTCMediaConstraints.h", + "objc/api/peerconnection/RTCMediaConstraints.mm", + ] + + public_configs = [ ":common_config_objc" ] + deps = [ + ":base_objc", + ":helpers_objc", + ":media_constraints", + ] + } + + # TODO(bugs.webrtc.org/9627): Remove, targets should depend on base_objc. + rtc_library("videorenderer_objc") { + visibility = [ "*" ] + sources = [ "objc/helpers/noop.mm" ] + + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + + deps = [ ":base_objc" ] + } + + rtc_library("videorendereradapter_objc") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + sources = [ + "objc/api/RTCVideoRendererAdapter+Private.h", + "objc/api/RTCVideoRendererAdapter.h", + "objc/api/RTCVideoRendererAdapter.mm", + ] + + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":native_api", + ":videoframebuffer_objc", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + ] + } + + rtc_library("mediasource_objc") { + sources = [ + "objc/api/peerconnection/RTCMediaSource+Private.h", + "objc/api/peerconnection/RTCMediaSource.h", + "objc/api/peerconnection/RTCMediaSource.mm", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + "../api:media_stream_interface", + "../rtc_base:checks", + ] + } + + rtc_library("base_native_additions_objc") { + sources = [ + "objc/api/peerconnection/RTCEncodedImage+Private.h", + "objc/api/peerconnection/RTCEncodedImage+Private.mm", + "objc/api/peerconnection/RTCVideoCodecInfo+Private.h", + "objc/api/peerconnection/RTCVideoCodecInfo+Private.mm", + "objc/api/peerconnection/RTCVideoEncoderSettings+Private.h", + "objc/api/peerconnection/RTCVideoEncoderSettings+Private.mm", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + "../api/video:encoded_image", + "../api/video_codecs:video_codecs_api", + "../modules/video_coding:video_codec_interface", + "../rtc_base:refcount", + "../rtc_base:safe_conversions", + ] + } + + rtc_library("peerconnectionfactory_base_objc") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", # TODO(bugs.webrtc.org/8396): Remove. + "default_task_queue", + ] + configs += [ "..:no_global_constructors" ] + sources = [ + "objc/api/peerconnection/RTCAudioSource+Private.h", + "objc/api/peerconnection/RTCAudioSource.h", + "objc/api/peerconnection/RTCAudioSource.mm", + "objc/api/peerconnection/RTCAudioTrack+Private.h", + "objc/api/peerconnection/RTCAudioTrack.h", + "objc/api/peerconnection/RTCAudioTrack.mm", + "objc/api/peerconnection/RTCCertificate.h", + "objc/api/peerconnection/RTCCertificate.mm", + "objc/api/peerconnection/RTCConfiguration+Native.h", + "objc/api/peerconnection/RTCConfiguration+Private.h", + "objc/api/peerconnection/RTCConfiguration.h", + "objc/api/peerconnection/RTCConfiguration.mm", + "objc/api/peerconnection/RTCCryptoOptions.h", + "objc/api/peerconnection/RTCCryptoOptions.mm", + "objc/api/peerconnection/RTCDataChannel+Private.h", + "objc/api/peerconnection/RTCDataChannel.h", + "objc/api/peerconnection/RTCDataChannel.mm", + "objc/api/peerconnection/RTCDataChannelConfiguration+Private.h", + "objc/api/peerconnection/RTCDataChannelConfiguration.h", + "objc/api/peerconnection/RTCDataChannelConfiguration.mm", + "objc/api/peerconnection/RTCDtmfSender+Private.h", + "objc/api/peerconnection/RTCDtmfSender.h", + "objc/api/peerconnection/RTCDtmfSender.mm", + "objc/api/peerconnection/RTCFieldTrials.h", + "objc/api/peerconnection/RTCFieldTrials.mm", + "objc/api/peerconnection/RTCIceCandidate+Private.h", + "objc/api/peerconnection/RTCIceCandidate.h", + "objc/api/peerconnection/RTCIceCandidate.mm", + "objc/api/peerconnection/RTCIceCandidateErrorEvent+Private.h", + "objc/api/peerconnection/RTCIceCandidateErrorEvent.h", + "objc/api/peerconnection/RTCIceCandidateErrorEvent.mm", + "objc/api/peerconnection/RTCIceServer+Private.h", + "objc/api/peerconnection/RTCIceServer.h", + "objc/api/peerconnection/RTCIceServer.mm", + "objc/api/peerconnection/RTCLegacyStatsReport+Private.h", + "objc/api/peerconnection/RTCLegacyStatsReport.h", + "objc/api/peerconnection/RTCLegacyStatsReport.mm", + "objc/api/peerconnection/RTCMediaStream+Private.h", + "objc/api/peerconnection/RTCMediaStream.h", + "objc/api/peerconnection/RTCMediaStream.mm", + "objc/api/peerconnection/RTCMediaStreamTrack+Private.h", + "objc/api/peerconnection/RTCMediaStreamTrack.h", + "objc/api/peerconnection/RTCMediaStreamTrack.mm", + "objc/api/peerconnection/RTCMetrics.h", + "objc/api/peerconnection/RTCMetrics.mm", + "objc/api/peerconnection/RTCMetricsSampleInfo+Private.h", + "objc/api/peerconnection/RTCMetricsSampleInfo.h", + "objc/api/peerconnection/RTCMetricsSampleInfo.mm", + "objc/api/peerconnection/RTCPeerConnection+DataChannel.mm", + "objc/api/peerconnection/RTCPeerConnection+Private.h", + "objc/api/peerconnection/RTCPeerConnection+Stats.mm", + "objc/api/peerconnection/RTCPeerConnection.h", + "objc/api/peerconnection/RTCPeerConnection.mm", + "objc/api/peerconnection/RTCPeerConnectionFactory+Native.h", + "objc/api/peerconnection/RTCPeerConnectionFactory+Private.h", + "objc/api/peerconnection/RTCPeerConnectionFactory.h", + "objc/api/peerconnection/RTCPeerConnectionFactory.mm", + "objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm", + "objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm", + "objc/api/peerconnection/RTCPeerConnectionFactoryOptions+Private.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryOptions.mm", + "objc/api/peerconnection/RTCRtcpParameters+Private.h", + "objc/api/peerconnection/RTCRtcpParameters.h", + "objc/api/peerconnection/RTCRtcpParameters.mm", + "objc/api/peerconnection/RTCRtpCodecParameters+Private.h", + "objc/api/peerconnection/RTCRtpCodecParameters.h", + "objc/api/peerconnection/RTCRtpCodecParameters.mm", + "objc/api/peerconnection/RTCRtpEncodingParameters+Private.h", + "objc/api/peerconnection/RTCRtpEncodingParameters.h", + "objc/api/peerconnection/RTCRtpEncodingParameters.mm", + "objc/api/peerconnection/RTCRtpHeaderExtension+Private.h", + "objc/api/peerconnection/RTCRtpHeaderExtension.h", + "objc/api/peerconnection/RTCRtpHeaderExtension.mm", + "objc/api/peerconnection/RTCRtpParameters+Private.h", + "objc/api/peerconnection/RTCRtpParameters.h", + "objc/api/peerconnection/RTCRtpParameters.mm", + "objc/api/peerconnection/RTCRtpReceiver+Native.h", + "objc/api/peerconnection/RTCRtpReceiver+Private.h", + "objc/api/peerconnection/RTCRtpReceiver.h", + "objc/api/peerconnection/RTCRtpReceiver.mm", + "objc/api/peerconnection/RTCRtpSender+Native.h", + "objc/api/peerconnection/RTCRtpSender+Private.h", + "objc/api/peerconnection/RTCRtpSender.h", + "objc/api/peerconnection/RTCRtpSender.mm", + "objc/api/peerconnection/RTCRtpTransceiver+Private.h", + "objc/api/peerconnection/RTCRtpTransceiver.h", + "objc/api/peerconnection/RTCRtpTransceiver.mm", + "objc/api/peerconnection/RTCSSLAdapter.h", + "objc/api/peerconnection/RTCSSLAdapter.mm", + "objc/api/peerconnection/RTCSessionDescription+Private.h", + "objc/api/peerconnection/RTCSessionDescription.h", + "objc/api/peerconnection/RTCSessionDescription.mm", + "objc/api/peerconnection/RTCStatisticsReport+Private.h", + "objc/api/peerconnection/RTCStatisticsReport.h", + "objc/api/peerconnection/RTCStatisticsReport.mm", + "objc/api/peerconnection/RTCTracing.h", + "objc/api/peerconnection/RTCTracing.mm", + "objc/api/peerconnection/RTCVideoTrack+Private.h", + "objc/api/peerconnection/RTCVideoTrack.h", + "objc/api/peerconnection/RTCVideoTrack.mm", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":audio_device_api_objc", + ":base_native_additions_objc", + ":base_objc", + ":file_logger_objc", + ":helpers_objc", + ":mediaconstraints_objc", + ":mediasource_objc", + ":native_api", + ":native_video", + ":objc_audio_device_module", + ":videoframebuffer_objc", + ":videorendereradapter_objc", + ":videosource_objc", + ":videotoolbox_objc", + "../api:dtmf_sender_interface", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + "../api:rtc_event_log_output_file", + "../api:rtc_stats_api", + "../api:rtp_parameters", + "../api:rtp_sender_interface", + "../api:scoped_refptr", + "../api/audio_codecs:audio_codecs_api", + "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/crypto:frame_decryptor_interface", + "../api/crypto:frame_encryptor_interface", + "../api/rtc_event_log:rtc_event_log_factory", + "../api/task_queue:default_task_queue_factory", + "../api/transport:field_trial_based_config", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../media:media_constants", + "../media:rtc_audio_video", + "../media:rtc_media_base", + "../modules/audio_device:audio_device_api", + "../modules/audio_processing", + "../modules/audio_processing:api", + "../modules/video_coding:video_codec_interface", + "../pc:peer_connection_factory", + "../pc:webrtc_sdp", + "../rtc_base:checks", + "../rtc_base:event_tracer", + "../rtc_base:logging", + "../rtc_base:network_constants", + "../rtc_base:rtc_certificate_generator", + "../rtc_base:safe_conversions", + "../rtc_base:ssl", + "../rtc_base:stringutils", + "../rtc_base:threading", + "../rtc_base:timeutils", + "../stats:rtc_stats", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + ] + + if (is_ios) { + deps += [ ":native_api_audio_device_module" ] + } + } + + if (rtc_include_tests) { + if (is_ios) { + rtc_library("sdk_unittests_sources") { + testonly = true + include_dirs = [ "objc/" ] + + sources = [ + "objc/unittests/ObjCVideoTrackSource_xctest.mm", + "objc/unittests/RTCAudioDeviceModule_xctest.mm", + "objc/unittests/RTCAudioDevice_xctest.mm", + "objc/unittests/RTCAudioSessionTest.mm", + "objc/unittests/RTCCVPixelBuffer_xctest.mm", + "objc/unittests/RTCCallbackLogger_xctest.m", + "objc/unittests/RTCCameraVideoCapturerTests.mm", + "objc/unittests/RTCCertificateTest.mm", + "objc/unittests/RTCConfigurationTest.mm", + "objc/unittests/RTCDataChannelConfigurationTest.mm", + "objc/unittests/RTCEncodedImage_xctest.mm", + "objc/unittests/RTCFileVideoCapturer_xctest.mm", + "objc/unittests/RTCH264ProfileLevelId_xctest.m", + "objc/unittests/RTCIceCandidateTest.mm", + "objc/unittests/RTCIceServerTest.mm", + "objc/unittests/RTCMTLVideoView_xctest.m", + "objc/unittests/RTCMediaConstraintsTest.mm", + "objc/unittests/RTCNV12TextureCache_xctest.m", + "objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm", + "objc/unittests/RTCPeerConnectionFactory_xctest.m", + "objc/unittests/RTCPeerConnectionTest.mm", + "objc/unittests/RTCSessionDescriptionTest.mm", + "objc/unittests/RTCTracingTest.mm", + "objc/unittests/frame_buffer_helpers.h", + "objc/unittests/frame_buffer_helpers.mm", + "objc/unittests/nalu_rewriter_xctest.mm", + "objc/unittests/objc_video_decoder_factory_tests.mm", + "objc/unittests/objc_video_encoder_factory_tests.mm", + "objc/unittests/scoped_cftyperef_tests.mm", + ] + + # TODO(bugs.webrtc.org/12937): Remove OpenGL deprecation warning + # workaround. + defines = [ "GLES_SILENCE_DEPRECATION" ] + + deps = [ + ":audio_device", + ":audio_session_objc", + ":base_native_additions_objc", + ":base_objc", + ":callback_logger_objc", + ":framework_objc", + ":helpers_objc", + ":mediaconstraints_objc", + ":metal_objc", + ":native_api", + ":native_api_audio_device_module", + ":native_video", + ":peerconnectionfactory_base_objc", + ":video_toolbox_cc", + ":videocapture_objc", + ":videocodec_objc", + ":videoframebuffer_objc", + ":videosource_objc", + ":videotoolbox_objc", + "../api:scoped_refptr", + "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/task_queue:default_task_queue_factory", + "../api/video:video_frame", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../media:codec", + "../media:rtc_media_base", + "../media:rtc_media_tests_utils", + "../modules/audio_device:audio_device_api", + "../modules/audio_processing:api", + "../modules/video_coding:video_codec_interface", + "../rtc_base:gunit_helpers", + "../rtc_base:macromagic", + "../rtc_base:refcount", + "../rtc_base:rtc_event", + "../rtc_base/system:unused", + "../system_wrappers", + "//third_party/libyuv", + ] + + if (rtc_ios_use_opengl_rendering) { + deps += [ ":opengl_objc" ] + } + + public_deps = [ + "//build/config/ios:xctest", + "//third_party/ocmock", + ] + } + + bundle_data("sdk_unittests_bundle_data") { + sources = [ + "objc/unittests/audio_short16.pcm", + "objc/unittests/audio_short44.pcm", + "objc/unittests/audio_short48.pcm", + + # Sample video taken from https://media.xiph.org/video/derf/ + "objc/unittests/foreman.mp4", + ] + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + + # These tests use static linking. + rtc_test("sdk_unittests") { + is_xctest = true + info_plist = "//test/ios/Info.plist" + sources = [ "objc/unittests/main.mm" ] + + extra_substitutions = [ "GTEST_BUNDLE_ID_SUFFIX=generic-unit-test" ] + deps = [ + ":peerconnectionfactory_base_objc", + ":sdk_unittests_bundle_data", + ":sdk_unittests_sources", + "../rtc_base:threading", + "//test:test_support", + ] + ldflags = [ "-all_load" ] + } + + # These tests link to the framework. + rtc_test("sdk_framework_unittests") { + is_xctest = true + info_plist = "//test/ios/Info.plist" + sources = [ + "objc/unittests/RTCDoNotPutCPlusPlusInFrameworkHeaders_xctest.m", + "objc/unittests/main.mm", + ] + + extra_substitutions = [ "GTEST_BUNDLE_ID_SUFFIX=generic-unit-test" ] + deps = [ + ":framework_objc+link", + ":ios_framework_bundle", + "../rtc_base:threading", + "//test:test_support", + ] + } + } + } + + if (is_ios) { + apple_framework_bundle_with_umbrella_header("framework_objc") { + info_plist = "objc/Info.plist" + output_name = "WebRTC" + + common_objc_headers = [ + "objc/base/RTCCodecSpecificInfo.h", + "objc/base/RTCEncodedImage.h", + "objc/base/RTCI420Buffer.h", + "objc/base/RTCLogging.h", + "objc/base/RTCMacros.h", + "objc/base/RTCMutableI420Buffer.h", + "objc/base/RTCMutableYUVPlanarBuffer.h", + "objc/base/RTCSSLCertificateVerifier.h", + "objc/base/RTCVideoCapturer.h", + "objc/base/RTCVideoCodecInfo.h", + "objc/base/RTCVideoDecoder.h", + "objc/base/RTCVideoDecoderFactory.h", + "objc/base/RTCVideoEncoder.h", + "objc/base/RTCVideoEncoderFactory.h", + "objc/base/RTCVideoEncoderQpThresholds.h", + "objc/base/RTCVideoEncoderSettings.h", + "objc/base/RTCVideoFrame.h", + "objc/base/RTCVideoFrameBuffer.h", + "objc/base/RTCVideoRenderer.h", + "objc/base/RTCYUVPlanarBuffer.h", + "objc/components/audio/RTCAudioDevice.h", + "objc/components/audio/RTCAudioSession.h", + "objc/components/audio/RTCAudioSessionConfiguration.h", + "objc/components/capturer/RTCCameraVideoCapturer.h", + "objc/components/capturer/RTCFileVideoCapturer.h", + "objc/components/network/RTCNetworkMonitor.h", + "objc/components/renderer/metal/RTCMTLVideoView.h", + "objc/components/renderer/opengl/RTCEAGLVideoView.h", + "objc/components/renderer/opengl/RTCVideoViewShading.h", + "objc/components/video_codec/RTCCodecSpecificInfoH264.h", + "objc/components/video_codec/RTCDefaultVideoDecoderFactory.h", + "objc/components/video_codec/RTCDefaultVideoEncoderFactory.h", + "objc/components/video_codec/RTCH264ProfileLevelId.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH264.h", + "objc/components/video_codec/RTCVideoDecoderH264.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH264.h", + "objc/components/video_codec/RTCVideoEncoderH264.h", + "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", + "objc/helpers/RTCCameraPreviewView.h", + "objc/helpers/RTCDispatcher.h", + "objc/helpers/UIDevice+RTCDevice.h", + "objc/api/peerconnection/RTCAudioSource.h", + "objc/api/peerconnection/RTCAudioTrack.h", + "objc/api/peerconnection/RTCConfiguration.h", + "objc/api/peerconnection/RTCDataChannel.h", + "objc/api/peerconnection/RTCDataChannelConfiguration.h", + "objc/api/peerconnection/RTCFieldTrials.h", + "objc/api/peerconnection/RTCIceCandidate.h", + "objc/api/peerconnection/RTCIceCandidateErrorEvent.h", + "objc/api/peerconnection/RTCIceServer.h", + "objc/api/peerconnection/RTCLegacyStatsReport.h", + "objc/api/peerconnection/RTCMediaConstraints.h", + "objc/api/peerconnection/RTCMediaSource.h", + "objc/api/peerconnection/RTCMediaStream.h", + "objc/api/peerconnection/RTCMediaStreamTrack.h", + "objc/api/peerconnection/RTCMetrics.h", + "objc/api/peerconnection/RTCMetricsSampleInfo.h", + "objc/api/peerconnection/RTCPeerConnection.h", + "objc/api/peerconnection/RTCPeerConnectionFactory.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h", + "objc/api/peerconnection/RTCRtcpParameters.h", + "objc/api/peerconnection/RTCRtpCodecParameters.h", + "objc/api/peerconnection/RTCRtpEncodingParameters.h", + "objc/api/peerconnection/RTCRtpHeaderExtension.h", + "objc/api/peerconnection/RTCRtpParameters.h", + "objc/api/peerconnection/RTCRtpReceiver.h", + "objc/api/peerconnection/RTCRtpSender.h", + "objc/api/peerconnection/RTCRtpTransceiver.h", + "objc/api/peerconnection/RTCDtmfSender.h", + "objc/api/peerconnection/RTCSSLAdapter.h", + "objc/api/peerconnection/RTCSessionDescription.h", + "objc/api/peerconnection/RTCStatisticsReport.h", + "objc/api/peerconnection/RTCTracing.h", + "objc/api/peerconnection/RTCCertificate.h", + "objc/api/peerconnection/RTCCryptoOptions.h", + "objc/api/peerconnection/RTCVideoSource.h", + "objc/api/peerconnection/RTCVideoTrack.h", + "objc/api/video_codec/RTCVideoCodecConstants.h", + "objc/api/video_codec/RTCVideoDecoderVP8.h", + "objc/api/video_codec/RTCVideoDecoderVP9.h", + "objc/api/video_codec/RTCVideoDecoderAV1.h", + "objc/api/video_codec/RTCVideoEncoderVP8.h", + "objc/api/video_codec/RTCVideoEncoderVP9.h", + "objc/api/video_codec/RTCVideoEncoderAV1.h", + "objc/api/video_frame_buffer/RTCNativeI420Buffer.h", + "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h", + ] + + if (!build_with_chromium) { + common_objc_headers += [ + "objc/api/logging/RTCCallbackLogger.h", + "objc/api/peerconnection/RTCFileLogger.h", + ] + } + + sources = common_objc_headers + public_headers = common_objc_headers + + ldflags = [ + "-all_load", + "-install_name", + "@rpath/$output_name.framework/$output_name", + ] + + deps = [ + ":audio_objc", + ":base_objc", + ":default_codec_factory_objc", + ":metal_objc", + ":native_api", + ":native_video", + ":peerconnectionfactory_base_objc", + ":videocapture_objc", + ":videocodec_objc", + ":videotoolbox_objc", + ] + if (!build_with_chromium) { + deps += [ + ":callback_logger_objc", + ":file_logger_objc", + ] + } + + frameworks = [ + "AVFoundation.framework", + "CoreGraphics.framework", + "CoreMedia.framework", + "Foundation.framework", + "UIKit.framework", + ] + + configs = [ + "..:common_objc", + ":used_from_extension", + ] + + public_configs = [ ":common_config_objc" ] + } + + bundle_data("ios_framework_bundle") { + deps = [ "../sdk:framework_objc" ] + sources = [ "$root_build_dir/WebRTC.framework" ] + outputs = [ "{{bundle_resources_dir}}/Frameworks/{{source_file_part}}" ] + } + } + + if (is_mac) { + apple_framework_bundle_with_umbrella_header("mac_framework_objc") { + info_plist = "objc/Info.plist" + output_name = "WebRTC" + + sources = [ + "objc/api/peerconnection/RTCAudioSource.h", + "objc/api/peerconnection/RTCAudioTrack.h", + "objc/api/peerconnection/RTCCertificate.h", + "objc/api/peerconnection/RTCConfiguration.h", + "objc/api/peerconnection/RTCCryptoOptions.h", + "objc/api/peerconnection/RTCDataChannel.h", + "objc/api/peerconnection/RTCDataChannelConfiguration.h", + "objc/api/peerconnection/RTCDtmfSender.h", + "objc/api/peerconnection/RTCFieldTrials.h", + "objc/api/peerconnection/RTCIceCandidate.h", + "objc/api/peerconnection/RTCIceCandidateErrorEvent.h", + "objc/api/peerconnection/RTCIceServer.h", + "objc/api/peerconnection/RTCLegacyStatsReport.h", + "objc/api/peerconnection/RTCMediaConstraints.h", + "objc/api/peerconnection/RTCMediaSource.h", + "objc/api/peerconnection/RTCMediaStream.h", + "objc/api/peerconnection/RTCMediaStreamTrack.h", + "objc/api/peerconnection/RTCMetrics.h", + "objc/api/peerconnection/RTCMetricsSampleInfo.h", + "objc/api/peerconnection/RTCPeerConnection.h", + "objc/api/peerconnection/RTCPeerConnectionFactory.h", + "objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h", + "objc/api/peerconnection/RTCRtcpParameters.h", + "objc/api/peerconnection/RTCRtpCodecParameters.h", + "objc/api/peerconnection/RTCRtpEncodingParameters.h", + "objc/api/peerconnection/RTCRtpHeaderExtension.h", + "objc/api/peerconnection/RTCRtpParameters.h", + "objc/api/peerconnection/RTCRtpReceiver.h", + "objc/api/peerconnection/RTCRtpSender.h", + "objc/api/peerconnection/RTCRtpTransceiver.h", + "objc/api/peerconnection/RTCSSLAdapter.h", + "objc/api/peerconnection/RTCSessionDescription.h", + "objc/api/peerconnection/RTCStatisticsReport.h", + "objc/api/peerconnection/RTCTracing.h", + "objc/api/peerconnection/RTCVideoSource.h", + "objc/api/peerconnection/RTCVideoTrack.h", + "objc/api/video_codec/RTCVideoDecoderAV1.h", + "objc/api/video_codec/RTCVideoDecoderVP8.h", + "objc/api/video_codec/RTCVideoDecoderVP9.h", + "objc/api/video_codec/RTCVideoEncoderAV1.h", + "objc/api/video_codec/RTCVideoEncoderVP8.h", + "objc/api/video_codec/RTCVideoEncoderVP9.h", + "objc/api/video_frame_buffer/RTCNativeI420Buffer.h", + "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h", + "objc/base/RTCCodecSpecificInfo.h", + "objc/base/RTCEncodedImage.h", + "objc/base/RTCI420Buffer.h", + "objc/base/RTCLogging.h", + "objc/base/RTCMacros.h", + "objc/base/RTCMutableI420Buffer.h", + "objc/base/RTCMutableYUVPlanarBuffer.h", + "objc/base/RTCSSLCertificateVerifier.h", + "objc/base/RTCVideoCapturer.h", + "objc/base/RTCVideoCodecInfo.h", + "objc/base/RTCVideoDecoder.h", + "objc/base/RTCVideoDecoderFactory.h", + "objc/base/RTCVideoEncoder.h", + "objc/base/RTCVideoEncoderFactory.h", + "objc/base/RTCVideoEncoderQpThresholds.h", + "objc/base/RTCVideoEncoderSettings.h", + "objc/base/RTCVideoFrame.h", + "objc/base/RTCVideoFrameBuffer.h", + "objc/base/RTCVideoRenderer.h", + "objc/base/RTCYUVPlanarBuffer.h", + "objc/components/capturer/RTCCameraVideoCapturer.h", + "objc/components/capturer/RTCFileVideoCapturer.h", + "objc/components/renderer/metal/RTCMTLNSVideoView.h", + "objc/components/renderer/opengl/RTCVideoViewShading.h", + "objc/components/video_codec/RTCCodecSpecificInfoH264.h", + "objc/components/video_codec/RTCDefaultVideoDecoderFactory.h", + "objc/components/video_codec/RTCDefaultVideoEncoderFactory.h", + "objc/components/video_codec/RTCH264ProfileLevelId.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH264.h", + "objc/components/video_codec/RTCVideoDecoderH264.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH264.h", + "objc/components/video_codec/RTCVideoEncoderH264.h", + "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", + "objc/helpers/RTCDispatcher.h", + ] + if (!build_with_chromium) { + sources += [ + "objc/api/logging/RTCCallbackLogger.h", + "objc/api/peerconnection/RTCFileLogger.h", + ] + } + + deps = [ + ":base_objc", + ":default_codec_factory_objc", + ":native_api", + ":native_video", + ":peerconnectionfactory_base_objc", + ":videocapture_objc", + ":videocodec_objc", + ":videotoolbox_objc", + ] + if (!build_with_chromium) { + deps += [ + ":callback_logger_objc", + ":file_logger_objc", + ] + } + + frameworks = [ + "AVFoundation.framework", + "CoreGraphics.framework", + "CoreMedia.framework", + "OpenGL.framework", + ] + + configs = [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + } + + bundle_data("mac_framework_bundle") { + deps = [ "../sdk:mac_framework_objc" ] + sources = [ "$root_build_dir/WebRTC.framework" ] + outputs = [ "{{bundle_contents_dir}}/Frameworks/{{source_file_part}}" ] + } + } + + rtc_library("wrapped_native_codec_objc") { + sources = [ + "objc/api/video_codec/RTCWrappedNativeVideoDecoder.h", + "objc/api/video_codec/RTCWrappedNativeVideoDecoder.mm", + "objc/api/video_codec/RTCWrappedNativeVideoEncoder.h", + "objc/api/video_codec/RTCWrappedNativeVideoEncoder.mm", + ] + + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":helpers_objc", + "../api/video_codecs:video_codecs_api", + "../media:codec", + "../media:rtc_media_base", + ] + } + + # The native API is currently experimental and may change without notice. + rtc_library("native_api") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + sources = [ + "objc/native/api/network_monitor_factory.h", + "objc/native/api/network_monitor_factory.mm", + "objc/native/api/ssl_certificate_verifier.h", + "objc/native/api/ssl_certificate_verifier.mm", + "objc/native/api/video_capturer.h", + "objc/native/api/video_capturer.mm", + "objc/native/api/video_decoder_factory.h", + "objc/native/api/video_decoder_factory.mm", + "objc/native/api/video_encoder_factory.h", + "objc/native/api/video_encoder_factory.mm", + "objc/native/api/video_frame.h", + "objc/native/api/video_frame.mm", + "objc/native/api/video_frame_buffer.h", + "objc/native/api/video_frame_buffer.mm", + "objc/native/api/video_renderer.h", + "objc/native/api/video_renderer.mm", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":native_video", + ":videoframebuffer_objc", + "../api:libjingle_peerconnection_api", + "../api:make_ref_counted", + "../api:media_stream_interface", + "../api:scoped_refptr", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../pc:video_track_source_proxy", + "../rtc_base:buffer", + "../rtc_base:logging", + "../rtc_base:ssl", + "../rtc_base:threading", + ] + if (is_ios) { + deps += [ ":native_network_monitor" ] + } + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] + } + + if (is_ios) { + rtc_library("native_network_monitor") { + visibility = [ "*" ] + + sources = [ + "objc/native/src/objc_network_monitor.h", + "objc/native/src/objc_network_monitor.mm", + ] + + deps = [ + ":network_monitor_objc", + ":network_monitor_observer", + "../api:field_trials_view", + "../api:sequence_checker", + "../api/task_queue:pending_task_safety_flag", + "../rtc_base:logging", + "../rtc_base:macromagic", + "../rtc_base:stringutils", + "../rtc_base:threading", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + } + + rtc_library("native_video") { + sources = [ + "objc/native/src/objc_frame_buffer.h", + "objc/native/src/objc_frame_buffer.mm", + "objc/native/src/objc_video_decoder_factory.h", + "objc/native/src/objc_video_decoder_factory.mm", + "objc/native/src/objc_video_encoder_factory.h", + "objc/native/src/objc_video_encoder_factory.mm", + "objc/native/src/objc_video_frame.h", + "objc/native/src/objc_video_frame.mm", + "objc/native/src/objc_video_renderer.h", + "objc/native/src/objc_video_renderer.mm", + "objc/native/src/objc_video_track_source.h", + "objc/native/src/objc_video_track_source.mm", + ] + + configs += [ "..:common_objc" ] + + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_native_additions_objc", + ":base_objc", + ":helpers_objc", + ":videocodec_objc", + ":videoframebuffer_objc", + ":vpx_codec_constants", + ":wrapped_native_codec_objc", + "../api:make_ref_counted", + "../api/video:video_frame", + "../api/video:video_rtp_headers", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../media:codec", + "../media:rtc_audio_video", + "../media:rtc_media_base", + "../modules/video_coding:video_codec_interface", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:timestamp_aligner", + "../rtc_base:timeutils", + ] + } + + rtc_library("video_toolbox_cc") { + visibility = [ + ":sdk_unittests_sources", + ":videotoolbox_objc", + ] + sources = [ + "objc/components/video_codec/helpers.cc", + "objc/components/video_codec/helpers.h", + "objc/components/video_codec/nalu_rewriter.cc", + "objc/components/video_codec/nalu_rewriter.h", + ] + deps = [ + "../common_video", + "../modules/video_coding:webrtc_h264", + "../rtc_base:buffer", + "../rtc_base:checks", + "../rtc_base:logging", + ] + } + + rtc_library("videotoolbox_objc") { + visibility = [ "*" ] + allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove. + sources = [ + "objc/components/video_codec/RTCVideoDecoderFactoryH264.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH264.m", + "objc/components/video_codec/RTCVideoDecoderH264.h", + "objc/components/video_codec/RTCVideoDecoderH264.mm", + "objc/components/video_codec/RTCVideoEncoderFactoryH264.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH264.m", + "objc/components/video_codec/RTCVideoEncoderH264.h", + "objc/components/video_codec/RTCVideoEncoderH264.mm", + ] + + configs += [ + "..:common_objc", + ":used_from_extension", + ] + + if (is_ios && rtc_apprtcmobile_broadcast_extension) { + defines = [ "RTC_APPRTCMOBILE_BROADCAST_EXTENSION" ] + } + + deps = [ + ":base_native_additions_objc", + ":base_objc", + ":helpers_objc", + ":video_toolbox_cc", + ":videocodec_objc", + ":videoframebuffer_objc", + "../api/video_codecs:video_codecs_api", + "../common_video", + "../modules/video_coding:video_codec_interface", + "../rtc_base:buffer", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:timeutils", + "//third_party/libyuv", + ] + + frameworks = [ + "CoreFoundation.framework", + "CoreMedia.framework", + "CoreVideo.framework", + "VideoToolbox.framework", + ] + } + } + } +} diff --git a/third_party/libwebrtc/sdk/OWNERS b/third_party/libwebrtc/sdk/OWNERS new file mode 100644 index 0000000000..4d31ffb663 --- /dev/null +++ b/third_party/libwebrtc/sdk/OWNERS @@ -0,0 +1 @@ +magjed@webrtc.org diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AddIceObserver.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AddIceObserver.java new file mode 100644 index 0000000000..ff2c690029 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AddIceObserver.java @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Interface to handle completion of addIceCandidate */ +public interface AddIceObserver { + /** Called when ICE candidate added successfully.*/ + @CalledByNative public void onAddSuccess(); + + /** Called when ICE candidate addition failed.*/ + @CalledByNative public void onAddFailure(String error); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java new file mode 100644 index 0000000000..dd3e262896 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java @@ -0,0 +1,21 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Implementations of this interface can create a native {@code webrtc::AudioDecoderFactory}. + */ +public interface AudioDecoderFactoryFactory { + /** + * Returns a pointer to a {@code webrtc::AudioDecoderFactory}. The caller takes ownership. + */ + long createNativeAudioDecoderFactory(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java new file mode 100644 index 0000000000..814b71aba1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java @@ -0,0 +1,21 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Implementations of this interface can create a native {@code webrtc::AudioEncoderFactory}. + */ +public interface AudioEncoderFactoryFactory { + /** + * Returns a pointer to a {@code webrtc::AudioEncoderFactory}. The caller takes ownership. + */ + long createNativeAudioEncoderFactory(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioProcessingFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioProcessingFactory.java new file mode 100644 index 0000000000..bd8fdb8989 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioProcessingFactory.java @@ -0,0 +1,20 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Factory for creating webrtc::AudioProcessing instances. */ +public interface AudioProcessingFactory { + /** + * Dynamically allocates a webrtc::AudioProcessing instance and returns a pointer to it. + * The caller takes ownership of the object. + */ + public long createNative(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioSource.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioSource.java new file mode 100644 index 0000000000..f8104e5904 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioSource.java @@ -0,0 +1,26 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Java wrapper for a C++ AudioSourceInterface. Used as the source for one or + * more {@code AudioTrack} objects. + */ +public class AudioSource extends MediaSource { + public AudioSource(long nativeSource) { + super(nativeSource); + } + + /** Returns a pointer to webrtc::AudioSourceInterface. */ + long getNativeAudioSource() { + return getNativeMediaSource(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioTrack.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioTrack.java new file mode 100644 index 0000000000..ca745db634 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/AudioTrack.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Java wrapper for a C++ AudioTrackInterface */ +public class AudioTrack extends MediaStreamTrack { + public AudioTrack(long nativeTrack) { + super(nativeTrack); + } + + /** Sets the volume for the underlying MediaSource. Volume is a gain value in the range + * 0 to 10. + */ + public void setVolume(double volume) { + nativeSetVolume(getNativeAudioTrack(), volume); + } + + /** Returns a pointer to webrtc::AudioTrackInterface. */ + long getNativeAudioTrack() { + return getNativeMediaStreamTrack(); + } + + private static native void nativeSetVolume(long track, double volume); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java new file mode 100644 index 0000000000..5ebc19f25d --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Creates a native {@code webrtc::AudioDecoderFactory} with the builtin audio decoders. + */ +public class BuiltinAudioDecoderFactoryFactory implements AudioDecoderFactoryFactory { + @Override + public long createNativeAudioDecoderFactory() { + return nativeCreateBuiltinAudioDecoderFactory(); + } + + private static native long nativeCreateBuiltinAudioDecoderFactory(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java new file mode 100644 index 0000000000..e884d4c3b9 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * This class creates a native {@code webrtc::AudioEncoderFactory} with the builtin audio encoders. + */ +public class BuiltinAudioEncoderFactoryFactory implements AudioEncoderFactoryFactory { + @Override + public long createNativeAudioEncoderFactory() { + return nativeCreateBuiltinAudioEncoderFactory(); + } + + private static native long nativeCreateBuiltinAudioEncoderFactory(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java new file mode 100644 index 0000000000..f4edb58847 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class CallSessionFileRotatingLogSink { + private long nativeSink; + + public static byte[] getLogData(String dirPath) { + if (dirPath == null) { + throw new IllegalArgumentException("dirPath may not be null."); + } + return nativeGetLogData(dirPath); + } + + public CallSessionFileRotatingLogSink( + String dirPath, int maxFileSize, Logging.Severity severity) { + if (dirPath == null) { + throw new IllegalArgumentException("dirPath may not be null."); + } + nativeSink = nativeAddSink(dirPath, maxFileSize, severity.ordinal()); + } + + public void dispose() { + if (nativeSink != 0) { + nativeDeleteSink(nativeSink); + nativeSink = 0; + } + } + + private static native long nativeAddSink(String dirPath, int maxFileSize, int severity); + private static native void nativeDeleteSink(long sink); + private static native byte[] nativeGetLogData(String dirPath); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Capturer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Capturer.java new file mode 100644 index 0000000000..de172aa1d7 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Capturer.java @@ -0,0 +1,33 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; + +public class Camera1Capturer extends CameraCapturer { + private final boolean captureToTexture; + + public Camera1Capturer( + String cameraName, CameraEventsHandler eventsHandler, boolean captureToTexture) { + super(cameraName, eventsHandler, new Camera1Enumerator(captureToTexture)); + + this.captureToTexture = captureToTexture; + } + + @Override + protected void createCameraSession(CameraSession.CreateSessionCallback createSessionCallback, + CameraSession.Events events, Context applicationContext, + SurfaceTextureHelper surfaceTextureHelper, String cameraName, int width, int height, + int framerate) { + Camera1Session.create(createSessionCallback, events, captureToTexture, applicationContext, + surfaceTextureHelper, cameraName, width, height, framerate); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Enumerator.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Enumerator.java new file mode 100644 index 0000000000..4a1aacdb05 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera1Enumerator.java @@ -0,0 +1,190 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.os.SystemClock; +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; + +@SuppressWarnings("deprecation") +public class Camera1Enumerator implements CameraEnumerator { + private final static String TAG = "Camera1Enumerator"; + // Each entry contains the supported formats for corresponding camera index. The formats for all + // cameras are enumerated on the first call to getSupportedFormats(), and cached for future + // reference. + private static List<List<CaptureFormat>> cachedSupportedFormats; + + private final boolean captureToTexture; + + public Camera1Enumerator() { + this(true /* captureToTexture */); + } + + public Camera1Enumerator(boolean captureToTexture) { + this.captureToTexture = captureToTexture; + } + + // Returns device names that can be used to create a new VideoCapturerAndroid. + @Override + public String[] getDeviceNames() { + ArrayList<String> namesList = new ArrayList<>(); + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { + String name = getDeviceName(i); + if (name != null) { + namesList.add(name); + Logging.d(TAG, "Index: " + i + ". " + name); + } else { + Logging.e(TAG, "Index: " + i + ". Failed to query camera name."); + } + } + String[] namesArray = new String[namesList.size()]; + return namesList.toArray(namesArray); + } + + @Override + public boolean isFrontFacing(String deviceName) { + android.hardware.Camera.CameraInfo info = getCameraInfo(getCameraIndex(deviceName)); + return info != null && info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT; + } + + @Override + public boolean isBackFacing(String deviceName) { + android.hardware.Camera.CameraInfo info = getCameraInfo(getCameraIndex(deviceName)); + return info != null && info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK; + } + + @Override + public boolean isInfrared(String deviceName) { + return false; + } + + @Override + public List<CaptureFormat> getSupportedFormats(String deviceName) { + return getSupportedFormats(getCameraIndex(deviceName)); + } + + @Override + public CameraVideoCapturer createCapturer( + String deviceName, CameraVideoCapturer.CameraEventsHandler eventsHandler) { + return new Camera1Capturer(deviceName, eventsHandler, captureToTexture); + } + + private static @Nullable android.hardware.Camera.CameraInfo getCameraInfo(int index) { + android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo(); + try { + android.hardware.Camera.getCameraInfo(index, info); + } catch (Exception e) { + Logging.e(TAG, "getCameraInfo failed on index " + index, e); + return null; + } + return info; + } + + static synchronized List<CaptureFormat> getSupportedFormats(int cameraId) { + if (cachedSupportedFormats == null) { + cachedSupportedFormats = new ArrayList<List<CaptureFormat>>(); + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { + cachedSupportedFormats.add(enumerateFormats(i)); + } + } + return cachedSupportedFormats.get(cameraId); + } + + private static List<CaptureFormat> enumerateFormats(int cameraId) { + Logging.d(TAG, "Get supported formats for camera index " + cameraId + "."); + final long startTimeMs = SystemClock.elapsedRealtime(); + final android.hardware.Camera.Parameters parameters; + android.hardware.Camera camera = null; + try { + Logging.d(TAG, "Opening camera with index " + cameraId); + camera = android.hardware.Camera.open(cameraId); + parameters = camera.getParameters(); + } catch (RuntimeException e) { + Logging.e(TAG, "Open camera failed on camera index " + cameraId, e); + return new ArrayList<CaptureFormat>(); + } finally { + if (camera != null) { + camera.release(); + } + } + + final List<CaptureFormat> formatList = new ArrayList<CaptureFormat>(); + try { + int minFps = 0; + int maxFps = 0; + final List<int[]> listFpsRange = parameters.getSupportedPreviewFpsRange(); + if (listFpsRange != null) { + // getSupportedPreviewFpsRange() returns a sorted list. Take the fps range + // corresponding to the highest fps. + final int[] range = listFpsRange.get(listFpsRange.size() - 1); + minFps = range[android.hardware.Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; + maxFps = range[android.hardware.Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; + } + for (android.hardware.Camera.Size size : parameters.getSupportedPreviewSizes()) { + formatList.add(new CaptureFormat(size.width, size.height, minFps, maxFps)); + } + } catch (Exception e) { + Logging.e(TAG, "getSupportedFormats() failed on camera index " + cameraId, e); + } + + final long endTimeMs = SystemClock.elapsedRealtime(); + Logging.d(TAG, "Get supported formats for camera index " + cameraId + " done." + + " Time spent: " + (endTimeMs - startTimeMs) + " ms."); + return formatList; + } + + // Convert from android.hardware.Camera.Size to Size. + static List<Size> convertSizes(List<android.hardware.Camera.Size> cameraSizes) { + final List<Size> sizes = new ArrayList<Size>(); + for (android.hardware.Camera.Size size : cameraSizes) { + sizes.add(new Size(size.width, size.height)); + } + return sizes; + } + + // Convert from int[2] to CaptureFormat.FramerateRange. + static List<CaptureFormat.FramerateRange> convertFramerates(List<int[]> arrayRanges) { + final List<CaptureFormat.FramerateRange> ranges = new ArrayList<CaptureFormat.FramerateRange>(); + for (int[] range : arrayRanges) { + ranges.add(new CaptureFormat.FramerateRange( + range[android.hardware.Camera.Parameters.PREVIEW_FPS_MIN_INDEX], + range[android.hardware.Camera.Parameters.PREVIEW_FPS_MAX_INDEX])); + } + return ranges; + } + + // Returns the camera index for camera with name `deviceName`, or throws IllegalArgumentException + // if no such camera can be found. + static int getCameraIndex(String deviceName) { + Logging.d(TAG, "getCameraIndex: " + deviceName); + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { + if (deviceName.equals(getDeviceName(i))) { + return i; + } + } + throw new IllegalArgumentException("No such camera: " + deviceName); + } + + // Returns the name of the camera with camera index. Returns null if the + // camera can not be used. + static @Nullable String getDeviceName(int index) { + android.hardware.Camera.CameraInfo info = getCameraInfo(index); + if (info == null) { + return null; + } + + String facing = + (info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT) ? "front" : "back"; + return "Camera " + index + ", Facing " + facing + ", Orientation " + info.orientation; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Capturer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Capturer.java new file mode 100644 index 0000000000..c4becf4819 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Capturer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.hardware.camera2.CameraManager; +import androidx.annotation.Nullable; + +public class Camera2Capturer extends CameraCapturer { + private final Context context; + @Nullable private final CameraManager cameraManager; + + public Camera2Capturer(Context context, String cameraName, CameraEventsHandler eventsHandler) { + super(cameraName, eventsHandler, new Camera2Enumerator(context)); + + this.context = context; + cameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); + } + + @Override + protected void createCameraSession(CameraSession.CreateSessionCallback createSessionCallback, + CameraSession.Events events, Context applicationContext, + SurfaceTextureHelper surfaceTextureHelper, String cameraName, int width, int height, + int framerate) { + Camera2Session.create(createSessionCallback, events, applicationContext, cameraManager, + surfaceTextureHelper, cameraName, width, height, framerate); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Enumerator.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Enumerator.java new file mode 100644 index 0000000000..44e239ad8e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Camera2Enumerator.java @@ -0,0 +1,251 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.graphics.Rect; +import android.graphics.SurfaceTexture; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CameraMetadata; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.os.Build; +import android.os.SystemClock; +import android.util.Range; +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; + +public class Camera2Enumerator implements CameraEnumerator { + private final static String TAG = "Camera2Enumerator"; + private final static double NANO_SECONDS_PER_SECOND = 1.0e9; + + // Each entry contains the supported formats for a given camera index. The formats are enumerated + // lazily in getSupportedFormats(), and cached for future reference. + private static final Map<String, List<CaptureFormat>> cachedSupportedFormats = + new HashMap<String, List<CaptureFormat>>(); + + final Context context; + @Nullable final CameraManager cameraManager; + + public Camera2Enumerator(Context context) { + this.context = context; + this.cameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); + } + + @Override + public String[] getDeviceNames() { + try { + return cameraManager.getCameraIdList(); + } catch (CameraAccessException e) { + Logging.e(TAG, "Camera access exception", e); + return new String[] {}; + } + } + + @Override + public boolean isFrontFacing(String deviceName) { + CameraCharacteristics characteristics = getCameraCharacteristics(deviceName); + + return characteristics != null + && characteristics.get(CameraCharacteristics.LENS_FACING) + == CameraMetadata.LENS_FACING_FRONT; + } + + @Override + public boolean isBackFacing(String deviceName) { + CameraCharacteristics characteristics = getCameraCharacteristics(deviceName); + + return characteristics != null + && characteristics.get(CameraCharacteristics.LENS_FACING) + == CameraMetadata.LENS_FACING_BACK; + } + + @Override + public boolean isInfrared(String deviceName) { + CameraCharacteristics characteristics = getCameraCharacteristics(deviceName); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + Integer colors = characteristics.get(CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); + return colors != null && colors.equals(CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_NIR); + } + + return false; + } + + @Nullable + @Override + public List<CaptureFormat> getSupportedFormats(String deviceName) { + return getSupportedFormats(context, deviceName); + } + + @Override + public CameraVideoCapturer createCapturer( + String deviceName, CameraVideoCapturer.CameraEventsHandler eventsHandler) { + return new Camera2Capturer(context, deviceName, eventsHandler); + } + + private @Nullable CameraCharacteristics getCameraCharacteristics(String deviceName) { + try { + return cameraManager.getCameraCharacteristics(deviceName); + } catch (CameraAccessException | RuntimeException e) { + Logging.e(TAG, "Camera access exception", e); + return null; + } + } + + /** + * Checks if API is supported and all cameras have better than legacy support. + */ + public static boolean isSupported(Context context) { + CameraManager cameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); + try { + String[] cameraIds = cameraManager.getCameraIdList(); + for (String id : cameraIds) { + CameraCharacteristics characteristics = cameraManager.getCameraCharacteristics(id); + if (characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) + == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { + return false; + } + } + } catch (CameraAccessException | RuntimeException e) { + Logging.e(TAG, "Failed to check if camera2 is supported", e); + return false; + } + return true; + } + + static int getFpsUnitFactor(Range<Integer>[] fpsRanges) { + if (fpsRanges.length == 0) { + return 1000; + } + return fpsRanges[0].getUpper() < 1000 ? 1000 : 1; + } + + static List<Size> getSupportedSizes(CameraCharacteristics cameraCharacteristics) { + final StreamConfigurationMap streamMap = + cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + final int supportLevel = + cameraCharacteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL); + + final android.util.Size[] nativeSizes = streamMap.getOutputSizes(SurfaceTexture.class); + final List<Size> sizes = convertSizes(nativeSizes); + + // Video may be stretched pre LMR1 on legacy implementations. + // Filter out formats that have different aspect ratio than the sensor array. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP_MR1 + && supportLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { + final Rect activeArraySize = + cameraCharacteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE); + final ArrayList<Size> filteredSizes = new ArrayList<Size>(); + + for (Size size : sizes) { + if (activeArraySize.width() * size.height == activeArraySize.height() * size.width) { + filteredSizes.add(size); + } + } + + return filteredSizes; + } else { + return sizes; + } + } + + @Nullable + static List<CaptureFormat> getSupportedFormats(Context context, String cameraId) { + return getSupportedFormats( + (CameraManager) context.getSystemService(Context.CAMERA_SERVICE), cameraId); + } + + @Nullable + static List<CaptureFormat> getSupportedFormats(CameraManager cameraManager, String cameraId) { + synchronized (cachedSupportedFormats) { + if (cachedSupportedFormats.containsKey(cameraId)) { + return cachedSupportedFormats.get(cameraId); + } + + Logging.d(TAG, "Get supported formats for camera index " + cameraId + "."); + final long startTimeMs = SystemClock.elapsedRealtime(); + + final CameraCharacteristics cameraCharacteristics; + try { + cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId); + } catch (Exception ex) { + Logging.e(TAG, "getCameraCharacteristics()", ex); + return new ArrayList<CaptureFormat>(); + } + + final StreamConfigurationMap streamMap = + cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + + Range<Integer>[] fpsRanges = + cameraCharacteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); + List<CaptureFormat.FramerateRange> framerateRanges = + convertFramerates(fpsRanges, getFpsUnitFactor(fpsRanges)); + List<Size> sizes = getSupportedSizes(cameraCharacteristics); + + int defaultMaxFps = 0; + for (CaptureFormat.FramerateRange framerateRange : framerateRanges) { + defaultMaxFps = Math.max(defaultMaxFps, framerateRange.max); + } + + final List<CaptureFormat> formatList = new ArrayList<CaptureFormat>(); + for (Size size : sizes) { + long minFrameDurationNs = 0; + try { + minFrameDurationNs = streamMap.getOutputMinFrameDuration( + SurfaceTexture.class, new android.util.Size(size.width, size.height)); + } catch (Exception e) { + // getOutputMinFrameDuration() is not supported on all devices. Ignore silently. + } + final int maxFps = (minFrameDurationNs == 0) + ? defaultMaxFps + : (int) Math.round(NANO_SECONDS_PER_SECOND / minFrameDurationNs) * 1000; + formatList.add(new CaptureFormat(size.width, size.height, 0, maxFps)); + Logging.d(TAG, "Format: " + size.width + "x" + size.height + "@" + maxFps); + } + + cachedSupportedFormats.put(cameraId, formatList); + final long endTimeMs = SystemClock.elapsedRealtime(); + Logging.d(TAG, "Get supported formats for camera index " + cameraId + " done." + + " Time spent: " + (endTimeMs - startTimeMs) + " ms."); + return formatList; + } + } + + // Convert from android.util.Size to Size. + private static List<Size> convertSizes(android.util.Size[] cameraSizes) { + if (cameraSizes == null || cameraSizes.length == 0) { + return Collections.emptyList(); + } + final List<Size> sizes = new ArrayList<>(cameraSizes.length); + for (android.util.Size size : cameraSizes) { + sizes.add(new Size(size.getWidth(), size.getHeight())); + } + return sizes; + } + + // Convert from android.util.Range<Integer> to CaptureFormat.FramerateRange. + static List<CaptureFormat.FramerateRange> convertFramerates( + Range<Integer>[] arrayRanges, int unitFactor) { + final List<CaptureFormat.FramerateRange> ranges = new ArrayList<CaptureFormat.FramerateRange>(); + for (Range<Integer> range : arrayRanges) { + ranges.add(new CaptureFormat.FramerateRange( + range.getLower() * unitFactor, range.getUpper() * unitFactor)); + } + return ranges; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerationAndroid.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerationAndroid.java new file mode 100644 index 0000000000..0c3188fffe --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerationAndroid.java @@ -0,0 +1,206 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static java.lang.Math.abs; + +import android.graphics.ImageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +@SuppressWarnings("deprecation") +public class CameraEnumerationAndroid { + private final static String TAG = "CameraEnumerationAndroid"; + + static final ArrayList<Size> COMMON_RESOLUTIONS = new ArrayList<Size>(Arrays.asList( + // 0, Unknown resolution + new Size(160, 120), // 1, QQVGA + new Size(240, 160), // 2, HQVGA + new Size(320, 240), // 3, QVGA + new Size(400, 240), // 4, WQVGA + new Size(480, 320), // 5, HVGA + new Size(640, 360), // 6, nHD + new Size(640, 480), // 7, VGA + new Size(768, 480), // 8, WVGA + new Size(854, 480), // 9, FWVGA + new Size(800, 600), // 10, SVGA + new Size(960, 540), // 11, qHD + new Size(960, 640), // 12, DVGA + new Size(1024, 576), // 13, WSVGA + new Size(1024, 600), // 14, WVSGA + new Size(1280, 720), // 15, HD + new Size(1280, 1024), // 16, SXGA + new Size(1920, 1080), // 17, Full HD + new Size(1920, 1440), // 18, Full HD 4:3 + new Size(2560, 1440), // 19, QHD + new Size(3840, 2160) // 20, UHD + )); + + public static class CaptureFormat { + // Class to represent a framerate range. The framerate varies because of lightning conditions. + // The values are multiplied by 1000, so 1000 represents one frame per second. + public static class FramerateRange { + public int min; + public int max; + + public FramerateRange(int min, int max) { + this.min = min; + this.max = max; + } + + @Override + public String toString() { + return "[" + (min / 1000.0f) + ":" + (max / 1000.0f) + "]"; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof FramerateRange)) { + return false; + } + final FramerateRange otherFramerate = (FramerateRange) other; + return min == otherFramerate.min && max == otherFramerate.max; + } + + @Override + public int hashCode() { + // Use prime close to 2^16 to avoid collisions for normal values less than 2^16. + return 1 + 65537 * min + max; + } + } + + public final int width; + public final int height; + public final FramerateRange framerate; + + // TODO(hbos): If VideoCapturer.startCapture is updated to support other image formats then this + // needs to be updated and VideoCapturer.getSupportedFormats need to return CaptureFormats of + // all imageFormats. + public final int imageFormat = ImageFormat.NV21; + + public CaptureFormat(int width, int height, int minFramerate, int maxFramerate) { + this.width = width; + this.height = height; + this.framerate = new FramerateRange(minFramerate, maxFramerate); + } + + public CaptureFormat(int width, int height, FramerateRange framerate) { + this.width = width; + this.height = height; + this.framerate = framerate; + } + + // Calculates the frame size of this capture format. + public int frameSize() { + return frameSize(width, height, imageFormat); + } + + // Calculates the frame size of the specified image format. Currently only + // supporting ImageFormat.NV21. + // The size is width * height * number of bytes per pixel. + // http://developer.android.com/reference/android/hardware/Camera.html#addCallbackBuffer(byte[]) + public static int frameSize(int width, int height, int imageFormat) { + if (imageFormat != ImageFormat.NV21) { + throw new UnsupportedOperationException("Don't know how to calculate " + + "the frame size of non-NV21 image formats."); + } + return (width * height * ImageFormat.getBitsPerPixel(imageFormat)) / 8; + } + + @Override + public String toString() { + return width + "x" + height + "@" + framerate; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CaptureFormat)) { + return false; + } + final CaptureFormat otherFormat = (CaptureFormat) other; + return width == otherFormat.width && height == otherFormat.height + && framerate.equals(otherFormat.framerate); + } + + @Override + public int hashCode() { + return 1 + (width * 65497 + height) * 251 + framerate.hashCode(); + } + } + + // Helper class for finding the closest supported format for the two functions below. It creates a + // comparator based on the difference to some requested parameters, where the element with the + // minimum difference is the element that is closest to the requested parameters. + private static abstract class ClosestComparator<T> implements Comparator<T> { + // Difference between supported and requested parameter. + abstract int diff(T supportedParameter); + + @Override + public int compare(T t1, T t2) { + return diff(t1) - diff(t2); + } + } + + // Prefer a fps range with an upper bound close to `framerate`. Also prefer a fps range with a low + // lower bound, to allow the framerate to fluctuate based on lightning conditions. + public static CaptureFormat.FramerateRange getClosestSupportedFramerateRange( + List<CaptureFormat.FramerateRange> supportedFramerates, final int requestedFps) { + return Collections.min( + supportedFramerates, new ClosestComparator<CaptureFormat.FramerateRange>() { + // Progressive penalty if the upper bound is further away than `MAX_FPS_DIFF_THRESHOLD` + // from requested. + private static final int MAX_FPS_DIFF_THRESHOLD = 5000; + private static final int MAX_FPS_LOW_DIFF_WEIGHT = 1; + private static final int MAX_FPS_HIGH_DIFF_WEIGHT = 3; + + // Progressive penalty if the lower bound is bigger than `MIN_FPS_THRESHOLD`. + private static final int MIN_FPS_THRESHOLD = 8000; + private static final int MIN_FPS_LOW_VALUE_WEIGHT = 1; + private static final int MIN_FPS_HIGH_VALUE_WEIGHT = 4; + + // Use one weight for small `value` less than `threshold`, and another weight above. + private int progressivePenalty(int value, int threshold, int lowWeight, int highWeight) { + return (value < threshold) ? value * lowWeight + : threshold * lowWeight + (value - threshold) * highWeight; + } + + @Override + int diff(CaptureFormat.FramerateRange range) { + final int minFpsError = progressivePenalty( + range.min, MIN_FPS_THRESHOLD, MIN_FPS_LOW_VALUE_WEIGHT, MIN_FPS_HIGH_VALUE_WEIGHT); + final int maxFpsError = progressivePenalty(Math.abs(requestedFps * 1000 - range.max), + MAX_FPS_DIFF_THRESHOLD, MAX_FPS_LOW_DIFF_WEIGHT, MAX_FPS_HIGH_DIFF_WEIGHT); + return minFpsError + maxFpsError; + } + }); + } + + public static Size getClosestSupportedSize( + List<Size> supportedSizes, final int requestedWidth, final int requestedHeight) { + return Collections.min(supportedSizes, new ClosestComparator<Size>() { + @Override + int diff(Size size) { + return abs(requestedWidth - size.width) + abs(requestedHeight - size.height); + } + }); + } + + // Helper method for camera classes. + static void reportCameraResolution(Histogram histogram, Size resolution) { + int index = COMMON_RESOLUTIONS.indexOf(resolution); + // 0 is reserved for unknown resolution, so add 1. + // indexOf returns -1 for unknown resolutions so it becomes 0 automatically. + histogram.addSample(index + 1); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerator.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerator.java new file mode 100644 index 0000000000..db34d542c8 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraEnumerator.java @@ -0,0 +1,26 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; + +import java.util.List; + +public interface CameraEnumerator { + public String[] getDeviceNames(); + public boolean isFrontFacing(String deviceName); + public boolean isBackFacing(String deviceName); + public boolean isInfrared(String deviceName); + public List<CaptureFormat> getSupportedFormats(String deviceName); + + public CameraVideoCapturer createCapturer( + String deviceName, CameraVideoCapturer.CameraEventsHandler eventsHandler); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraVideoCapturer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraVideoCapturer.java new file mode 100644 index 0000000000..ec26868b5c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CameraVideoCapturer.java @@ -0,0 +1,172 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaRecorder; + +/** + * Base interface for camera1 and camera2 implementations. Extends VideoCapturer with a + * switchCamera() function. Also provides subinterfaces for handling camera events, and a helper + * class for detecting camera freezes. + */ +public interface CameraVideoCapturer extends VideoCapturer { + /** + * Camera events handler - can be used to be notifed about camera events. The callbacks are + * executed from an arbitrary thread. + */ + public interface CameraEventsHandler { + // Camera error handler - invoked when camera can not be opened + // or any camera exception happens on camera thread. + void onCameraError(String errorDescription); + + // Called when camera is disconnected. + void onCameraDisconnected(); + + // Invoked when camera stops receiving frames. + void onCameraFreezed(String errorDescription); + + // Callback invoked when camera is opening. + void onCameraOpening(String cameraName); + + // Callback invoked when first camera frame is available after camera is started. + void onFirstFrameAvailable(); + + // Callback invoked when camera is closed. + void onCameraClosed(); + } + + /** + * Camera switch handler - one of these functions are invoked with the result of switchCamera(). + * The callback may be called on an arbitrary thread. + */ + public interface CameraSwitchHandler { + // Invoked on success. `isFrontCamera` is true if the new camera is front facing. + void onCameraSwitchDone(boolean isFrontCamera); + + // Invoked on failure, e.g. camera is stopped or only one camera available. + void onCameraSwitchError(String errorDescription); + } + + /** + * Switch camera to the next valid camera id. This can only be called while the camera is running. + * This function can be called from any thread. + */ + void switchCamera(CameraSwitchHandler switchEventsHandler); + + /** + * Switch camera to the specified camera id. This can only be called while the camera is running. + * This function can be called from any thread. + */ + void switchCamera(CameraSwitchHandler switchEventsHandler, String cameraName); + + /** + * MediaRecorder add/remove handler - one of these functions are invoked with the result of + * addMediaRecorderToCamera() or removeMediaRecorderFromCamera calls. + * The callback may be called on an arbitrary thread. + */ + @Deprecated + public interface MediaRecorderHandler { + // Invoked on success. + void onMediaRecorderSuccess(); + + // Invoked on failure, e.g. camera is stopped or any exception happens. + void onMediaRecorderError(String errorDescription); + } + + /** + * Add MediaRecorder to camera pipeline. This can only be called while the camera is running. + * Once MediaRecorder is added to camera pipeline camera switch is not allowed. + * This function can be called from any thread. + */ + @Deprecated + default void addMediaRecorderToCamera( + MediaRecorder mediaRecorder, MediaRecorderHandler resultHandler) { + throw new UnsupportedOperationException("Deprecated and not implemented."); + } + + /** + * Remove MediaRecorder from camera pipeline. This can only be called while the camera is running. + * This function can be called from any thread. + */ + @Deprecated + default void removeMediaRecorderFromCamera(MediaRecorderHandler resultHandler) { + throw new UnsupportedOperationException("Deprecated and not implemented."); + } + + /** + * Helper class to log framerate and detect if the camera freezes. It will run periodic callbacks + * on the SurfaceTextureHelper thread passed in the ctor, and should only be operated from that + * thread. + */ + public static class CameraStatistics { + private final static String TAG = "CameraStatistics"; + private final static int CAMERA_OBSERVER_PERIOD_MS = 2000; + private final static int CAMERA_FREEZE_REPORT_TIMOUT_MS = 4000; + + private final SurfaceTextureHelper surfaceTextureHelper; + private final CameraEventsHandler eventsHandler; + private int frameCount; + private int freezePeriodCount; + // Camera observer - monitors camera framerate. Observer is executed on camera thread. + private final Runnable cameraObserver = new Runnable() { + @Override + public void run() { + final int cameraFps = Math.round(frameCount * 1000.0f / CAMERA_OBSERVER_PERIOD_MS); + Logging.d(TAG, "Camera fps: " + cameraFps + "."); + if (frameCount == 0) { + ++freezePeriodCount; + if (CAMERA_OBSERVER_PERIOD_MS * freezePeriodCount >= CAMERA_FREEZE_REPORT_TIMOUT_MS + && eventsHandler != null) { + Logging.e(TAG, "Camera freezed."); + if (surfaceTextureHelper.isTextureInUse()) { + // This can only happen if we are capturing to textures. + eventsHandler.onCameraFreezed("Camera failure. Client must return video buffers."); + } else { + eventsHandler.onCameraFreezed("Camera failure."); + } + return; + } + } else { + freezePeriodCount = 0; + } + frameCount = 0; + surfaceTextureHelper.getHandler().postDelayed(this, CAMERA_OBSERVER_PERIOD_MS); + } + }; + + public CameraStatistics( + SurfaceTextureHelper surfaceTextureHelper, CameraEventsHandler eventsHandler) { + if (surfaceTextureHelper == null) { + throw new IllegalArgumentException("SurfaceTextureHelper is null"); + } + this.surfaceTextureHelper = surfaceTextureHelper; + this.eventsHandler = eventsHandler; + this.frameCount = 0; + this.freezePeriodCount = 0; + surfaceTextureHelper.getHandler().postDelayed(cameraObserver, CAMERA_OBSERVER_PERIOD_MS); + } + + private void checkThread() { + if (Thread.currentThread() != surfaceTextureHelper.getHandler().getLooper().getThread()) { + throw new IllegalStateException("Wrong thread"); + } + } + + public void addFrame() { + checkThread(); + ++frameCount; + } + + public void release() { + surfaceTextureHelper.getHandler().removeCallbacks(cameraObserver); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CapturerObserver.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CapturerObserver.java new file mode 100644 index 0000000000..382dc15b3a --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CapturerObserver.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for observering a capturer. Passed to {@link VideoCapturer#initialize}. Provided by + * {@link VideoSource#getCapturerObserver}. + * + * All callbacks must be executed on a single thread. + */ +public interface CapturerObserver { + /** Notify if the capturer have been started successfully or not. */ + void onCapturerStarted(boolean success); + /** Notify that the capturer has been stopped. */ + void onCapturerStopped(); + + /** Delivers a captured frame. */ + void onFrameCaptured(VideoFrame frame); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/CryptoOptions.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/CryptoOptions.java new file mode 100644 index 0000000000..6e06bc6426 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/CryptoOptions.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * CryptoOptions defines advanced cryptographic settings for native WebRTC. + * These settings must be passed into RTCConfiguration. WebRTC is secur by + * default and you should not need to set any of these options unless you are + * specifically looking for an additional crypto feature such as AES_GCM + * support. This class is the Java binding of native api/crypto/cryptooptions.h + */ +public final class CryptoOptions { + /** + * SRTP Related Peer Connection Options. + */ + public final class Srtp { + /** + * Enable GCM crypto suites from RFC 7714 for SRTP. GCM will only be used + * if both sides enable it + */ + private final boolean enableGcmCryptoSuites; + /** + * If set to true, the (potentially insecure) crypto cipher + * kSrtpAes128CmSha1_32 will be included in the list of supported ciphers + * during negotiation. It will only be used if both peers support it and no + * other ciphers get preferred. + */ + private final boolean enableAes128Sha1_32CryptoCipher; + /** + * If set to true, encrypted RTP header extensions as defined in RFC 6904 + * will be negotiated. They will only be used if both peers support them. + */ + private final boolean enableEncryptedRtpHeaderExtensions; + + private Srtp(boolean enableGcmCryptoSuites, boolean enableAes128Sha1_32CryptoCipher, + boolean enableEncryptedRtpHeaderExtensions) { + this.enableGcmCryptoSuites = enableGcmCryptoSuites; + this.enableAes128Sha1_32CryptoCipher = enableAes128Sha1_32CryptoCipher; + this.enableEncryptedRtpHeaderExtensions = enableEncryptedRtpHeaderExtensions; + } + + @CalledByNative("Srtp") + public boolean getEnableGcmCryptoSuites() { + return enableGcmCryptoSuites; + } + + @CalledByNative("Srtp") + public boolean getEnableAes128Sha1_32CryptoCipher() { + return enableAes128Sha1_32CryptoCipher; + } + + @CalledByNative("Srtp") + public boolean getEnableEncryptedRtpHeaderExtensions() { + return enableEncryptedRtpHeaderExtensions; + } + } + + /** + * Options to be used when the FrameEncryptor / FrameDecryptor APIs are used. + */ + public final class SFrame { + /** + * If set all RtpSenders must have an FrameEncryptor attached to them before + * they are allowed to send packets. All RtpReceivers must have a + * FrameDecryptor attached to them before they are able to receive packets. + */ + private final boolean requireFrameEncryption; + + private SFrame(boolean requireFrameEncryption) { + this.requireFrameEncryption = requireFrameEncryption; + } + + @CalledByNative("SFrame") + public boolean getRequireFrameEncryption() { + return requireFrameEncryption; + } + } + + private final Srtp srtp; + private final SFrame sframe; + + private CryptoOptions(boolean enableGcmCryptoSuites, boolean enableAes128Sha1_32CryptoCipher, + boolean enableEncryptedRtpHeaderExtensions, boolean requireFrameEncryption) { + this.srtp = new Srtp( + enableGcmCryptoSuites, enableAes128Sha1_32CryptoCipher, enableEncryptedRtpHeaderExtensions); + this.sframe = new SFrame(requireFrameEncryption); + } + + public static Builder builder() { + return new Builder(); + } + + @CalledByNative + public Srtp getSrtp() { + return srtp; + } + + @CalledByNative + public SFrame getSFrame() { + return sframe; + } + + public static class Builder { + private boolean enableGcmCryptoSuites; + private boolean enableAes128Sha1_32CryptoCipher; + private boolean enableEncryptedRtpHeaderExtensions; + private boolean requireFrameEncryption; + + private Builder() {} + + public Builder setEnableGcmCryptoSuites(boolean enableGcmCryptoSuites) { + this.enableGcmCryptoSuites = enableGcmCryptoSuites; + return this; + } + + public Builder setEnableAes128Sha1_32CryptoCipher(boolean enableAes128Sha1_32CryptoCipher) { + this.enableAes128Sha1_32CryptoCipher = enableAes128Sha1_32CryptoCipher; + return this; + } + + public Builder setEnableEncryptedRtpHeaderExtensions( + boolean enableEncryptedRtpHeaderExtensions) { + this.enableEncryptedRtpHeaderExtensions = enableEncryptedRtpHeaderExtensions; + return this; + } + + public Builder setRequireFrameEncryption(boolean requireFrameEncryption) { + this.requireFrameEncryption = requireFrameEncryption; + return this; + } + + public CryptoOptions createCryptoOptions() { + return new CryptoOptions(enableGcmCryptoSuites, enableAes128Sha1_32CryptoCipher, + enableEncryptedRtpHeaderExtensions, requireFrameEncryption); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/DataChannel.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/DataChannel.java new file mode 100644 index 0000000000..b9301f1faa --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/DataChannel.java @@ -0,0 +1,196 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** Java wrapper for a C++ DataChannelInterface. */ +public class DataChannel { + /** Java wrapper for WebIDL RTCDataChannel. */ + public static class Init { + public boolean ordered = true; + // Optional unsigned short in WebIDL, -1 means unspecified. + public int maxRetransmitTimeMs = -1; + // Optional unsigned short in WebIDL, -1 means unspecified. + public int maxRetransmits = -1; + public String protocol = ""; + public boolean negotiated; + // Optional unsigned short in WebIDL, -1 means unspecified. + public int id = -1; + + @CalledByNative("Init") + boolean getOrdered() { + return ordered; + } + + @CalledByNative("Init") + int getMaxRetransmitTimeMs() { + return maxRetransmitTimeMs; + } + + @CalledByNative("Init") + int getMaxRetransmits() { + return maxRetransmits; + } + + @CalledByNative("Init") + String getProtocol() { + return protocol; + } + + @CalledByNative("Init") + boolean getNegotiated() { + return negotiated; + } + + @CalledByNative("Init") + int getId() { + return id; + } + } + + /** Java version of C++ DataBuffer. The atom of data in a DataChannel. */ + public static class Buffer { + /** The underlying data. */ + public final ByteBuffer data; + + /** + * Indicates whether `data` contains UTF-8 text or "binary data" + * (i.e. anything else). + */ + public final boolean binary; + + @CalledByNative("Buffer") + public Buffer(ByteBuffer data, boolean binary) { + this.data = data; + this.binary = binary; + } + } + + /** Java version of C++ DataChannelObserver. */ + public interface Observer { + /** The data channel's bufferedAmount has changed. */ + @CalledByNative("Observer") public void onBufferedAmountChange(long previousAmount); + /** The data channel state has changed. */ + @CalledByNative("Observer") public void onStateChange(); + /** + * A data buffer was successfully received. NOTE: `buffer.data` will be + * freed once this function returns so callers who want to use the data + * asynchronously must make sure to copy it first. + */ + @CalledByNative("Observer") public void onMessage(Buffer buffer); + } + + /** Keep in sync with DataChannelInterface::DataState. */ + public enum State { + CONNECTING, + OPEN, + CLOSING, + CLOSED; + + @CalledByNative("State") + static State fromNativeIndex(int nativeIndex) { + return values()[nativeIndex]; + } + } + + private long nativeDataChannel; + private long nativeObserver; + + @CalledByNative + public DataChannel(long nativeDataChannel) { + this.nativeDataChannel = nativeDataChannel; + } + + /** Register `observer`, replacing any previously-registered observer. */ + public void registerObserver(Observer observer) { + checkDataChannelExists(); + if (nativeObserver != 0) { + nativeUnregisterObserver(nativeObserver); + } + nativeObserver = nativeRegisterObserver(observer); + } + + /** Unregister the (only) observer. */ + public void unregisterObserver() { + checkDataChannelExists(); + nativeUnregisterObserver(nativeObserver); + nativeObserver = 0; + } + + public String label() { + checkDataChannelExists(); + return nativeLabel(); + } + + public int id() { + checkDataChannelExists(); + return nativeId(); + } + + public State state() { + checkDataChannelExists(); + return nativeState(); + } + + /** + * Return the number of bytes of application data (UTF-8 text and binary data) + * that have been queued using SendBuffer but have not yet been transmitted + * to the network. + */ + public long bufferedAmount() { + checkDataChannelExists(); + return nativeBufferedAmount(); + } + + /** Close the channel. */ + public void close() { + checkDataChannelExists(); + nativeClose(); + } + + /** Send `data` to the remote peer; return success. */ + public boolean send(Buffer buffer) { + checkDataChannelExists(); + // TODO(fischman): this could be cleverer about avoiding copies if the + // ByteBuffer is direct and/or is backed by an array. + byte[] data = new byte[buffer.data.remaining()]; + buffer.data.get(data); + return nativeSend(data, buffer.binary); + } + + /** Dispose of native resources attached to this channel. */ + public void dispose() { + checkDataChannelExists(); + JniCommon.nativeReleaseRef(nativeDataChannel); + nativeDataChannel = 0; + } + + @CalledByNative + long getNativeDataChannel() { + return nativeDataChannel; + } + + private void checkDataChannelExists() { + if (nativeDataChannel == 0) { + throw new IllegalStateException("DataChannel has been disposed."); + } + } + + private native long nativeRegisterObserver(Observer observer); + private native void nativeUnregisterObserver(long observer); + private native String nativeLabel(); + private native int nativeId(); + private native State nativeState(); + private native long nativeBufferedAmount(); + private native void nativeClose(); + private native boolean nativeSend(byte[] data, boolean binary); +}; diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Dav1dDecoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Dav1dDecoder.java new file mode 100644 index 0000000000..ecb16bc3a1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Dav1dDecoder.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class Dav1dDecoder extends WrappedNativeVideoDecoder { + @Override + public long createNativeVideoDecoder() { + return nativeCreateDecoder(); + } + + static native long nativeCreateDecoder(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java new file mode 100644 index 0000000000..d7a8694d3d --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.LinkedHashSet; + +/** + * Helper class that combines HW and SW decoders. + */ +public class DefaultVideoDecoderFactory implements VideoDecoderFactory { + private final VideoDecoderFactory hardwareVideoDecoderFactory; + private final VideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + private final @Nullable VideoDecoderFactory platformSoftwareVideoDecoderFactory; + + /** + * Create decoder factory using default hardware decoder factory. + */ + public DefaultVideoDecoderFactory(@Nullable EglBase.Context eglContext) { + this.hardwareVideoDecoderFactory = new HardwareVideoDecoderFactory(eglContext); + this.platformSoftwareVideoDecoderFactory = new PlatformSoftwareVideoDecoderFactory(eglContext); + } + + /** + * Create decoder factory using explicit hardware decoder factory. + */ + DefaultVideoDecoderFactory(VideoDecoderFactory hardwareVideoDecoderFactory) { + this.hardwareVideoDecoderFactory = hardwareVideoDecoderFactory; + this.platformSoftwareVideoDecoderFactory = null; + } + + @Override + public @Nullable VideoDecoder createDecoder(VideoCodecInfo codecType) { + VideoDecoder softwareDecoder = softwareVideoDecoderFactory.createDecoder(codecType); + final VideoDecoder hardwareDecoder = hardwareVideoDecoderFactory.createDecoder(codecType); + if (softwareDecoder == null && platformSoftwareVideoDecoderFactory != null) { + softwareDecoder = platformSoftwareVideoDecoderFactory.createDecoder(codecType); + } + if (hardwareDecoder != null && softwareDecoder != null) { + // Both hardware and software supported, wrap it in a software fallback + return new VideoDecoderFallback( + /* fallback= */ softwareDecoder, /* primary= */ hardwareDecoder); + } + return hardwareDecoder != null ? hardwareDecoder : softwareDecoder; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet<VideoCodecInfo> supportedCodecInfos = new LinkedHashSet<VideoCodecInfo>(); + + supportedCodecInfos.addAll(Arrays.asList(softwareVideoDecoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(hardwareVideoDecoderFactory.getSupportedCodecs())); + if (platformSoftwareVideoDecoderFactory != null) { + supportedCodecInfos.addAll( + Arrays.asList(platformSoftwareVideoDecoderFactory.getSupportedCodecs())); + } + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/DtmfSender.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/DtmfSender.java new file mode 100644 index 0000000000..6549823089 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/DtmfSender.java @@ -0,0 +1,96 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Java wrapper for a C++ DtmfSenderInterface. */ +public class DtmfSender { + private long nativeDtmfSender; + + public DtmfSender(long nativeDtmfSender) { + this.nativeDtmfSender = nativeDtmfSender; + } + + /** + * @return true if this DtmfSender is capable of sending DTMF. Otherwise false. + */ + public boolean canInsertDtmf() { + checkDtmfSenderExists(); + return nativeCanInsertDtmf(nativeDtmfSender); + } + + /** + * Queues a task that sends the provided DTMF tones. + * <p> + * If insertDtmf is called on the same object while an existing task for this + * object to generate DTMF is still running, the previous task is canceled. + * + * @param tones This parameter is treated as a series of characters. The characters 0 + * through 9, A through D, #, and * generate the associated DTMF tones. The + * characters a to d are equivalent to A to D. The character ',' indicates a + * delay of 2 seconds before processing the next character in the tones + * parameter. Unrecognized characters are ignored. + * @param duration Indicates the duration in ms to use for each character passed in the tones + * parameter. The duration cannot be more than 6000 or less than 70. + * @param interToneGap Indicates the gap between tones in ms. Must be at least 50 ms but should be + * as short as possible. + * @return true on success and false on failure. + */ + public boolean insertDtmf(String tones, int duration, int interToneGap) { + checkDtmfSenderExists(); + return nativeInsertDtmf(nativeDtmfSender, tones, duration, interToneGap); + } + + /** + * @return The tones remaining to be played out + */ + public String tones() { + checkDtmfSenderExists(); + return nativeTones(nativeDtmfSender); + } + + /** + * @return The current tone duration value in ms. This value will be the value last set via the + * insertDtmf() method, or the default value of 100 ms if insertDtmf() was never called. + */ + public int duration() { + checkDtmfSenderExists(); + return nativeDuration(nativeDtmfSender); + } + + /** + * @return The current value of the between-tone gap in ms. This value will be the value last set + * via the insertDtmf() method, or the default value of 50 ms if insertDtmf() was never + * called. + */ + public int interToneGap() { + checkDtmfSenderExists(); + return nativeInterToneGap(nativeDtmfSender); + } + + public void dispose() { + checkDtmfSenderExists(); + JniCommon.nativeReleaseRef(nativeDtmfSender); + nativeDtmfSender = 0; + } + + private void checkDtmfSenderExists() { + if (nativeDtmfSender == 0) { + throw new IllegalStateException("DtmfSender has been disposed."); + } + } + + private static native boolean nativeCanInsertDtmf(long dtmfSender); + private static native boolean nativeInsertDtmf( + long dtmfSender, String tones, int duration, int interToneGap); + private static native String nativeTones(long dtmfSender); + private static native int nativeDuration(long dtmfSender); + private static native int nativeInterToneGap(long dtmfSender); +}; diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase.java new file mode 100644 index 0000000000..3b45e3575b --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase.java @@ -0,0 +1,305 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.SurfaceTexture; +import android.view.Surface; +import androidx.annotation.Nullable; +import java.util.ArrayList; +import javax.microedition.khronos.egl.EGL10; + +/** + * Holds EGL state and utility methods for handling an egl 1.0 EGLContext, an EGLDisplay, + * and an EGLSurface. + */ +public interface EglBase { + // EGL wrapper for an actual EGLContext. + public interface Context { + public final static long NO_CONTEXT = 0; + + /** + * Returns an EGL context that can be used by native code. Returns NO_CONTEXT if the method is + * unsupported. + * + * @note This is currently only supported for EGL 1.4 and not for EGL 1.0. + */ + long getNativeEglContext(); + } + + /** + * Wraps the objects needed to interact with EGL that are independent of a particular EGLSurface. + * In practice this means EGLContext, EGLDisplay and EGLConfig objects. Separating them out in a + * standalone object allows for multiple EglBase instances to use the same underlying EGLContext, + * while still operating on their own EGLSurface. + */ + public interface EglConnection extends RefCounted { + /** Analogous to corresponding EglBase#create below. */ + public static EglConnection create(@Nullable Context sharedContext, int[] configAttributes) { + if (sharedContext == null) { + return EglConnection.createEgl14(configAttributes); + } else if (sharedContext instanceof EglBase14.Context) { + return new EglBase14Impl.EglConnection( + ((EglBase14.Context) sharedContext).getRawContext(), configAttributes); + } else if (sharedContext instanceof EglBase10.Context) { + return new EglBase10Impl.EglConnection( + ((EglBase10.Context) sharedContext).getRawContext(), configAttributes); + } + throw new IllegalArgumentException("Unrecognized Context"); + } + + /** Analogous to corresponding EglBase#createEgl10 below. */ + public static EglConnection createEgl10(int[] configAttributes) { + return new EglBase10Impl.EglConnection(/* sharedContext= */ null, configAttributes); + } + + /** Analogous to corresponding EglBase#createEgl14 below. */ + public static EglConnection createEgl14(int[] configAttributes) { + return new EglBase14Impl.EglConnection(/* sharedContext= */ null, configAttributes); + } + } + + // According to the documentation, EGL can be used from multiple threads at the same time if each + // thread has its own EGLContext, but in practice it deadlocks on some devices when doing this. + // Therefore, synchronize on this global lock before calling dangerous EGL functions that might + // deadlock. See https://bugs.chromium.org/p/webrtc/issues/detail?id=5702 for more info. + public static final Object lock = new Object(); + + // These constants are taken from EGL14.EGL_OPENGL_ES2_BIT and EGL14.EGL_CONTEXT_CLIENT_VERSION. + // https://android.googlesource.com/platform/frameworks/base/+/master/opengl/java/android/opengl/EGL14.java + // This is similar to how GlSurfaceView does: + // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.android/android/5.1.1_r1/android/opengl/GLSurfaceView.java#760 + public static final int EGL_OPENGL_ES2_BIT = 4; + public static final int EGL_OPENGL_ES3_BIT = 0x40; + // Android-specific extension. + public static final int EGL_RECORDABLE_ANDROID = 0x3142; + + public static ConfigBuilder configBuilder() { + return new ConfigBuilder(); + } + + public static class ConfigBuilder { + private int openGlesVersion = 2; + private boolean hasAlphaChannel; + private boolean supportsPixelBuffer; + private boolean isRecordable; + + public ConfigBuilder setOpenGlesVersion(int version) { + if (version < 1 || version > 3) { + throw new IllegalArgumentException("OpenGL ES version " + version + " not supported"); + } + this.openGlesVersion = version; + return this; + } + + public ConfigBuilder setHasAlphaChannel(boolean hasAlphaChannel) { + this.hasAlphaChannel = hasAlphaChannel; + return this; + } + + public ConfigBuilder setSupportsPixelBuffer(boolean supportsPixelBuffer) { + this.supportsPixelBuffer = supportsPixelBuffer; + return this; + } + + public ConfigBuilder setIsRecordable(boolean isRecordable) { + this.isRecordable = isRecordable; + return this; + } + + public int[] createConfigAttributes() { + ArrayList<Integer> list = new ArrayList<>(); + list.add(EGL10.EGL_RED_SIZE); + list.add(8); + list.add(EGL10.EGL_GREEN_SIZE); + list.add(8); + list.add(EGL10.EGL_BLUE_SIZE); + list.add(8); + if (hasAlphaChannel) { + list.add(EGL10.EGL_ALPHA_SIZE); + list.add(8); + } + if (openGlesVersion == 2 || openGlesVersion == 3) { + list.add(EGL10.EGL_RENDERABLE_TYPE); + list.add(openGlesVersion == 3 ? EGL_OPENGL_ES3_BIT : EGL_OPENGL_ES2_BIT); + } + if (supportsPixelBuffer) { + list.add(EGL10.EGL_SURFACE_TYPE); + list.add(EGL10.EGL_PBUFFER_BIT); + } + if (isRecordable) { + list.add(EGL_RECORDABLE_ANDROID); + list.add(1); + } + list.add(EGL10.EGL_NONE); + + final int[] res = new int[list.size()]; + for (int i = 0; i < list.size(); ++i) { + res[i] = list.get(i); + } + return res; + } + } + + public static final int[] CONFIG_PLAIN = configBuilder().createConfigAttributes(); + public static final int[] CONFIG_RGBA = + configBuilder().setHasAlphaChannel(true).createConfigAttributes(); + public static final int[] CONFIG_PIXEL_BUFFER = + configBuilder().setSupportsPixelBuffer(true).createConfigAttributes(); + public static final int[] CONFIG_PIXEL_RGBA_BUFFER = configBuilder() + .setHasAlphaChannel(true) + .setSupportsPixelBuffer(true) + .createConfigAttributes(); + public static final int[] CONFIG_RECORDABLE = + configBuilder().setIsRecordable(true).createConfigAttributes(); + + static int getOpenGlesVersionFromConfig(int[] configAttributes) { + for (int i = 0; i < configAttributes.length - 1; ++i) { + if (configAttributes[i] == EGL10.EGL_RENDERABLE_TYPE) { + switch (configAttributes[i + 1]) { + case EGL_OPENGL_ES2_BIT: + return 2; + case EGL_OPENGL_ES3_BIT: + return 3; + default: + return 1; + } + } + } + // Default to V1 if no renderable type is specified. + return 1; + } + + /** + * Creates a new EglBase with a shared EglConnection. EglBase instances sharing the same + * EglConnection should be used on the same thread to avoid the underlying EGLContext being made + * current on multiple threads. It is up to the client of EglBase to ensure that instances with a + * shared EglConnection are current on that thread before each use since other EglBase instances + * may have used the same EGLContext since the last interaction. + */ + public static EglBase create(EglConnection eglConnection) { + if (eglConnection == null) { + return create(); + } else if (eglConnection instanceof EglBase14Impl.EglConnection) { + return new EglBase14Impl((EglBase14Impl.EglConnection) eglConnection); + } else if (eglConnection instanceof EglBase10Impl.EglConnection) { + return new EglBase10Impl((EglBase10Impl.EglConnection) eglConnection); + } + throw new IllegalArgumentException("Unrecognized EglConnection"); + } + + /** + * Create a new context with the specified config attributes, sharing data with `sharedContext`. + * If `sharedContext` is null, a root EGL 1.4 context is created. + */ + public static EglBase create(@Nullable Context sharedContext, int[] configAttributes) { + if (sharedContext == null) { + return createEgl14(configAttributes); + } else if (sharedContext instanceof EglBase14.Context) { + return createEgl14((EglBase14.Context) sharedContext, configAttributes); + } else if (sharedContext instanceof EglBase10.Context) { + return createEgl10((EglBase10.Context) sharedContext, configAttributes); + } + throw new IllegalArgumentException("Unrecognized Context"); + } + + /** + * Helper function for creating a plain root context. This function will try to create an EGL 1.4 + * context if possible, and an EGL 1.0 context otherwise. + */ + public static EglBase create() { + return create(null /* shaderContext */, CONFIG_PLAIN); + } + + /** + * Helper function for creating a plain context, sharing data with `sharedContext`. This function + * will try to create an EGL 1.4 context if possible, and an EGL 1.0 context otherwise. + */ + public static EglBase create(Context sharedContext) { + return create(sharedContext, CONFIG_PLAIN); + } + + /** Explicitly create a root EGl 1.0 context with the specified config attributes. */ + public static EglBase10 createEgl10(int[] configAttributes) { + return new EglBase10Impl(/* sharedContext= */ null, configAttributes); + } + + /** + * Explicitly create a root EGl 1.0 context with the specified config attributes and shared + * context. + */ + public static EglBase10 createEgl10(EglBase10.Context sharedContext, int[] configAttributes) { + return new EglBase10Impl( + sharedContext == null ? null : sharedContext.getRawContext(), configAttributes); + } + + /** + * Explicitly create a root EGl 1.0 context with the specified config attributes + * and shared context. + */ + public static EglBase10 createEgl10( + javax.microedition.khronos.egl.EGLContext sharedContext, int[] configAttributes) { + return new EglBase10Impl(sharedContext, configAttributes); + } + + /** Explicitly create a root EGl 1.4 context with the specified config attributes. */ + public static EglBase14 createEgl14(int[] configAttributes) { + return new EglBase14Impl(/* sharedContext= */ null, configAttributes); + } + + /** + * Explicitly create a root EGl 1.4 context with the specified config attributes and shared + * context. + */ + public static EglBase14 createEgl14(EglBase14.Context sharedContext, int[] configAttributes) { + return new EglBase14Impl( + sharedContext == null ? null : sharedContext.getRawContext(), configAttributes); + } + + /** + * Explicitly create a root EGl 1.4 context with the specified config attributes + * and shared context. + */ + public static EglBase14 createEgl14( + android.opengl.EGLContext sharedContext, int[] configAttributes) { + return new EglBase14Impl(sharedContext, configAttributes); + } + + void createSurface(Surface surface); + + // Create EGLSurface from the Android SurfaceTexture. + void createSurface(SurfaceTexture surfaceTexture); + + // Create dummy 1x1 pixel buffer surface so the context can be made current. + void createDummyPbufferSurface(); + + void createPbufferSurface(int width, int height); + + Context getEglBaseContext(); + + boolean hasSurface(); + + int surfaceWidth(); + + int surfaceHeight(); + + void releaseSurface(); + + void release(); + + void makeCurrent(); + + // Detach the current EGL context, so that it can be made current on another thread. + void detachCurrent(); + + void swapBuffers(); + + void swapBuffers(long presentationTimeStampNs); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase10.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase10.java new file mode 100644 index 0000000000..ad2eb1c0a1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase10.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; + +/** EGL 1.0 implementation of EglBase. */ +public interface EglBase10 extends EglBase { + interface Context extends EglBase.Context { + EGLContext getRawContext(); + } + + interface EglConnection extends EglBase.EglConnection { + EGL10 getEgl(); + + EGLContext getContext(); + + EGLDisplay getDisplay(); + + EGLConfig getConfig(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase14.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase14.java new file mode 100644 index 0000000000..74553625a2 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglBase14.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.opengl.EGLConfig; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; + +/** EGL 1.4 implementation of EglBase. */ +public interface EglBase14 extends EglBase { + interface Context extends EglBase.Context { + EGLContext getRawContext(); + } + + interface EglConnection extends EglBase.EglConnection { + EGLContext getContext(); + + EGLDisplay getDisplay(); + + EGLConfig getConfig(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EglRenderer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglRenderer.java new file mode 100644 index 0000000000..921abe66e4 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglRenderer.java @@ -0,0 +1,780 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Bitmap; +import android.graphics.Matrix; +import android.graphics.SurfaceTexture; +import android.opengl.GLES20; +import android.view.Surface; +import androidx.annotation.GuardedBy; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Implements VideoSink by displaying the video stream on an EGL Surface. This class is intended to + * be used as a helper class for rendering on SurfaceViews and TextureViews. + */ +public class EglRenderer implements VideoSink { + private static final String TAG = "EglRenderer"; + private static final long LOG_INTERVAL_SEC = 4; + + public interface FrameListener { void onFrame(Bitmap frame); } + + /** Callback for clients to be notified about errors encountered during rendering. */ + public static interface ErrorCallback { + /** Called if GLES20.GL_OUT_OF_MEMORY is encountered during rendering. */ + void onGlOutOfMemory(); + } + + private static class FrameListenerAndParams { + public final FrameListener listener; + public final float scale; + public final RendererCommon.GlDrawer drawer; + public final boolean applyFpsReduction; + + public FrameListenerAndParams(FrameListener listener, float scale, + RendererCommon.GlDrawer drawer, boolean applyFpsReduction) { + this.listener = listener; + this.scale = scale; + this.drawer = drawer; + this.applyFpsReduction = applyFpsReduction; + } + } + + private class EglSurfaceCreation implements Runnable { + private Object surface; + + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void setSurface(Object surface) { + this.surface = surface; + } + + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void run() { + if (surface != null && eglBase != null && !eglBase.hasSurface()) { + if (surface instanceof Surface) { + eglBase.createSurface((Surface) surface); + } else if (surface instanceof SurfaceTexture) { + eglBase.createSurface((SurfaceTexture) surface); + } else { + throw new IllegalStateException("Invalid surface: " + surface); + } + eglBase.makeCurrent(); + // Necessary for YUV frames with odd width. + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + } + } + } + + protected final String name; + + // `eglThread` is used for rendering, and is synchronized on `threadLock`. + private final Object threadLock = new Object(); + @GuardedBy("threadLock") @Nullable private EglThread eglThread; + + private final Runnable eglExceptionCallback = new Runnable() { + @Override + public void run() { + synchronized (threadLock) { + eglThread = null; + } + } + }; + + private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>(); + + private volatile ErrorCallback errorCallback; + + // Variables for fps reduction. + private final Object fpsReductionLock = new Object(); + // Time for when next frame should be rendered. + private long nextFrameTimeNs; + // Minimum duration between frames when fps reduction is active, or -1 if video is completely + // paused. + private long minRenderPeriodNs; + + // EGL and GL resources for drawing YUV/OES textures. After initialization, these are only + // accessed from the render thread. + @Nullable private EglBase eglBase; + private final VideoFrameDrawer frameDrawer; + @Nullable private RendererCommon.GlDrawer drawer; + private boolean usePresentationTimeStamp; + private final Matrix drawMatrix = new Matrix(); + + // Pending frame to render. Serves as a queue with size 1. Synchronized on `frameLock`. + private final Object frameLock = new Object(); + @Nullable private VideoFrame pendingFrame; + + // These variables are synchronized on `layoutLock`. + private final Object layoutLock = new Object(); + private float layoutAspectRatio; + // If true, mirrors the video stream horizontally. + private boolean mirrorHorizontally; + // If true, mirrors the video stream vertically. + private boolean mirrorVertically; + + // These variables are synchronized on `statisticsLock`. + private final Object statisticsLock = new Object(); + // Total number of video frames received in renderFrame() call. + private int framesReceived; + // Number of video frames dropped by renderFrame() because previous frame has not been rendered + // yet. + private int framesDropped; + // Number of rendered video frames. + private int framesRendered; + // Start time for counting these statistics, or 0 if we haven't started measuring yet. + private long statisticsStartTimeNs; + // Time in ns spent in renderFrameOnRenderThread() function. + private long renderTimeNs; + // Time in ns spent by the render thread in the swapBuffers() function. + private long renderSwapBufferTimeNs; + + // Used for bitmap capturing. + private final GlTextureFrameBuffer bitmapTextureFramebuffer = + new GlTextureFrameBuffer(GLES20.GL_RGBA); + + private final Runnable logStatisticsRunnable = new Runnable() { + @Override + public void run() { + logStatistics(); + synchronized (threadLock) { + if (eglThread != null) { + eglThread.getHandler().removeCallbacks(logStatisticsRunnable); + eglThread.getHandler().postDelayed( + logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC)); + } + } + } + }; + + private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation(); + + /** + * Standard constructor. The name will be included when logging. In order to render something, + * you must first call init() and createEglSurface. + */ + public EglRenderer(String name) { + this(name, new VideoFrameDrawer()); + } + + public EglRenderer(String name, VideoFrameDrawer videoFrameDrawer) { + this.name = name; + this.frameDrawer = videoFrameDrawer; + } + + public void init( + EglThread eglThread, RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { + synchronized (threadLock) { + if (this.eglThread != null) { + throw new IllegalStateException(name + "Already initialized"); + } + + logD("Initializing EglRenderer"); + this.eglThread = eglThread; + this.drawer = drawer; + this.usePresentationTimeStamp = usePresentationTimeStamp; + + eglThread.addExceptionCallback(eglExceptionCallback); + + eglBase = eglThread.createEglBaseWithSharedConnection(); + eglThread.getHandler().post(eglSurfaceCreationRunnable); + + final long currentTimeNs = System.nanoTime(); + resetStatistics(currentTimeNs); + + eglThread.getHandler().postDelayed( + logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC)); + } + } + + /** + * Initialize this class, sharing resources with `sharedContext`. The custom `drawer` will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * `drawer`. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. If usePresentationTimeStamp is true, eglPresentationTimeANDROID will be + * set with the frame timestamps, which specifies desired presentation time and might be useful + * for e.g. syncing audio and video. + */ + public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { + EglThread thread = + EglThread.create(/* releaseMonitor= */ null, sharedContext, configAttributes); + init(thread, drawer, usePresentationTimeStamp); + } + + /** + * Same as above with usePresentationTimeStamp set to false. + * + * @see #init(EglBase.Context, int[], RendererCommon.GlDrawer, boolean) + */ + public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + init(sharedContext, configAttributes, drawer, /* usePresentationTimeStamp= */ false); + } + + public void createEglSurface(Surface surface) { + createEglSurfaceInternal(surface); + } + + public void createEglSurface(SurfaceTexture surfaceTexture) { + createEglSurfaceInternal(surfaceTexture); + } + + private void createEglSurfaceInternal(Object surface) { + eglSurfaceCreationRunnable.setSurface(surface); + postToRenderThread(eglSurfaceCreationRunnable); + } + + /** + * Block until any pending frame is returned and all GL resources released, even if an interrupt + * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function + * should be called before the Activity is destroyed and the EGLContext is still valid. If you + * don't call this function, the GL resources might leak. + */ + public void release() { + logD("Releasing."); + final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); + synchronized (threadLock) { + if (eglThread == null) { + logD("Already released"); + return; + } + eglThread.getHandler().removeCallbacks(logStatisticsRunnable); + eglThread.removeExceptionCallback(eglExceptionCallback); + + // Release EGL and GL resources on render thread. + eglThread.getHandler().postAtFrontOfQueue(() -> { + // Detach current shader program. + synchronized (EglBase.lock) { + GLES20.glUseProgram(/* program= */ 0); + } + if (drawer != null) { + drawer.release(); + drawer = null; + } + frameDrawer.release(); + bitmapTextureFramebuffer.release(); + + if (eglBase != null) { + logD("eglBase detach and release."); + eglBase.detachCurrent(); + eglBase.release(); + eglBase = null; + } + + frameListeners.clear(); + eglCleanupBarrier.countDown(); + }); + + // Don't accept any more frames or messages to the render thread. + eglThread.release(); + eglThread = null; + } + // Make sure the EGL/GL cleanup posted above is executed. + ThreadUtils.awaitUninterruptibly(eglCleanupBarrier); + synchronized (frameLock) { + if (pendingFrame != null) { + pendingFrame.release(); + pendingFrame = null; + } + } + logD("Releasing done."); + } + + /** + * Reset the statistics logged in logStatistics(). + */ + private void resetStatistics(long currentTimeNs) { + synchronized (statisticsLock) { + statisticsStartTimeNs = currentTimeNs; + framesReceived = 0; + framesDropped = 0; + framesRendered = 0; + renderTimeNs = 0; + renderSwapBufferTimeNs = 0; + } + } + + public void printStackTrace() { + synchronized (threadLock) { + final Thread renderThread = + (eglThread == null) ? null : eglThread.getHandler().getLooper().getThread(); + if (renderThread != null) { + final StackTraceElement[] renderStackTrace = renderThread.getStackTrace(); + if (renderStackTrace.length > 0) { + logW("EglRenderer stack trace:"); + for (StackTraceElement traceElem : renderStackTrace) { + logW(traceElem.toString()); + } + } + } + } + } + + /** + * Set if the video stream should be mirrored horizontally or not. + */ + public void setMirror(final boolean mirror) { + logD("setMirrorHorizontally: " + mirror); + synchronized (layoutLock) { + this.mirrorHorizontally = mirror; + } + } + + /** + * Set if the video stream should be mirrored vertically or not. + */ + public void setMirrorVertically(final boolean mirrorVertically) { + logD("setMirrorVertically: " + mirrorVertically); + synchronized (layoutLock) { + this.mirrorVertically = mirrorVertically; + } + } + + /** + * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video. + * Set this to 0 to disable cropping. + */ + public void setLayoutAspectRatio(float layoutAspectRatio) { + logD("setLayoutAspectRatio: " + layoutAspectRatio); + synchronized (layoutLock) { + this.layoutAspectRatio = layoutAspectRatio; + } + } + + /** + * Limit render framerate. + * + * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps + * reduction. + */ + public void setFpsReduction(float fps) { + logD("setFpsReduction: " + fps); + synchronized (fpsReductionLock) { + final long previousRenderPeriodNs = minRenderPeriodNs; + if (fps <= 0) { + minRenderPeriodNs = Long.MAX_VALUE; + } else { + minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps); + } + if (minRenderPeriodNs != previousRenderPeriodNs) { + // Fps reduction changed - reset frame time. + nextFrameTimeNs = System.nanoTime(); + } + } + } + + public void disableFpsReduction() { + setFpsReduction(Float.POSITIVE_INFINITY /* fps */); + } + + public void pauseVideo() { + setFpsReduction(0 /* fps */); + } + + /** + * Register a callback to be invoked when a new video frame has been received. This version uses + * the drawer of the EglRenderer that was passed in init. + * + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. + * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is + * required. + */ + public void addFrameListener(final FrameListener listener, final float scale) { + addFrameListener(listener, scale, null, false /* applyFpsReduction */); + } + + /** + * Register a callback to be invoked when a new video frame has been received. + * + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. + * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is + * required. + * @param drawer Custom drawer to use for this frame listener or null to use the default one. + */ + public void addFrameListener( + final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) { + addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */); + } + + /** + * Register a callback to be invoked when a new video frame has been received. + * + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. + * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is + * required. + * @param drawer Custom drawer to use for this frame listener or null to use the default one. + * @param applyFpsReduction This callback will not be called for frames that have been dropped by + * FPS reduction. + */ + public void addFrameListener(final FrameListener listener, final float scale, + @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) { + postToRenderThread(() -> { + final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam; + frameListeners.add( + new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction)); + }); + } + + /** + * Remove any pending callback that was added with addFrameListener. If the callback is not in + * the queue, nothing happens. It is ensured that callback won't be called after this method + * returns. + * + * @param runnable The callback to remove. + */ + public void removeFrameListener(final FrameListener listener) { + final CountDownLatch latch = new CountDownLatch(1); + synchronized (threadLock) { + if (eglThread == null) { + return; + } + if (Thread.currentThread() == eglThread.getHandler().getLooper().getThread()) { + throw new RuntimeException("removeFrameListener must not be called on the render thread."); + } + postToRenderThread(() -> { + latch.countDown(); + final Iterator<FrameListenerAndParams> iter = frameListeners.iterator(); + while (iter.hasNext()) { + if (iter.next().listener == listener) { + iter.remove(); + } + } + }); + } + ThreadUtils.awaitUninterruptibly(latch); + } + + /** Can be set in order to be notified about errors encountered during rendering. */ + public void setErrorCallback(ErrorCallback errorCallback) { + this.errorCallback = errorCallback; + } + + // VideoSink interface. + @Override + public void onFrame(VideoFrame frame) { + synchronized (statisticsLock) { + ++framesReceived; + } + final boolean dropOldFrame; + synchronized (threadLock) { + if (eglThread == null) { + logD("Dropping frame - Not initialized or already released."); + return; + } + synchronized (frameLock) { + dropOldFrame = (pendingFrame != null); + if (dropOldFrame) { + pendingFrame.release(); + } + pendingFrame = frame; + pendingFrame.retain(); + eglThread.getHandler().post(this::renderFrameOnRenderThread); + } + } + if (dropOldFrame) { + synchronized (statisticsLock) { + ++framesDropped; + } + } + } + + /** + * Release EGL surface. This function will block until the EGL surface is released. + */ + public void releaseEglSurface(final Runnable completionCallback) { + // Ensure that the render thread is no longer touching the Surface before returning from this + // function. + eglSurfaceCreationRunnable.setSurface(null /* surface */); + synchronized (threadLock) { + if (eglThread != null) { + eglThread.getHandler().removeCallbacks(eglSurfaceCreationRunnable); + eglThread.getHandler().postAtFrontOfQueue(() -> { + if (eglBase != null) { + eglBase.detachCurrent(); + eglBase.releaseSurface(); + } + completionCallback.run(); + }); + return; + } + } + completionCallback.run(); + } + + /** + * Private helper function to post tasks safely. + */ + private void postToRenderThread(Runnable runnable) { + synchronized (threadLock) { + if (eglThread != null) { + eglThread.getHandler().post(runnable); + } + } + } + + private void clearSurfaceOnRenderThread(float r, float g, float b, float a) { + if (eglBase != null && eglBase.hasSurface()) { + logD("clearSurface"); + eglBase.makeCurrent(); + GLES20.glClearColor(r, g, b, a); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + eglBase.swapBuffers(); + } + } + + /** + * Post a task to clear the surface to a transparent uniform color. + */ + public void clearImage() { + clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + } + + /** + * Post a task to clear the surface to a specific color. + */ + public void clearImage(final float r, final float g, final float b, final float a) { + synchronized (threadLock) { + if (eglThread == null) { + return; + } + eglThread.getHandler().postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a)); + } + } + + private void swapBuffersOnRenderThread(final VideoFrame frame, long swapBuffersStartTimeNs) { + synchronized (threadLock) { + if (eglThread != null) { + eglThread.scheduleRenderUpdate( + runsInline -> { + if (!runsInline) { + if (eglBase == null || !eglBase.hasSurface()) { + return; + } + eglBase.makeCurrent(); + } + + if (usePresentationTimeStamp) { + eglBase.swapBuffers(frame.getTimestampNs()); + } else { + eglBase.swapBuffers(); + } + + synchronized (statisticsLock) { + renderSwapBufferTimeNs += (System.nanoTime() - swapBuffersStartTimeNs); + } + }); + } + } + } + + /** + * Renders and releases `pendingFrame`. + */ + private void renderFrameOnRenderThread() { + // Fetch and render `pendingFrame`. + final VideoFrame frame; + synchronized (frameLock) { + if (pendingFrame == null) { + return; + } + frame = pendingFrame; + pendingFrame = null; + } + if (eglBase == null || !eglBase.hasSurface()) { + logD("Dropping frame - No surface"); + frame.release(); + return; + } + eglBase.makeCurrent(); + + // Check if fps reduction is active. + final boolean shouldRenderFrame; + synchronized (fpsReductionLock) { + if (minRenderPeriodNs == Long.MAX_VALUE) { + // Rendering is paused. + shouldRenderFrame = false; + } else if (minRenderPeriodNs <= 0) { + // FPS reduction is disabled. + shouldRenderFrame = true; + } else { + final long currentTimeNs = System.nanoTime(); + if (currentTimeNs < nextFrameTimeNs) { + logD("Skipping frame rendering - fps reduction is active."); + shouldRenderFrame = false; + } else { + nextFrameTimeNs += minRenderPeriodNs; + // The time for the next frame should always be in the future. + nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs); + shouldRenderFrame = true; + } + } + } + + final long startTimeNs = System.nanoTime(); + + final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight(); + final float drawnAspectRatio; + synchronized (layoutLock) { + drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio; + } + + final float scaleX; + final float scaleY; + + if (frameAspectRatio > drawnAspectRatio) { + scaleX = drawnAspectRatio / frameAspectRatio; + scaleY = 1f; + } else { + scaleX = 1f; + scaleY = frameAspectRatio / drawnAspectRatio; + } + + drawMatrix.reset(); + drawMatrix.preTranslate(0.5f, 0.5f); + drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); + drawMatrix.preScale(scaleX, scaleY); + drawMatrix.preTranslate(-0.5f, -0.5f); + + try { + if (shouldRenderFrame) { + GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */, + eglBase.surfaceWidth(), eglBase.surfaceHeight()); + + final long swapBuffersStartTimeNs = System.nanoTime(); + swapBuffersOnRenderThread(frame, swapBuffersStartTimeNs); + + synchronized (statisticsLock) { + ++framesRendered; + renderTimeNs += (swapBuffersStartTimeNs - startTimeNs); + } + } + + notifyCallbacks(frame, shouldRenderFrame); + } catch (GlUtil.GlOutOfMemoryException e) { + logE("Error while drawing frame", e); + final ErrorCallback errorCallback = this.errorCallback; + if (errorCallback != null) { + errorCallback.onGlOutOfMemory(); + } + // Attempt to free up some resources. + drawer.release(); + frameDrawer.release(); + bitmapTextureFramebuffer.release(); + // Continue here on purpose and retry again for next frame. In worst case, this is a + // continuous problem and no more frames will be drawn. + } finally { + frame.release(); + } + } + + private void notifyCallbacks(VideoFrame frame, boolean wasRendered) { + if (frameListeners.isEmpty()) + return; + + drawMatrix.reset(); + drawMatrix.preTranslate(0.5f, 0.5f); + drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); + drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap. + drawMatrix.preTranslate(-0.5f, -0.5f); + + Iterator<FrameListenerAndParams> it = frameListeners.iterator(); + while (it.hasNext()) { + FrameListenerAndParams listenerAndParams = it.next(); + if (!wasRendered && listenerAndParams.applyFpsReduction) { + continue; + } + it.remove(); + + final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth()); + final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight()); + + if (scaledWidth == 0 || scaledHeight == 0) { + listenerAndParams.listener.onFrame(null); + continue; + } + + bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight); + + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId()); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, + GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0); + + GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */, + 0 /* viewportY */, scaledWidth, scaledHeight); + + final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4); + GLES20.glViewport(0, 0, scaledWidth, scaledHeight); + GLES20.glReadPixels( + 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer); + + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks"); + + final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888); + bitmap.copyPixelsFromBuffer(bitmapBuffer); + listenerAndParams.listener.onFrame(bitmap); + } + } + + private String averageTimeAsString(long sumTimeNs, int count) { + return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " us"; + } + + private void logStatistics() { + final DecimalFormat fpsFormat = new DecimalFormat("#.0"); + final long currentTimeNs = System.nanoTime(); + synchronized (statisticsLock) { + final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs; + if (elapsedTimeNs <= 0 || (minRenderPeriodNs == Long.MAX_VALUE && framesReceived == 0)) { + return; + } + final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs; + logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms." + + " Frames received: " + framesReceived + "." + + " Dropped: " + framesDropped + "." + + " Rendered: " + framesRendered + "." + + " Render fps: " + fpsFormat.format(renderFps) + "." + + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "." + + " Average swapBuffer time: " + + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + "."); + resetStatistics(currentTimeNs); + } + } + + private void logE(String string, Throwable e) { + Logging.e(TAG, name + string, e); + } + + private void logD(String string) { + Logging.d(TAG, name + string); + } + + private void logW(String string) { + Logging.w(TAG, name + string); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EglThread.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglThread.java new file mode 100644 index 0000000000..73323d59c8 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EglThread.java @@ -0,0 +1,216 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.os.Message; +import androidx.annotation.GuardedBy; +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import org.webrtc.EglBase.EglConnection; + +/** EGL graphics thread that allows multiple clients to share the same underlying EGLContext. */ +public class EglThread implements RenderSynchronizer.Listener { + /** Callback for externally managed reference count. */ + public interface ReleaseMonitor { + /** + * Called by EglThread when a client releases its reference. Returns true when there are no more + * references and resources should be released. + */ + boolean onRelease(EglThread eglThread); + } + + /** Interface for clients to schedule rendering updates that will run synchronized. */ + public interface RenderUpdate { + + /** + * Called by EglThread when the rendering window is open. `runsInline` is true when the update + * is executed directly while the client schedules the update. + */ + void update(boolean runsInline); + } + + public static EglThread create( + @Nullable ReleaseMonitor releaseMonitor, + @Nullable final EglBase.Context sharedContext, + final int[] configAttributes, + @Nullable RenderSynchronizer renderSynchronizer) { + final HandlerThread renderThread = new HandlerThread("EglThread"); + renderThread.start(); + HandlerWithExceptionCallbacks handler = + new HandlerWithExceptionCallbacks(renderThread.getLooper()); + + // Not creating the EGLContext on the thread it will be used on seems to cause issues with + // creating window surfaces on certain devices. So keep the same legacy behavior as EglRenderer + // and create the context on the render thread. + EglConnection eglConnection = ThreadUtils.invokeAtFrontUninterruptibly(handler, () -> { + // If sharedContext is null, then texture frames are disabled. This is typically for old + // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has + // caused trouble on some weird devices. + if (sharedContext == null) { + return EglConnection.createEgl10(configAttributes); + } else { + return EglConnection.create(sharedContext, configAttributes); + } + }); + + return new EglThread( + releaseMonitor != null ? releaseMonitor : eglThread -> true, + handler, + eglConnection, + renderSynchronizer); + } + + public static EglThread create( + @Nullable ReleaseMonitor releaseMonitor, + @Nullable final EglBase.Context sharedContext, + final int[] configAttributes) { + return create(releaseMonitor, sharedContext, configAttributes, /* renderSynchronizer= */ null); + } + + /** + * Handler that triggers callbacks when an uncaught exception happens when handling a message. + */ + private static class HandlerWithExceptionCallbacks extends Handler { + private final Object callbackLock = new Object(); + @GuardedBy("callbackLock") private final List<Runnable> exceptionCallbacks = new ArrayList<>(); + + public HandlerWithExceptionCallbacks(Looper looper) { + super(looper); + } + + @Override + public void dispatchMessage(Message msg) { + try { + super.dispatchMessage(msg); + } catch (Exception e) { + Logging.e("EglThread", "Exception on EglThread", e); + synchronized (callbackLock) { + for (Runnable callback : exceptionCallbacks) { + callback.run(); + } + } + throw e; + } + } + + public void addExceptionCallback(Runnable callback) { + synchronized (callbackLock) { + exceptionCallbacks.add(callback); + } + } + + public void removeExceptionCallback(Runnable callback) { + synchronized (callbackLock) { + exceptionCallbacks.remove(callback); + } + } + } + + private final ReleaseMonitor releaseMonitor; + private final HandlerWithExceptionCallbacks handler; + private final EglConnection eglConnection; + private final RenderSynchronizer renderSynchronizer; + private final List<RenderUpdate> pendingRenderUpdates = new ArrayList<>(); + private boolean renderWindowOpen = true; + + private EglThread( + ReleaseMonitor releaseMonitor, + HandlerWithExceptionCallbacks handler, + EglConnection eglConnection, + RenderSynchronizer renderSynchronizer) { + this.releaseMonitor = releaseMonitor; + this.handler = handler; + this.eglConnection = eglConnection; + this.renderSynchronizer = renderSynchronizer; + if (renderSynchronizer != null) { + renderSynchronizer.registerListener(this); + } + } + + public void release() { + if (!releaseMonitor.onRelease(this)) { + // Thread is still in use, do not release yet. + return; + } + + if (renderSynchronizer != null) { + renderSynchronizer.removeListener(this); + } + + handler.post(eglConnection::release); + handler.getLooper().quitSafely(); + } + + /** + * Creates an EglBase instance with the EglThread's EglConnection. This method can be called on + * any thread, but the returned EglBase instance should only be used on this EglThread's Handler. + */ + public EglBase createEglBaseWithSharedConnection() { + return EglBase.create(eglConnection); + } + + /** + * Returns the Handler to interact with Gl/EGL on. Callers need to make sure that their own + * EglBase is current on the handler before running any graphics operations since the EglThread + * can be shared by multiple clients. + */ + public Handler getHandler() { + return handler; + } + + /** + * Adds a callback that will be called on the EGL thread if there is an exception on the thread. + */ + public void addExceptionCallback(Runnable callback) { + handler.addExceptionCallback(callback); + } + + /** + * Removes a previously added exception callback. + */ + public void removeExceptionCallback(Runnable callback) { + handler.removeExceptionCallback(callback); + } + + /** + * Schedules a render update (like swapBuffers) to be run in sync with other updates on the next + * open render window. If the render window is currently open the update will run immediately. + * This method must be called on the EglThread during a render pass. + */ + public void scheduleRenderUpdate(RenderUpdate update) { + if (renderWindowOpen) { + update.update(/* runsInline = */true); + } else { + pendingRenderUpdates.add(update); + } + } + + @Override + public void onRenderWindowOpen() { + handler.post( + () -> { + renderWindowOpen = true; + for (RenderUpdate update : pendingRenderUpdates) { + update.update(/* runsInline = */false); + } + pendingRenderUpdates.clear(); + }); + } + + @Override + public void onRenderWindowClose() { + handler.post(() -> renderWindowOpen = false); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/EncodedImage.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/EncodedImage.java new file mode 100644 index 0000000000..a6eef67da8 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/EncodedImage.java @@ -0,0 +1,183 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +/** + * An encoded frame from a video stream. Used as an input for decoders and as an output for + * encoders. + */ +public class EncodedImage implements RefCounted { + // Must be kept in sync with common_types.h FrameType. + public enum FrameType { + EmptyFrame(0), + VideoFrameKey(3), + VideoFrameDelta(4); + + private final int nativeIndex; + + private FrameType(int nativeIndex) { + this.nativeIndex = nativeIndex; + } + + public int getNative() { + return nativeIndex; + } + + @CalledByNative("FrameType") + static FrameType fromNativeIndex(int nativeIndex) { + for (FrameType type : FrameType.values()) { + if (type.getNative() == nativeIndex) { + return type; + } + } + throw new IllegalArgumentException("Unknown native frame type: " + nativeIndex); + } + } + + private final RefCountDelegate refCountDelegate; + public final ByteBuffer buffer; + public final int encodedWidth; + public final int encodedHeight; + public final long captureTimeMs; // Deprecated + public final long captureTimeNs; + public final FrameType frameType; + public final int rotation; + public final @Nullable Integer qp; + + // TODO(bugs.webrtc.org/9378): Use retain and release from jni code. + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @CalledByNative + private EncodedImage(ByteBuffer buffer, @Nullable Runnable releaseCallback, int encodedWidth, + int encodedHeight, long captureTimeNs, FrameType frameType, int rotation, + @Nullable Integer qp) { + this.buffer = buffer; + this.encodedWidth = encodedWidth; + this.encodedHeight = encodedHeight; + this.captureTimeMs = TimeUnit.NANOSECONDS.toMillis(captureTimeNs); + this.captureTimeNs = captureTimeNs; + this.frameType = frameType; + this.rotation = rotation; + this.qp = qp; + this.refCountDelegate = new RefCountDelegate(releaseCallback); + } + + @CalledByNative + private ByteBuffer getBuffer() { + return buffer; + } + + @CalledByNative + private int getEncodedWidth() { + return encodedWidth; + } + + @CalledByNative + private int getEncodedHeight() { + return encodedHeight; + } + + @CalledByNative + private long getCaptureTimeNs() { + return captureTimeNs; + } + + @CalledByNative + private int getFrameType() { + return frameType.getNative(); + } + + @CalledByNative + private int getRotation() { + return rotation; + } + + @CalledByNative + private @Nullable Integer getQp() { + return qp; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private ByteBuffer buffer; + private @Nullable Runnable releaseCallback; + private int encodedWidth; + private int encodedHeight; + private long captureTimeNs; + private EncodedImage.FrameType frameType; + private int rotation; + private @Nullable Integer qp; + + private Builder() {} + + public Builder setBuffer(ByteBuffer buffer, @Nullable Runnable releaseCallback) { + this.buffer = buffer; + this.releaseCallback = releaseCallback; + return this; + } + + public Builder setEncodedWidth(int encodedWidth) { + this.encodedWidth = encodedWidth; + return this; + } + + public Builder setEncodedHeight(int encodedHeight) { + this.encodedHeight = encodedHeight; + return this; + } + + @Deprecated + public Builder setCaptureTimeMs(long captureTimeMs) { + this.captureTimeNs = TimeUnit.MILLISECONDS.toNanos(captureTimeMs); + return this; + } + + public Builder setCaptureTimeNs(long captureTimeNs) { + this.captureTimeNs = captureTimeNs; + return this; + } + + public Builder setFrameType(EncodedImage.FrameType frameType) { + this.frameType = frameType; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public Builder setQp(@Nullable Integer qp) { + this.qp = qp; + return this; + } + + public EncodedImage createEncodedImage() { + return new EncodedImage(buffer, releaseCallback, encodedWidth, encodedHeight, captureTimeNs, + frameType, rotation, qp); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java new file mode 100644 index 0000000000..6d39390f72 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Factory for creating webrtc::FecControllerFactory instances. + */ +public interface FecControllerFactoryFactoryInterface { + /** + * Dynamically allocates a webrtc::FecControllerFactory instance and returns a pointer to it. + * The caller takes ownership of the object. + */ + public long createNative(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/FileVideoCapturer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/FileVideoCapturer.java new file mode 100644 index 0000000000..8270367970 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/FileVideoCapturer.java @@ -0,0 +1,201 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.os.SystemClock; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + +public class FileVideoCapturer implements VideoCapturer { + private interface VideoReader { + VideoFrame getNextFrame(); + void close(); + } + + /** + * Read video data from file for the .y4m container. + */ + @SuppressWarnings("StringSplitter") + private static class VideoReaderY4M implements VideoReader { + private static final String TAG = "VideoReaderY4M"; + private static final String Y4M_FRAME_DELIMETER = "FRAME"; + private static final int FRAME_DELIMETER_LENGTH = Y4M_FRAME_DELIMETER.length() + 1; + + private final int frameWidth; + private final int frameHeight; + // First char after header + private final long videoStart; + private final RandomAccessFile mediaFile; + private final FileChannel mediaFileChannel; + + public VideoReaderY4M(String file) throws IOException { + mediaFile = new RandomAccessFile(file, "r"); + mediaFileChannel = mediaFile.getChannel(); + StringBuilder builder = new StringBuilder(); + for (;;) { + int c = mediaFile.read(); + if (c == -1) { + // End of file reached. + throw new RuntimeException("Found end of file before end of header for file: " + file); + } + if (c == '\n') { + // End of header found. + break; + } + builder.append((char) c); + } + videoStart = mediaFileChannel.position(); + String header = builder.toString(); + String[] headerTokens = header.split("[ ]"); + int w = 0; + int h = 0; + String colorSpace = ""; + for (String tok : headerTokens) { + char c = tok.charAt(0); + switch (c) { + case 'W': + w = Integer.parseInt(tok.substring(1)); + break; + case 'H': + h = Integer.parseInt(tok.substring(1)); + break; + case 'C': + colorSpace = tok.substring(1); + break; + } + } + Logging.d(TAG, "Color space: " + colorSpace); + if (!colorSpace.equals("420") && !colorSpace.equals("420mpeg2")) { + throw new IllegalArgumentException( + "Does not support any other color space than I420 or I420mpeg2"); + } + if ((w % 2) == 1 || (h % 2) == 1) { + throw new IllegalArgumentException("Does not support odd width or height"); + } + frameWidth = w; + frameHeight = h; + Logging.d(TAG, "frame dim: (" + w + ", " + h + ")"); + } + + @Override + public VideoFrame getNextFrame() { + final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); + final JavaI420Buffer buffer = JavaI420Buffer.allocate(frameWidth, frameHeight); + final ByteBuffer dataY = buffer.getDataY(); + final ByteBuffer dataU = buffer.getDataU(); + final ByteBuffer dataV = buffer.getDataV(); + final int chromaHeight = (frameHeight + 1) / 2; + final int sizeY = frameHeight * buffer.getStrideY(); + final int sizeU = chromaHeight * buffer.getStrideU(); + final int sizeV = chromaHeight * buffer.getStrideV(); + + try { + ByteBuffer frameDelim = ByteBuffer.allocate(FRAME_DELIMETER_LENGTH); + if (mediaFileChannel.read(frameDelim) < FRAME_DELIMETER_LENGTH) { + // We reach end of file, loop + mediaFileChannel.position(videoStart); + if (mediaFileChannel.read(frameDelim) < FRAME_DELIMETER_LENGTH) { + throw new RuntimeException("Error looping video"); + } + } + String frameDelimStr = new String(frameDelim.array(), Charset.forName("US-ASCII")); + if (!frameDelimStr.equals(Y4M_FRAME_DELIMETER + "\n")) { + throw new RuntimeException( + "Frames should be delimited by FRAME plus newline, found delimter was: '" + + frameDelimStr + "'"); + } + + mediaFileChannel.read(dataY); + mediaFileChannel.read(dataU); + mediaFileChannel.read(dataV); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return new VideoFrame(buffer, 0 /* rotation */, captureTimeNs); + } + + @Override + public void close() { + try { + // Closing a file also closes the channel. + mediaFile.close(); + } catch (IOException e) { + Logging.e(TAG, "Problem closing file", e); + } + } + } + + private final static String TAG = "FileVideoCapturer"; + private final VideoReader videoReader; + private CapturerObserver capturerObserver; + private final Timer timer = new Timer(); + + private final TimerTask tickTask = new TimerTask() { + @Override + public void run() { + tick(); + } + }; + + public FileVideoCapturer(String inputFile) throws IOException { + try { + videoReader = new VideoReaderY4M(inputFile); + } catch (IOException e) { + Logging.d(TAG, "Could not open video file: " + inputFile); + throw e; + } + } + + public void tick() { + VideoFrame videoFrame = videoReader.getNextFrame(); + capturerObserver.onFrameCaptured(videoFrame); + videoFrame.release(); + } + + @Override + public void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext, + CapturerObserver capturerObserver) { + this.capturerObserver = capturerObserver; + } + + @Override + public void startCapture(int width, int height, int framerate) { + timer.schedule(tickTask, 0, 1000 / framerate); + } + + @Override + public void stopCapture() throws InterruptedException { + timer.cancel(); + } + + @Override + public void changeCaptureFormat(int width, int height, int framerate) { + // Empty on purpose + } + + @Override + public void dispose() { + videoReader.close(); + } + + @Override + public boolean isScreencast() { + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameDecryptor.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameDecryptor.java new file mode 100644 index 0000000000..2932f3d94a --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameDecryptor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * The FrameDecryptor interface allows Java API users to provide a + * pointer to their native implementation of the FrameDecryptorInterface. + * FrameDecryptors are extremely performance sensitive as they must process all + * incoming video and audio frames. Due to this reason they should always be + * backed by a native implementation + * @note Not ready for production use. + */ +public interface FrameDecryptor { + /** + * @return A FrameDecryptorInterface pointer. + */ + long getNativeFrameDecryptor(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameEncryptor.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameEncryptor.java new file mode 100644 index 0000000000..bc81223f21 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/FrameEncryptor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * The FrameEncryptor interface allows Java API users to provide a pointer to + * their native implementation of the FrameEncryptorInterface. + * FrameEncyptors are extremely performance sensitive as they must process all + * outgoing video and audio frames. Due to this reason they should always be + * backed by a native implementation. + * @note Not ready for production use. + */ +public interface FrameEncryptor { + /** + * @return A FrameEncryptorInterface pointer. + */ + long getNativeFrameEncryptor(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/GlRectDrawer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlRectDrawer.java new file mode 100644 index 0000000000..d1fbd1b7bc --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlRectDrawer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Simplest possible GL shader that just draws frames as opaque quads. */ +public class GlRectDrawer extends GlGenericDrawer { + private static final String FRAGMENT_SHADER = "void main() {\n" + + " gl_FragColor = sample(tc);\n" + + "}\n"; + + private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks { + @Override + public void onNewShader(GlShader shader) {} + + @Override + public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, + int viewportWidth, int viewportHeight) {} + } + + public GlRectDrawer() { + super(FRAGMENT_SHADER, new ShaderCallbacks()); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/GlShader.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlShader.java new file mode 100644 index 0000000000..7efd8d3a95 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlShader.java @@ -0,0 +1,131 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.opengl.GLES20; + +import java.nio.FloatBuffer; + +// Helper class for handling OpenGL shaders and shader programs. +public class GlShader { + private static final String TAG = "GlShader"; + + private static int compileShader(int shaderType, String source) { + final int shader = GLES20.glCreateShader(shaderType); + if (shader == 0) { + throw new RuntimeException("glCreateShader() failed. GLES20 error: " + GLES20.glGetError()); + } + GLES20.glShaderSource(shader, source); + GLES20.glCompileShader(shader); + int[] compileStatus = new int[] {GLES20.GL_FALSE}; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compileStatus, 0); + if (compileStatus[0] != GLES20.GL_TRUE) { + Logging.e( + TAG, "Compile error " + GLES20.glGetShaderInfoLog(shader) + " in shader:\n" + source); + throw new RuntimeException(GLES20.glGetShaderInfoLog(shader)); + } + GlUtil.checkNoGLES2Error("compileShader"); + return shader; + } + + private int program; + + public GlShader(String vertexSource, String fragmentSource) { + final int vertexShader = compileShader(GLES20.GL_VERTEX_SHADER, vertexSource); + final int fragmentShader = compileShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); + program = GLES20.glCreateProgram(); + if (program == 0) { + throw new RuntimeException("glCreateProgram() failed. GLES20 error: " + GLES20.glGetError()); + } + GLES20.glAttachShader(program, vertexShader); + GLES20.glAttachShader(program, fragmentShader); + GLES20.glLinkProgram(program); + int[] linkStatus = new int[] {GLES20.GL_FALSE}; + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); + if (linkStatus[0] != GLES20.GL_TRUE) { + Logging.e(TAG, "Could not link program: " + GLES20.glGetProgramInfoLog(program)); + throw new RuntimeException(GLES20.glGetProgramInfoLog(program)); + } + // According to the documentation of glLinkProgram(): + // "After the link operation, applications are free to modify attached shader objects, compile + // attached shader objects, detach shader objects, delete shader objects, and attach additional + // shader objects. None of these operations affects the information log or the program that is + // part of the program object." + // But in practice, detaching shaders from the program seems to break some devices. Deleting the + // shaders are fine however - it will delete them when they are no longer attached to a program. + GLES20.glDeleteShader(vertexShader); + GLES20.glDeleteShader(fragmentShader); + GlUtil.checkNoGLES2Error("Creating GlShader"); + } + + public int getAttribLocation(String label) { + if (program == -1) { + throw new RuntimeException("The program has been released"); + } + int location = GLES20.glGetAttribLocation(program, label); + if (location < 0) { + throw new RuntimeException("Could not locate '" + label + "' in program"); + } + return location; + } + + /** + * Enable and upload a vertex array for attribute `label`. The vertex data is specified in + * `buffer` with `dimension` number of components per vertex. + */ + public void setVertexAttribArray(String label, int dimension, FloatBuffer buffer) { + setVertexAttribArray(label, dimension, 0 /* stride */, buffer); + } + + /** + * Enable and upload a vertex array for attribute `label`. The vertex data is specified in + * `buffer` with `dimension` number of components per vertex and specified `stride`. + */ + public void setVertexAttribArray(String label, int dimension, int stride, FloatBuffer buffer) { + if (program == -1) { + throw new RuntimeException("The program has been released"); + } + int location = getAttribLocation(label); + GLES20.glEnableVertexAttribArray(location); + GLES20.glVertexAttribPointer(location, dimension, GLES20.GL_FLOAT, false, stride, buffer); + GlUtil.checkNoGLES2Error("setVertexAttribArray"); + } + + public int getUniformLocation(String label) { + if (program == -1) { + throw new RuntimeException("The program has been released"); + } + int location = GLES20.glGetUniformLocation(program, label); + if (location < 0) { + throw new RuntimeException("Could not locate uniform '" + label + "' in program"); + } + return location; + } + + public void useProgram() { + if (program == -1) { + throw new RuntimeException("The program has been released"); + } + synchronized (EglBase.lock) { + GLES20.glUseProgram(program); + } + GlUtil.checkNoGLES2Error("glUseProgram"); + } + + public void release() { + Logging.d(TAG, "Deleting shader."); + // Delete program, automatically detaching any shaders from it. + if (program != -1) { + GLES20.glDeleteProgram(program); + program = -1; + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/GlTextureFrameBuffer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlTextureFrameBuffer.java new file mode 100644 index 0000000000..b906fe56e0 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlTextureFrameBuffer.java @@ -0,0 +1,122 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.opengl.GLES20; + +/** + * Helper class for handling OpenGL framebuffer with only color attachment and no depth or stencil + * buffer. Intended for simple tasks such as texture copy, texture downscaling, and texture color + * conversion. This class is not thread safe and must be used by a thread with an active GL context. + */ +// TODO(magjed): Add unittests for this class. +public class GlTextureFrameBuffer { + private final int pixelFormat; + private int frameBufferId; + private int textureId; + private int width; + private int height; + + /** + * Generate texture and framebuffer resources. An EGLContext must be bound on the current thread + * when calling this function. The framebuffer is not complete until setSize() is called. + */ + public GlTextureFrameBuffer(int pixelFormat) { + switch (pixelFormat) { + case GLES20.GL_LUMINANCE: + case GLES20.GL_RGB: + case GLES20.GL_RGBA: + this.pixelFormat = pixelFormat; + break; + default: + throw new IllegalArgumentException("Invalid pixel format: " + pixelFormat); + } + this.width = 0; + this.height = 0; + } + + /** + * (Re)allocate texture. Will do nothing if the requested size equals the current size. An + * EGLContext must be bound on the current thread when calling this function. Must be called at + * least once before using the framebuffer. May be called multiple times to change size. + */ + public void setSize(int width, int height) { + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("Invalid size: " + width + "x" + height); + } + if (width == this.width && height == this.height) { + return; + } + this.width = width; + this.height = height; + // Lazy allocation the first time setSize() is called. + if (textureId == 0) { + textureId = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); + } + if (frameBufferId == 0) { + final int frameBuffers[] = new int[1]; + GLES20.glGenFramebuffers(1, frameBuffers, 0); + frameBufferId = frameBuffers[0]; + } + + // Allocate texture. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, pixelFormat, width, height, 0, pixelFormat, + GLES20.GL_UNSIGNED_BYTE, null); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + GlUtil.checkNoGLES2Error("GlTextureFrameBuffer setSize"); + + // Attach the texture to the framebuffer as color attachment. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBufferId); + GLES20.glFramebufferTexture2D( + GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, textureId, 0); + + // Check that the framebuffer is in a good state. + final int status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER); + if (status != GLES20.GL_FRAMEBUFFER_COMPLETE) { + throw new IllegalStateException("Framebuffer not complete, status: " + status); + } + + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + /** Gets the OpenGL frame buffer id. This value is only valid after setSize() has been called. */ + public int getFrameBufferId() { + return frameBufferId; + } + + /** Gets the OpenGL texture id. This value is only valid after setSize() has been called. */ + public int getTextureId() { + return textureId; + } + + /** + * Release texture and framebuffer. An EGLContext must be bound on the current thread when calling + * this function. This object should not be used after this call. + */ + public void release() { + GLES20.glDeleteTextures(1, new int[] {textureId}, 0); + textureId = 0; + GLES20.glDeleteFramebuffers(1, new int[] {frameBufferId}, 0); + frameBufferId = 0; + width = 0; + height = 0; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/GlUtil.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlUtil.java new file mode 100644 index 0000000000..e2dd0c56d6 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/GlUtil.java @@ -0,0 +1,66 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.opengl.GLES20; +import android.opengl.GLException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; + +/** + * Some OpenGL static utility functions. + */ +public class GlUtil { + private GlUtil() {} + + public static class GlOutOfMemoryException extends GLException { + public GlOutOfMemoryException(int error, String msg) { + super(error, msg); + } + } + + // Assert that no OpenGL ES 2.0 error has been raised. + public static void checkNoGLES2Error(String msg) { + int error = GLES20.glGetError(); + if (error != GLES20.GL_NO_ERROR) { + throw error == GLES20.GL_OUT_OF_MEMORY + ? new GlOutOfMemoryException(error, msg) + : new GLException(error, msg + ": GLES20 error: " + error); + } + } + + public static FloatBuffer createFloatBuffer(float[] coords) { + // Allocate a direct ByteBuffer, using 4 bytes per float, and copy coords into it. + ByteBuffer bb = ByteBuffer.allocateDirect(coords.length * 4); + bb.order(ByteOrder.nativeOrder()); + FloatBuffer fb = bb.asFloatBuffer(); + fb.put(coords); + fb.position(0); + return fb; + } + + /** + * Generate texture with standard parameters. + */ + public static int generateTexture(int target) { + final int textureArray[] = new int[1]; + GLES20.glGenTextures(1, textureArray, 0); + final int textureId = textureArray[0]; + GLES20.glBindTexture(target, textureId); + GLES20.glTexParameterf(target, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf(target, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf(target, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameterf(target, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + checkNoGLES2Error("generateTexture"); + return textureId; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java new file mode 100644 index 0000000000..215598a85d --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaCodecInfo; +import androidx.annotation.Nullable; +import java.util.Arrays; + +/** Factory for Android hardware VideoDecoders. */ +public class HardwareVideoDecoderFactory extends MediaCodecVideoDecoderFactory { + private final static Predicate<MediaCodecInfo> defaultAllowedPredicate = + new Predicate<MediaCodecInfo>() { + @Override + public boolean test(MediaCodecInfo arg) { + return MediaCodecUtils.isHardwareAccelerated(arg); + } + }; + + /** Creates a HardwareVideoDecoderFactory that does not use surface textures. */ + @Deprecated // Not removed yet to avoid breaking callers. + public HardwareVideoDecoderFactory() { + this(null); + } + + /** + * Creates a HardwareVideoDecoderFactory that supports surface texture rendering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + */ + public HardwareVideoDecoderFactory(@Nullable EglBase.Context sharedContext) { + this(sharedContext, /* codecAllowedPredicate= */ null); + } + + /** + * Creates a HardwareVideoDecoderFactory that supports surface texture rendering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + * @param codecAllowedPredicate predicate to filter codecs. It is combined with the default + * predicate that only allows hardware codecs. + */ + public HardwareVideoDecoderFactory(@Nullable EglBase.Context sharedContext, + @Nullable Predicate<MediaCodecInfo> codecAllowedPredicate) { + super(sharedContext, + (codecAllowedPredicate == null ? defaultAllowedPredicate + : codecAllowedPredicate.and(defaultAllowedPredicate))); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/IceCandidateErrorEvent.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/IceCandidateErrorEvent.java new file mode 100644 index 0000000000..aae9da7061 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/IceCandidateErrorEvent.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public final class IceCandidateErrorEvent { + /** The local IP address used to communicate with the STUN or TURN server. */ + public final String address; + /** The port used to communicate with the STUN or TURN server. */ + public final int port; + /** + * The STUN or TURN URL that identifies the STUN or TURN server for which the failure occurred. + */ + public final String url; + /** + * The numeric STUN error code returned by the STUN or TURN server. If no host candidate can reach + * the server, errorCode will be set to the value 701 which is outside the STUN error code range. + * This error is only fired once per server URL while in the RTCIceGatheringState of "gathering". + */ + public final int errorCode; + /** + * The STUN reason text returned by the STUN or TURN server. If the server could not be reached, + * errorText will be set to an implementation-specific value providing details about the error. + */ + public final String errorText; + + @CalledByNative + public IceCandidateErrorEvent( + String address, int port, String url, int errorCode, String errorText) { + this.address = address; + this.port = port; + this.url = url; + this.errorCode = errorCode; + this.errorText = errorText; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/JavaI420Buffer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/JavaI420Buffer.java new file mode 100644 index 0000000000..322b8f38c9 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/JavaI420Buffer.java @@ -0,0 +1,200 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import org.webrtc.VideoFrame.I420Buffer; + +/** Implementation of VideoFrame.I420Buffer backed by Java direct byte buffers. */ +public class JavaI420Buffer implements VideoFrame.I420Buffer { + private final int width; + private final int height; + private final ByteBuffer dataY; + private final ByteBuffer dataU; + private final ByteBuffer dataV; + private final int strideY; + private final int strideU; + private final int strideV; + private final RefCountDelegate refCountDelegate; + + private JavaI420Buffer(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU, + int strideU, ByteBuffer dataV, int strideV, @Nullable Runnable releaseCallback) { + this.width = width; + this.height = height; + this.dataY = dataY; + this.dataU = dataU; + this.dataV = dataV; + this.strideY = strideY; + this.strideU = strideU; + this.strideV = strideV; + this.refCountDelegate = new RefCountDelegate(releaseCallback); + } + + private static void checkCapacity(ByteBuffer data, int width, int height, int stride) { + // The last row does not necessarily need padding. + final int minCapacity = stride * (height - 1) + width; + if (data.capacity() < minCapacity) { + throw new IllegalArgumentException( + "Buffer must be at least " + minCapacity + " bytes, but was " + data.capacity()); + } + } + + /** Wraps existing ByteBuffers into JavaI420Buffer object without copying the contents. */ + public static JavaI420Buffer wrap(int width, int height, ByteBuffer dataY, int strideY, + ByteBuffer dataU, int strideU, ByteBuffer dataV, int strideV, + @Nullable Runnable releaseCallback) { + if (dataY == null || dataU == null || dataV == null) { + throw new IllegalArgumentException("Data buffers cannot be null."); + } + if (!dataY.isDirect() || !dataU.isDirect() || !dataV.isDirect()) { + throw new IllegalArgumentException("Data buffers must be direct byte buffers."); + } + + // Slice the buffers to prevent external modifications to the position / limit of the buffer. + // Note that this doesn't protect the contents of the buffers from modifications. + dataY = dataY.slice(); + dataU = dataU.slice(); + dataV = dataV.slice(); + + final int chromaWidth = (width + 1) / 2; + final int chromaHeight = (height + 1) / 2; + checkCapacity(dataY, width, height, strideY); + checkCapacity(dataU, chromaWidth, chromaHeight, strideU); + checkCapacity(dataV, chromaWidth, chromaHeight, strideV); + + return new JavaI420Buffer( + width, height, dataY, strideY, dataU, strideU, dataV, strideV, releaseCallback); + } + + /** Allocates an empty I420Buffer suitable for an image of the given dimensions. */ + public static JavaI420Buffer allocate(int width, int height) { + int chromaHeight = (height + 1) / 2; + int strideUV = (width + 1) / 2; + int yPos = 0; + int uPos = yPos + width * height; + int vPos = uPos + strideUV * chromaHeight; + + ByteBuffer buffer = + JniCommon.nativeAllocateByteBuffer(width * height + 2 * strideUV * chromaHeight); + + buffer.position(yPos); + buffer.limit(uPos); + ByteBuffer dataY = buffer.slice(); + + buffer.position(uPos); + buffer.limit(vPos); + ByteBuffer dataU = buffer.slice(); + + buffer.position(vPos); + buffer.limit(vPos + strideUV * chromaHeight); + ByteBuffer dataV = buffer.slice(); + + return new JavaI420Buffer(width, height, dataY, width, dataU, strideUV, dataV, strideUV, + () -> { JniCommon.nativeFreeByteBuffer(buffer); }); + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public ByteBuffer getDataY() { + // Return a slice to prevent relative reads from changing the position. + return dataY.slice(); + } + + @Override + public ByteBuffer getDataU() { + // Return a slice to prevent relative reads from changing the position. + return dataU.slice(); + } + + @Override + public ByteBuffer getDataV() { + // Return a slice to prevent relative reads from changing the position. + return dataV.slice(); + } + + @Override + public int getStrideY() { + return strideY; + } + + @Override + public int getStrideU() { + return strideU; + } + + @Override + public int getStrideV() { + return strideV; + } + + @Override + public I420Buffer toI420() { + retain(); + return this; + } + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return cropAndScaleI420(this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + } + + public static VideoFrame.Buffer cropAndScaleI420(final I420Buffer buffer, int cropX, int cropY, + int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + if (cropWidth == scaleWidth && cropHeight == scaleHeight) { + // No scaling. + ByteBuffer dataY = buffer.getDataY(); + ByteBuffer dataU = buffer.getDataU(); + ByteBuffer dataV = buffer.getDataV(); + + dataY.position(cropX + cropY * buffer.getStrideY()); + dataU.position(cropX / 2 + cropY / 2 * buffer.getStrideU()); + dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV()); + + buffer.retain(); + return JavaI420Buffer.wrap(scaleWidth, scaleHeight, dataY.slice(), buffer.getStrideY(), + dataU.slice(), buffer.getStrideU(), dataV.slice(), buffer.getStrideV(), buffer::release); + } + + JavaI420Buffer newBuffer = JavaI420Buffer.allocate(scaleWidth, scaleHeight); + nativeCropAndScaleI420(buffer.getDataY(), buffer.getStrideY(), buffer.getDataU(), + buffer.getStrideU(), buffer.getDataV(), buffer.getStrideV(), cropX, cropY, cropWidth, + cropHeight, newBuffer.getDataY(), newBuffer.getStrideY(), newBuffer.getDataU(), + newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV(), scaleWidth, + scaleHeight); + return newBuffer; + } + + private static native void nativeCropAndScaleI420(ByteBuffer srcY, int srcStrideY, + ByteBuffer srcU, int srcStrideU, ByteBuffer srcV, int srcStrideV, int cropX, int cropY, + int cropWidth, int cropHeight, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU, + int dstStrideU, ByteBuffer dstV, int dstStrideV, int scaleWidth, int scaleHeight); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibaomAv1Encoder.java new file mode 100644 index 0000000000..569a719f44 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class LibaomAv1Encoder extends WrappedNativeVideoEncoder { + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(); + } + + static native long nativeCreateEncoder(); + + @Override + public boolean isHardwareEncoder() { + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Decoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Decoder.java new file mode 100644 index 0000000000..54ad0aa137 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Decoder.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class LibvpxVp8Decoder extends WrappedNativeVideoDecoder { + @Override + public long createNativeVideoDecoder() { + return nativeCreateDecoder(); + } + + static native long nativeCreateDecoder(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java new file mode 100644 index 0000000000..4be9e52c14 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp8Encoder.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class LibvpxVp8Encoder extends WrappedNativeVideoEncoder { + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(); + } + + static native long nativeCreateEncoder(); + + @Override + public boolean isHardwareEncoder() { + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Decoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Decoder.java new file mode 100644 index 0000000000..90a24433a3 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Decoder.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class LibvpxVp9Decoder extends WrappedNativeVideoDecoder { + @Override + public long createNativeVideoDecoder() { + return nativeCreateDecoder(); + } + + static native long nativeCreateDecoder(); + + static native boolean nativeIsSupported(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java new file mode 100644 index 0000000000..1211ae93fb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder { + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(); + } + + static native long nativeCreateEncoder(); + + @Override + public boolean isHardwareEncoder() { + return false; + } + + static native boolean nativeIsSupported(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaConstraints.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaConstraints.java new file mode 100644 index 0000000000..bae04e532c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaConstraints.java @@ -0,0 +1,99 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +/** + * Description of media constraints for {@code MediaStream} and + * {@code PeerConnection}. + */ +public class MediaConstraints { + /** Simple String key/value pair. */ + public static class KeyValuePair { + private final String key; + private final String value; + + public KeyValuePair(String key, String value) { + this.key = key; + this.value = value; + } + + @CalledByNative("KeyValuePair") + public String getKey() { + return key; + } + + @CalledByNative("KeyValuePair") + public String getValue() { + return value; + } + + @Override + public String toString() { + return key + ": " + value; + } + + @Override + public boolean equals(@Nullable Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + KeyValuePair that = (KeyValuePair) other; + return key.equals(that.key) && value.equals(that.value); + } + + @Override + public int hashCode() { + return key.hashCode() + value.hashCode(); + } + } + + public final List<KeyValuePair> mandatory; + public final List<KeyValuePair> optional; + + public MediaConstraints() { + mandatory = new ArrayList<KeyValuePair>(); + optional = new ArrayList<KeyValuePair>(); + } + + private static String stringifyKeyValuePairList(List<KeyValuePair> list) { + StringBuilder builder = new StringBuilder("["); + for (KeyValuePair pair : list) { + if (builder.length() > 1) { + builder.append(", "); + } + builder.append(pair.toString()); + } + return builder.append("]").toString(); + } + + @Override + public String toString() { + return "mandatory: " + stringifyKeyValuePairList(mandatory) + ", optional: " + + stringifyKeyValuePairList(optional); + } + + @CalledByNative + List<KeyValuePair> getMandatory() { + return mandatory; + } + + @CalledByNative + List<KeyValuePair> getOptional() { + return optional; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaSource.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaSource.java new file mode 100644 index 0000000000..9245e3e2eb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaSource.java @@ -0,0 +1,74 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Java wrapper for a C++ MediaSourceInterface. */ +public class MediaSource { + /** Tracks MediaSourceInterface.SourceState */ + public enum State { + INITIALIZING, + LIVE, + ENDED, + MUTED; + + @CalledByNative("State") + static State fromNativeIndex(int nativeIndex) { + return values()[nativeIndex]; + } + } + + private final RefCountDelegate refCountDelegate; + private long nativeSource; + + public MediaSource(long nativeSource) { + refCountDelegate = new RefCountDelegate(() -> JniCommon.nativeReleaseRef(nativeSource)); + this.nativeSource = nativeSource; + } + + public State state() { + checkMediaSourceExists(); + return nativeGetState(nativeSource); + } + + public void dispose() { + checkMediaSourceExists(); + refCountDelegate.release(); + nativeSource = 0; + } + + /** Returns a pointer to webrtc::MediaSourceInterface. */ + protected long getNativeMediaSource() { + checkMediaSourceExists(); + return nativeSource; + } + + /** + * Runs code in {@code runnable} holding a reference to the media source. If the object has + * already been released, does nothing. + */ + void runWithReference(Runnable runnable) { + if (refCountDelegate.safeRetain()) { + try { + runnable.run(); + } finally { + refCountDelegate.release(); + } + } + } + + private void checkMediaSourceExists() { + if (nativeSource == 0) { + throw new IllegalStateException("MediaSource has been disposed."); + } + } + + private static native State nativeGetState(long pointer); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaStreamTrack.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaStreamTrack.java new file mode 100644 index 0000000000..2e4c3e18f7 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/MediaStreamTrack.java @@ -0,0 +1,129 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +/** Java wrapper for a C++ MediaStreamTrackInterface. */ +public class MediaStreamTrack { + public static final String AUDIO_TRACK_KIND = "audio"; + public static final String VIDEO_TRACK_KIND = "video"; + + /** Tracks MediaStreamTrackInterface.TrackState */ + public enum State { + LIVE, + ENDED; + + @CalledByNative("State") + static State fromNativeIndex(int nativeIndex) { + return values()[nativeIndex]; + } + } + + // Must be kept in sync with cricket::MediaType. + public enum MediaType { + MEDIA_TYPE_AUDIO(0), + MEDIA_TYPE_VIDEO(1); + + private final int nativeIndex; + + private MediaType(int nativeIndex) { + this.nativeIndex = nativeIndex; + } + + @CalledByNative("MediaType") + int getNative() { + return nativeIndex; + } + + @CalledByNative("MediaType") + static MediaType fromNativeIndex(int nativeIndex) { + for (MediaType type : MediaType.values()) { + if (type.getNative() == nativeIndex) { + return type; + } + } + throw new IllegalArgumentException("Unknown native media type: " + nativeIndex); + } + } + + /** Factory method to create an AudioTrack or VideoTrack subclass. */ + static @Nullable MediaStreamTrack createMediaStreamTrack(long nativeTrack) { + if (nativeTrack == 0) { + return null; + } + String trackKind = nativeGetKind(nativeTrack); + if (trackKind.equals(AUDIO_TRACK_KIND)) { + return new AudioTrack(nativeTrack); + } else if (trackKind.equals(VIDEO_TRACK_KIND)) { + return new VideoTrack(nativeTrack); + } else { + return null; + } + } + + private long nativeTrack; + + public MediaStreamTrack(long nativeTrack) { + if (nativeTrack == 0) { + throw new IllegalArgumentException("nativeTrack may not be null"); + } + this.nativeTrack = nativeTrack; + } + + public String id() { + checkMediaStreamTrackExists(); + return nativeGetId(nativeTrack); + } + + public String kind() { + checkMediaStreamTrackExists(); + return nativeGetKind(nativeTrack); + } + + public boolean enabled() { + checkMediaStreamTrackExists(); + return nativeGetEnabled(nativeTrack); + } + + public boolean setEnabled(boolean enable) { + checkMediaStreamTrackExists(); + return nativeSetEnabled(nativeTrack, enable); + } + + public State state() { + checkMediaStreamTrackExists(); + return nativeGetState(nativeTrack); + } + + public void dispose() { + checkMediaStreamTrackExists(); + JniCommon.nativeReleaseRef(nativeTrack); + nativeTrack = 0; + } + + long getNativeMediaStreamTrack() { + checkMediaStreamTrackExists(); + return nativeTrack; + } + + private void checkMediaStreamTrackExists() { + if (nativeTrack == 0) { + throw new IllegalStateException("MediaStreamTrack has been disposed."); + } + } + + private static native String nativeGetId(long track); + private static native String nativeGetKind(long track); + private static native boolean nativeGetEnabled(long track); + private static native boolean nativeSetEnabled(long track, boolean enabled); + private static native State nativeGetState(long track); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Metrics.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Metrics.java new file mode 100644 index 0000000000..253376831c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Metrics.java @@ -0,0 +1,81 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.HashMap; +import java.util.Map; + +// Java-side of androidmetrics.cc +// +// Rtc histograms can be queried through the API, getAndReset(). +// The returned map holds the name of a histogram and its samples. +// +// Example of `map` with one histogram: +// `name`: "WebRTC.Video.InputFramesPerSecond" +// `min`: 1 +// `max`: 100 +// `bucketCount`: 50 +// `samples`: [30]:1 +// +// Most histograms are not updated frequently (e.g. most video metrics are an +// average over the call and recorded when a stream is removed). +// The metrics can for example be retrieved when a peer connection is closed. +public class Metrics { + private static final String TAG = "Metrics"; + + public final Map<String, HistogramInfo> map = + new HashMap<String, HistogramInfo>(); // <name, HistogramInfo> + + @CalledByNative + Metrics() {} + + /** + * Class holding histogram information. + */ + public static class HistogramInfo { + public final int min; + public final int max; + public final int bucketCount; + public final Map<Integer, Integer> samples = + new HashMap<Integer, Integer>(); // <value, # of events> + + @CalledByNative("HistogramInfo") + public HistogramInfo(int min, int max, int bucketCount) { + this.min = min; + this.max = max; + this.bucketCount = bucketCount; + } + + @CalledByNative("HistogramInfo") + public void addSample(int value, int numEvents) { + samples.put(value, numEvents); + } + } + + @CalledByNative + private void add(String name, HistogramInfo info) { + map.put(name, info); + } + + // Enables gathering of metrics (which can be fetched with getAndReset()). + // Must be called before PeerConnectionFactory is created. + public static void enable() { + nativeEnable(); + } + + // Gets and clears native histograms. + public static Metrics getAndReset() { + return nativeGetAndReset(); + } + + private static native void nativeEnable(); + private static native Metrics nativeGetAndReset(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/NativeLibraryLoader.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/NativeLibraryLoader.java new file mode 100644 index 0000000000..8bd7b3b250 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/NativeLibraryLoader.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for loading native libraries. A custom loader can be passed to + * PeerConnectionFactory.initialize. + */ +public interface NativeLibraryLoader { + /** + * Loads a native library with the given name. + * + * @return True on success + */ + boolean load(String name); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/NativePeerConnectionFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/NativePeerConnectionFactory.java new file mode 100644 index 0000000000..aeb91e1750 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/NativePeerConnectionFactory.java @@ -0,0 +1,20 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Factory for creating webrtc::jni::OwnedPeerConnection instances. */ +public interface NativePeerConnectionFactory { + /** + * Create a new webrtc::jni::OwnedPeerConnection instance and returns a pointer to it. + * The caller takes ownership of the object. + */ + long createNativePeerConnection(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/NetEqFactoryFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/NetEqFactoryFactory.java new file mode 100644 index 0000000000..8464324cbc --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/NetEqFactoryFactory.java @@ -0,0 +1,21 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Implementations of this interface can create a native {@code webrtc::NetEqFactory}. + */ +public interface NetEqFactoryFactory { + /** + * Returns a pointer to a {@code webrtc::NetEqFactory}. The caller takes ownership. + */ + long createNativeNetEqFactory(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java new file mode 100644 index 0000000000..caca5e5889 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaCodecInfo; +import androidx.annotation.Nullable; +import java.util.Arrays; + +/** Factory for Android platform software VideoDecoders. */ +public class PlatformSoftwareVideoDecoderFactory extends MediaCodecVideoDecoderFactory { + /** + * Default allowed predicate. + */ + private static final Predicate<MediaCodecInfo> defaultAllowedPredicate = + new Predicate<MediaCodecInfo>() { + @Override + public boolean test(MediaCodecInfo arg) { + return MediaCodecUtils.isSoftwareOnly(arg); + } + }; + + /** + * Creates a PlatformSoftwareVideoDecoderFactory that supports surface texture rendering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + */ + public PlatformSoftwareVideoDecoderFactory(@Nullable EglBase.Context sharedContext) { + super(sharedContext, defaultAllowedPredicate); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/Predicate.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/Predicate.java new file mode 100644 index 0000000000..50e6975000 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/Predicate.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Represents a predicate (boolean-valued function) of one argument. + */ +public interface Predicate<T> { + /** + * Evaluates this predicate on the given argument. + * + * @param arg the input argument + * @return true if the input argument matches the predicate, otherwise false + */ + boolean test(T arg); + + /** + * Returns a composed predicate that represents a short-circuiting logical OR of this predicate + * and another. When evaluating the composed predicate, if this predicate is true, then the other + * predicate is not evaluated. + * + * @param other a predicate that will be logically-ORed with this predicate + * @return a composed predicate that represents the short-circuiting logical OR of this predicate + * and the other predicate + */ + default Predicate<T> or(Predicate<? super T> other) { + return new Predicate<T>() { + @Override + public boolean test(T arg) { + return Predicate.this.test(arg) || other.test(arg); + } + }; + } + + /** + * Returns a composed predicate that represents a short-circuiting logical AND of this predicate + * and another. + * + * @param other a predicate that will be logically-ANDed with this predicate + * @return a composed predicate that represents the short-circuiting logical AND of this predicate + * and the other predicate + */ + default Predicate<T> and(Predicate<? super T> other) { + return new Predicate<T>() { + @Override + public boolean test(T arg) { + return Predicate.this.test(arg) && other.test(arg); + } + }; + } + + /** + * Returns a predicate that represents the logical negation of this predicate. + * + * @return a predicate that represents the logical negation of this predicate + */ + default Predicate<T> negate() { + return new Predicate<T>() { + @Override + public boolean test(T arg) { + return !Predicate.this.test(arg); + } + }; + } +}
\ No newline at end of file diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/RefCounted.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/RefCounted.java new file mode 100644 index 0000000000..0c1c3bf1f9 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/RefCounted.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for ref counted objects in WebRTC. These objects have significant resources that need + * to be freed when they are no longer in use. Each objects starts with ref count of one when + * created. If a reference is passed as a parameter to a method, the caller has ownesrship of the + * object by default - calling release is not necessary unless retain is called. + */ +public interface RefCounted { + /** Increases ref count by one. */ + @CalledByNative void retain(); + + /** + * Decreases ref count by one. When the ref count reaches zero, resources related to the object + * will be freed. + */ + @CalledByNative void release(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/RenderSynchronizer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/RenderSynchronizer.java new file mode 100644 index 0000000000..b1ade84298 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/RenderSynchronizer.java @@ -0,0 +1,116 @@ +/* + * Copyright 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.os.Handler; +import android.os.Looper; +import android.view.Choreographer; +import androidx.annotation.GuardedBy; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +/** + * Class to synchronize rendering updates with display refresh cycles and save power by blocking + * updates that exceeds the target frame rate. + */ +public final class RenderSynchronizer { + + /** Interface for listening to render window updates. */ + public interface Listener { + void onRenderWindowOpen(); + + void onRenderWindowClose(); + } + + private static final String TAG = "RenderSynchronizer"; + private static final float DEFAULT_TARGET_FPS = 30f; + private final Object lock = new Object(); + private final List<Listener> listeners = new CopyOnWriteArrayList<>(); + private final long targetFrameIntervalNanos; + private final Handler mainThreadHandler; + private Choreographer choreographer; + + @GuardedBy("lock") + private boolean isListening; + + private boolean renderWindowOpen; + private long lastRefreshTimeNanos; + private long lastOpenedTimeNanos; + + public RenderSynchronizer(float targetFrameRateFps) { + this.targetFrameIntervalNanos = Math.round(TimeUnit.SECONDS.toNanos(1) / targetFrameRateFps); + this.mainThreadHandler = new Handler(Looper.getMainLooper()); + mainThreadHandler.post(() -> this.choreographer = Choreographer.getInstance()); + Logging.d(TAG, "Created"); + } + + public RenderSynchronizer() { + this(DEFAULT_TARGET_FPS); + } + + public void registerListener(Listener listener) { + listeners.add(listener); + + synchronized (lock) { + if (!isListening) { + Logging.d(TAG, "First listener, subscribing to frame callbacks"); + isListening = true; + mainThreadHandler.post( + () -> choreographer.postFrameCallback(this::onDisplayRefreshCycleBegin)); + } + } + } + + public void removeListener(Listener listener) { + listeners.remove(listener); + } + + private void onDisplayRefreshCycleBegin(long refreshTimeNanos) { + synchronized (lock) { + if (listeners.isEmpty()) { + Logging.d(TAG, "No listeners, unsubscribing to frame callbacks"); + isListening = false; + return; + } + } + choreographer.postFrameCallback(this::onDisplayRefreshCycleBegin); + + long lastOpenDeltaNanos = refreshTimeNanos - lastOpenedTimeNanos; + long refreshDeltaNanos = refreshTimeNanos - lastRefreshTimeNanos; + lastRefreshTimeNanos = refreshTimeNanos; + + // Make a greedy choice whether to open (or keep open) the render window. If the current time + // since the render window was last opened is closer to the target than what we predict it would + // be in the next refresh cycle then we open the window. + if (Math.abs(lastOpenDeltaNanos - targetFrameIntervalNanos) + < Math.abs(lastOpenDeltaNanos - targetFrameIntervalNanos + refreshDeltaNanos)) { + lastOpenedTimeNanos = refreshTimeNanos; + openRenderWindow(); + } else if (renderWindowOpen) { + closeRenderWindow(); + } + } + + private void openRenderWindow() { + renderWindowOpen = true; + for (Listener listener : listeners) { + listener.onRenderWindowOpen(); + } + } + + private void closeRenderWindow() { + renderWindowOpen = false; + for (Listener listener : listeners) { + listener.onRenderWindowClose(); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/RendererCommon.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/RendererCommon.java new file mode 100644 index 0000000000..b97901c634 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/RendererCommon.java @@ -0,0 +1,259 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Point; +import android.opengl.Matrix; +import android.view.View; + +/** + * Static helper functions for renderer implementations. + */ +public class RendererCommon { + /** Interface for reporting rendering events. */ + public static interface RendererEvents { + /** + * Callback fired once first frame is rendered. + */ + public void onFirstFrameRendered(); + + /** + * Callback fired when rendered frame resolution or rotation has changed. + */ + public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation); + } + + /** + * Interface for rendering frames on an EGLSurface with specified viewport location. Rotation, + * mirror, and cropping is specified using a 4x4 texture coordinate transform matrix. The frame + * input can either be an OES texture, RGB texture, or YUV textures in I420 format. The function + * release() must be called manually to free the resources held by this object. + */ + public static interface GlDrawer { + /** + * Functions for drawing frames with different sources. The rendering surface target is + * implied by the current EGL context of the calling thread and requires no explicit argument. + * The coordinates specify the viewport location on the surface target. + */ + void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight, + int viewportX, int viewportY, int viewportWidth, int viewportHeight); + void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight, int viewportX, + int viewportY, int viewportWidth, int viewportHeight); + void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight, + int viewportX, int viewportY, int viewportWidth, int viewportHeight); + + /** + * Release all GL resources. This needs to be done manually, otherwise resources may leak. + */ + void release(); + } + + /** + * Helper class for determining layout size based on layout requirements, scaling type, and video + * aspect ratio. + */ + public static class VideoLayoutMeasure { + // The scaling type determines how the video will fill the allowed layout area in measure(). It + // can be specified separately for the case when video has matched orientation with layout size + // and when there is an orientation mismatch. + private float visibleFractionMatchOrientation = + convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED); + private float visibleFractionMismatchOrientation = + convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED); + + public void setScalingType(ScalingType scalingType) { + setScalingType(/* scalingTypeMatchOrientation= */ scalingType, + /* scalingTypeMismatchOrientation= */ scalingType); + } + + public void setScalingType( + ScalingType scalingTypeMatchOrientation, ScalingType scalingTypeMismatchOrientation) { + this.visibleFractionMatchOrientation = + convertScalingTypeToVisibleFraction(scalingTypeMatchOrientation); + this.visibleFractionMismatchOrientation = + convertScalingTypeToVisibleFraction(scalingTypeMismatchOrientation); + } + + public void setVisibleFraction( + float visibleFractionMatchOrientation, float visibleFractionMismatchOrientation) { + this.visibleFractionMatchOrientation = visibleFractionMatchOrientation; + this.visibleFractionMismatchOrientation = visibleFractionMismatchOrientation; + } + + public Point measure(int widthSpec, int heightSpec, int frameWidth, int frameHeight) { + // Calculate max allowed layout size. + final int maxWidth = View.getDefaultSize(Integer.MAX_VALUE, widthSpec); + final int maxHeight = View.getDefaultSize(Integer.MAX_VALUE, heightSpec); + if (frameWidth == 0 || frameHeight == 0 || maxWidth == 0 || maxHeight == 0) { + return new Point(maxWidth, maxHeight); + } + // Calculate desired display size based on scaling type, video aspect ratio, + // and maximum layout size. + final float frameAspect = frameWidth / (float) frameHeight; + final float displayAspect = maxWidth / (float) maxHeight; + final float visibleFraction = (frameAspect > 1.0f) == (displayAspect > 1.0f) + ? visibleFractionMatchOrientation + : visibleFractionMismatchOrientation; + final Point layoutSize = getDisplaySize(visibleFraction, frameAspect, maxWidth, maxHeight); + + // If the measure specification is forcing a specific size - yield. + if (View.MeasureSpec.getMode(widthSpec) == View.MeasureSpec.EXACTLY) { + layoutSize.x = maxWidth; + } + if (View.MeasureSpec.getMode(heightSpec) == View.MeasureSpec.EXACTLY) { + layoutSize.y = maxHeight; + } + return layoutSize; + } + } + + // Types of video scaling: + // SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by + // maintaining the aspect ratio (black borders may be displayed). + // SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by + // maintaining the aspect ratio. Some portion of the video frame may be + // clipped. + // SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as + // possible of the view while maintaining aspect ratio, under the constraint that at least + // `BALANCED_VISIBLE_FRACTION` of the frame content will be shown. + public static enum ScalingType { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED } + // The minimum fraction of the frame content that will be shown for `SCALE_ASPECT_BALANCED`. + // This limits excessive cropping when adjusting display size. + private static float BALANCED_VISIBLE_FRACTION = 0.5625f; + + /** + * Returns layout transformation matrix that applies an optional mirror effect and compensates + * for video vs display aspect ratio. + */ + public static float[] getLayoutMatrix( + boolean mirror, float videoAspectRatio, float displayAspectRatio) { + float scaleX = 1; + float scaleY = 1; + // Scale X or Y dimension so that video and display size have same aspect ratio. + if (displayAspectRatio > videoAspectRatio) { + scaleY = videoAspectRatio / displayAspectRatio; + } else { + scaleX = displayAspectRatio / videoAspectRatio; + } + // Apply optional horizontal flip. + if (mirror) { + scaleX *= -1; + } + final float matrix[] = new float[16]; + Matrix.setIdentityM(matrix, 0); + Matrix.scaleM(matrix, 0, scaleX, scaleY, 1); + adjustOrigin(matrix); + return matrix; + } + + /** Converts a float[16] matrix array to android.graphics.Matrix. */ + public static android.graphics.Matrix convertMatrixToAndroidGraphicsMatrix(float[] matrix4x4) { + // clang-format off + float[] values = { + matrix4x4[0 * 4 + 0], matrix4x4[1 * 4 + 0], matrix4x4[3 * 4 + 0], + matrix4x4[0 * 4 + 1], matrix4x4[1 * 4 + 1], matrix4x4[3 * 4 + 1], + matrix4x4[0 * 4 + 3], matrix4x4[1 * 4 + 3], matrix4x4[3 * 4 + 3], + }; + // clang-format on + + android.graphics.Matrix matrix = new android.graphics.Matrix(); + matrix.setValues(values); + return matrix; + } + + /** Converts android.graphics.Matrix to a float[16] matrix array. */ + public static float[] convertMatrixFromAndroidGraphicsMatrix(android.graphics.Matrix matrix) { + float[] values = new float[9]; + matrix.getValues(values); + + // The android.graphics.Matrix looks like this: + // [x1 y1 w1] + // [x2 y2 w2] + // [x3 y3 w3] + // We want to contruct a matrix that looks like this: + // [x1 y1 0 w1] + // [x2 y2 0 w2] + // [ 0 0 1 0] + // [x3 y3 0 w3] + // Since it is stored in column-major order, it looks like this: + // [x1 x2 0 x3 + // y1 y2 0 y3 + // 0 0 1 0 + // w1 w2 0 w3] + // clang-format off + float[] matrix4x4 = { + values[0 * 3 + 0], values[1 * 3 + 0], 0, values[2 * 3 + 0], + values[0 * 3 + 1], values[1 * 3 + 1], 0, values[2 * 3 + 1], + 0, 0, 1, 0, + values[0 * 3 + 2], values[1 * 3 + 2], 0, values[2 * 3 + 2], + }; + // clang-format on + return matrix4x4; + } + + /** + * Calculate display size based on scaling type, video aspect ratio, and maximum display size. + */ + public static Point getDisplaySize( + ScalingType scalingType, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) { + return getDisplaySize(convertScalingTypeToVisibleFraction(scalingType), videoAspectRatio, + maxDisplayWidth, maxDisplayHeight); + } + + /** + * Move `matrix` transformation origin to (0.5, 0.5). This is the origin for texture coordinates + * that are in the range 0 to 1. + */ + private static void adjustOrigin(float[] matrix) { + // Note that OpenGL is using column-major order. + // Pre translate with -0.5 to move coordinates to range [-0.5, 0.5]. + matrix[12] -= 0.5f * (matrix[0] + matrix[4]); + matrix[13] -= 0.5f * (matrix[1] + matrix[5]); + // Post translate with 0.5 to move coordinates to range [0, 1]. + matrix[12] += 0.5f; + matrix[13] += 0.5f; + } + + /** + * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video + * that must remain visible. + */ + private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) { + switch (scalingType) { + case SCALE_ASPECT_FIT: + return 1.0f; + case SCALE_ASPECT_FILL: + return 0.0f; + case SCALE_ASPECT_BALANCED: + return BALANCED_VISIBLE_FRACTION; + default: + throw new IllegalArgumentException(); + } + } + + /** + * Calculate display size based on minimum fraction of the video that must remain visible, + * video aspect ratio, and maximum display size. + */ + public static Point getDisplaySize( + float minVisibleFraction, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) { + // If there is no constraint on the amount of cropping, fill the allowed display area. + if (minVisibleFraction == 0 || videoAspectRatio == 0) { + return new Point(maxDisplayWidth, maxDisplayHeight); + } + // Each dimension is constrained on max display size and how much we are allowed to crop. + final int width = Math.min( + maxDisplayWidth, Math.round(maxDisplayHeight / minVisibleFraction * videoAspectRatio)); + final int height = Math.min( + maxDisplayHeight, Math.round(maxDisplayWidth / minVisibleFraction / videoAspectRatio)); + return new Point(width, height); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SSLCertificateVerifier.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SSLCertificateVerifier.java new file mode 100644 index 0000000000..461cd3b143 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SSLCertificateVerifier.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * The SSLCertificateVerifier interface allows API users to provide custom + * logic to verify certificates. + */ +public interface SSLCertificateVerifier { + /** + * Implementations of verify allow applications to provide custom logic for + * verifying certificates. This is not required by default and should be used + * with care. + * + * @param certificate A byte array containing a DER encoded X509 certificate. + * @return True if the certificate is verified and trusted else false. + */ + @CalledByNative boolean verify(byte[] certificate); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/ScreenCapturerAndroid.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/ScreenCapturerAndroid.java new file mode 100644 index 0000000000..08b03bd684 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/ScreenCapturerAndroid.java @@ -0,0 +1,223 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.app.Activity; +import android.content.Context; +import android.content.Intent; +import android.hardware.display.DisplayManager; +import android.hardware.display.VirtualDisplay; +import android.media.projection.MediaProjection; +import android.media.projection.MediaProjectionManager; +import android.os.Build.VERSION; +import android.os.Build.VERSION_CODES; +import android.view.Surface; +import androidx.annotation.Nullable; + +/** + * An implementation of VideoCapturer to capture the screen content as a video stream. + * Capturing is done by {@code MediaProjection} on a {@code SurfaceTexture}. We interact with this + * {@code SurfaceTexture} using a {@code SurfaceTextureHelper}. + * The {@code SurfaceTextureHelper} is created by the native code and passed to this capturer in + * {@code VideoCapturer.initialize()}. On receiving a new frame, this capturer passes it + * as a texture to the native code via {@code CapturerObserver.onFrameCaptured()}. This takes + * place on the HandlerThread of the given {@code SurfaceTextureHelper}. When done with each frame, + * the native code returns the buffer to the {@code SurfaceTextureHelper} to be used for new + * frames. At any time, at most one frame is being processed. + */ +public class ScreenCapturerAndroid implements VideoCapturer, VideoSink { + private static final int DISPLAY_FLAGS = + DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC | DisplayManager.VIRTUAL_DISPLAY_FLAG_PRESENTATION; + // DPI for VirtualDisplay, does not seem to matter for us. + private static final int VIRTUAL_DISPLAY_DPI = 400; + + private final Intent mediaProjectionPermissionResultData; + private final MediaProjection.Callback mediaProjectionCallback; + + private int width; + private int height; + @Nullable private VirtualDisplay virtualDisplay; + @Nullable private SurfaceTextureHelper surfaceTextureHelper; + @Nullable private CapturerObserver capturerObserver; + private long numCapturedFrames; + @Nullable private MediaProjection mediaProjection; + private boolean isDisposed; + @Nullable private MediaProjectionManager mediaProjectionManager; + + /** + * Constructs a new Screen Capturer. + * + * @param mediaProjectionPermissionResultData the result data of MediaProjection permission + * activity; the calling app must validate that result code is Activity.RESULT_OK before + * calling this method. + * @param mediaProjectionCallback MediaProjection callback to implement application specific + * logic in events such as when the user revokes a previously granted capture permission. + **/ + public ScreenCapturerAndroid(Intent mediaProjectionPermissionResultData, + MediaProjection.Callback mediaProjectionCallback) { + this.mediaProjectionPermissionResultData = mediaProjectionPermissionResultData; + this.mediaProjectionCallback = mediaProjectionCallback; + } + + private void checkNotDisposed() { + if (isDisposed) { + throw new RuntimeException("capturer is disposed."); + } + } + + @Nullable + public MediaProjection getMediaProjection() { + return mediaProjection; + } + + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void initialize(final SurfaceTextureHelper surfaceTextureHelper, + final Context applicationContext, final CapturerObserver capturerObserver) { + checkNotDisposed(); + + if (capturerObserver == null) { + throw new RuntimeException("capturerObserver not set."); + } + this.capturerObserver = capturerObserver; + + if (surfaceTextureHelper == null) { + throw new RuntimeException("surfaceTextureHelper not set."); + } + this.surfaceTextureHelper = surfaceTextureHelper; + + mediaProjectionManager = (MediaProjectionManager) applicationContext.getSystemService( + Context.MEDIA_PROJECTION_SERVICE); + } + + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void startCapture( + final int width, final int height, final int ignoredFramerate) { + checkNotDisposed(); + + this.width = width; + this.height = height; + + mediaProjection = mediaProjectionManager.getMediaProjection( + Activity.RESULT_OK, mediaProjectionPermissionResultData); + + // Let MediaProjection callback use the SurfaceTextureHelper thread. + mediaProjection.registerCallback(mediaProjectionCallback, surfaceTextureHelper.getHandler()); + + updateVirtualDisplay(); + capturerObserver.onCapturerStarted(true); + surfaceTextureHelper.startListening(ScreenCapturerAndroid.this); + } + + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void stopCapture() { + checkNotDisposed(); + ThreadUtils.invokeAtFrontUninterruptibly(surfaceTextureHelper.getHandler(), new Runnable() { + @Override + public void run() { + surfaceTextureHelper.stopListening(); + capturerObserver.onCapturerStopped(); + + if (virtualDisplay != null) { + virtualDisplay.release(); + virtualDisplay = null; + } + + if (mediaProjection != null) { + // Unregister the callback before stopping, otherwise the callback recursively + // calls this method. + mediaProjection.unregisterCallback(mediaProjectionCallback); + mediaProjection.stop(); + mediaProjection = null; + } + } + }); + } + + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void dispose() { + isDisposed = true; + } + + /** + * Changes output video format. This method can be used to scale the output + * video, or to change orientation when the captured screen is rotated for example. + * + * @param width new output video width + * @param height new output video height + * @param ignoredFramerate ignored + */ + @Override + // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. + @SuppressWarnings("NoSynchronizedMethodCheck") + public synchronized void changeCaptureFormat( + final int width, final int height, final int ignoredFramerate) { + checkNotDisposed(); + + this.width = width; + this.height = height; + + if (virtualDisplay == null) { + // Capturer is stopped, the virtual display will be created in startCapture(). + return; + } + + // Create a new virtual display on the surfaceTextureHelper thread to avoid interference + // with frame processing, which happens on the same thread (we serialize events by running + // them on the same thread). + ThreadUtils.invokeAtFrontUninterruptibly( + surfaceTextureHelper.getHandler(), this::updateVirtualDisplay); + } + + private void updateVirtualDisplay() { + surfaceTextureHelper.setTextureSize(width, height); + // Before Android S (12), resizing the virtual display can cause the captured screen to be + // scaled incorrectly, so keep the behavior of recreating the virtual display prior to Android + // S. + if (virtualDisplay == null || VERSION.SDK_INT < VERSION_CODES.S) { + createVirtualDisplay(); + } else { + virtualDisplay.resize(width, height, VIRTUAL_DISPLAY_DPI); + virtualDisplay.setSurface(new Surface(surfaceTextureHelper.getSurfaceTexture())); + } + } + private void createVirtualDisplay() { + if (virtualDisplay != null) { + virtualDisplay.release(); + } + virtualDisplay = mediaProjection.createVirtualDisplay("WebRTC_ScreenCapture", width, height, + VIRTUAL_DISPLAY_DPI, DISPLAY_FLAGS, new Surface(surfaceTextureHelper.getSurfaceTexture()), + null /* callback */, null /* callback handler */); + } + + // This is called on the internal looper thread of {@Code SurfaceTextureHelper}. + @Override + public void onFrame(VideoFrame frame) { + numCapturedFrames++; + capturerObserver.onFrameCaptured(frame); + } + + @Override + public boolean isScreencast() { + return true; + } + + public long getNumCapturedFrames() { + return numCapturedFrames; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SdpObserver.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SdpObserver.java new file mode 100644 index 0000000000..afa99bc552 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SdpObserver.java @@ -0,0 +1,26 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Interface for observing SDP-related events. */ +public interface SdpObserver { + /** Called on success of Create{Offer,Answer}(). */ + @CalledByNative void onCreateSuccess(SessionDescription sdp); + + /** Called on success of Set{Local,Remote}Description(). */ + @CalledByNative void onSetSuccess(); + + /** Called on error of Create{Offer,Answer}(). */ + @CalledByNative void onCreateFailure(String error); + + /** Called on error of Set{Local,Remote}Description(). */ + @CalledByNative void onSetFailure(String error); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SessionDescription.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SessionDescription.java new file mode 100644 index 0000000000..be89599a5f --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SessionDescription.java @@ -0,0 +1,56 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.Locale; + +/** + * Description of an RFC 4566 Session. + * SDPs are passed as serialized Strings in Java-land and are materialized + * to SessionDescriptionInterface as appropriate in the JNI layer. + */ +public class SessionDescription { + /** Java-land enum version of SessionDescriptionInterface's type() string. */ + public static enum Type { + OFFER, + PRANSWER, + ANSWER, + ROLLBACK; + + public String canonicalForm() { + return name().toLowerCase(Locale.US); + } + + @CalledByNative("Type") + public static Type fromCanonicalForm(String canonical) { + return Type.valueOf(Type.class, canonical.toUpperCase(Locale.US)); + } + } + + public final Type type; + public final String description; + + @CalledByNative + public SessionDescription(Type type, String description) { + this.type = type; + this.description = description; + } + + @CalledByNative + String getDescription() { + return description; + } + + @CalledByNative + String getTypeInCanonicalForm() { + return type.canonicalForm(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java new file mode 100644 index 0000000000..2ac42e834e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.List; + +public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { + private static final String TAG = "SoftwareVideoDecoderFactory"; + + private final long nativeFactory; + + public SoftwareVideoDecoderFactory() { + this.nativeFactory = nativeCreateFactory(); + } + + @Nullable + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + long nativeDecoder = nativeCreateDecoder(nativeFactory, info); + if (nativeDecoder == 0) { + Logging.w(TAG, "Trying to create decoder for unsupported format. " + info); + return null; + } + + return new WrappedNativeVideoDecoder() { + @Override + public long createNativeVideoDecoder() { + return nativeDecoder; + } + }; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]); + } + + private static native long nativeCreateFactory(); + + private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo); + + private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java new file mode 100644 index 0000000000..7f4c457b97 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.List; + +public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { + private static final String TAG = "SoftwareVideoEncoderFactory"; + + private final long nativeFactory; + + public SoftwareVideoEncoderFactory() { + this.nativeFactory = nativeCreateFactory(); + } + + @Nullable + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + long nativeEncoder = nativeCreateEncoder(nativeFactory, info); + if (nativeEncoder == 0) { + Logging.w(TAG, "Trying to create encoder for unsupported format. " + info); + return null; + } + + return new WrappedNativeVideoEncoder() { + @Override + public long createNativeVideoEncoder() { + return nativeEncoder; + } + + @Override + public boolean isHardwareEncoder() { + return false; + } + }; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]); + } + + private static native long nativeCreateFactory(); + + private static native long nativeCreateEncoder(long factory, VideoCodecInfo videoCodecInfo); + + private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsObserver.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsObserver.java new file mode 100644 index 0000000000..b9984c18db --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsObserver.java @@ -0,0 +1,17 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Interface for observing Stats reports (see webrtc::StatsObservers). */ +public interface StatsObserver { + /** Called when the reports are ready.*/ + @CalledByNative public void onComplete(StatsReport[] reports); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsReport.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsReport.java new file mode 100644 index 0000000000..b8f1cf87fe --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/StatsReport.java @@ -0,0 +1,63 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Java version of webrtc::StatsReport. */ +public class StatsReport { + /** Java version of webrtc::StatsReport::Value. */ + public static class Value { + public final String name; + public final String value; + + @CalledByNative("Value") + public Value(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("[").append(name).append(": ").append(value).append("]"); + return builder.toString(); + } + } + + public final String id; + public final String type; + // Time since 1970-01-01T00:00:00Z in milliseconds. + public final double timestamp; + public final Value[] values; + + @CalledByNative + public StatsReport(String id, String type, double timestamp, Value[] values) { + this.id = id; + this.type = type; + this.timestamp = timestamp; + this.values = values; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("id: ") + .append(id) + .append(", type: ") + .append(type) + .append(", timestamp: ") + .append(timestamp) + .append(", values: "); + for (int i = 0; i < values.length; ++i) { + builder.append(values[i].toString()).append(", "); + } + return builder.toString(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceEglRenderer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceEglRenderer.java new file mode 100644 index 0000000000..6cba3f473b --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceEglRenderer.java @@ -0,0 +1,160 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.view.SurfaceHolder; +import java.util.concurrent.CountDownLatch; + +/** + * Display the video stream on a Surface. + * renderFrame() is asynchronous to avoid blocking the calling thread. + * This class is thread safe and handles access from potentially three different threads: + * Interaction from the main app in init, release and setMirror. + * Interaction from C++ rtc::VideoSinkInterface in renderFrame. + * Interaction from SurfaceHolder lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed. + */ +public class SurfaceEglRenderer extends EglRenderer implements SurfaceHolder.Callback { + private static final String TAG = "SurfaceEglRenderer"; + + // Callback for reporting renderer events. Read-only after initialization so no lock required. + private RendererCommon.RendererEvents rendererEvents; + + private final Object layoutLock = new Object(); + private boolean isRenderingPaused; + private boolean isFirstFrameRendered; + private int rotatedFrameWidth; + private int rotatedFrameHeight; + private int frameRotation; + + /** + * In order to render something, you must first call init(). + */ + public SurfaceEglRenderer(String name) { + super(name); + } + + /** + * Initialize this class, sharing resources with `sharedContext`. The custom `drawer` will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * `drawer`. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. + */ + public void init(final EglBase.Context sharedContext, + RendererCommon.RendererEvents rendererEvents, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + ThreadUtils.checkIsOnMainThread(); + this.rendererEvents = rendererEvents; + synchronized (layoutLock) { + isFirstFrameRendered = false; + rotatedFrameWidth = 0; + rotatedFrameHeight = 0; + frameRotation = 0; + } + super.init(sharedContext, configAttributes, drawer); + } + + @Override + public void init(final EglBase.Context sharedContext, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + init(sharedContext, null /* rendererEvents */, configAttributes, drawer); + } + + /** + * Limit render framerate. + * + * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps + * reduction. + */ + @Override + public void setFpsReduction(float fps) { + synchronized (layoutLock) { + isRenderingPaused = fps == 0f; + } + super.setFpsReduction(fps); + } + + @Override + public void disableFpsReduction() { + synchronized (layoutLock) { + isRenderingPaused = false; + } + super.disableFpsReduction(); + } + + @Override + public void pauseVideo() { + synchronized (layoutLock) { + isRenderingPaused = true; + } + super.pauseVideo(); + } + + // VideoSink interface. + @Override + public void onFrame(VideoFrame frame) { + updateFrameDimensionsAndReportEvents(frame); + super.onFrame(frame); + } + + // SurfaceHolder.Callback interface. + @Override + public void surfaceCreated(final SurfaceHolder holder) { + ThreadUtils.checkIsOnMainThread(); + createEglSurface(holder.getSurface()); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + ThreadUtils.checkIsOnMainThread(); + final CountDownLatch completionLatch = new CountDownLatch(1); + releaseEglSurface(completionLatch::countDown); + ThreadUtils.awaitUninterruptibly(completionLatch); + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + ThreadUtils.checkIsOnMainThread(); + logD("surfaceChanged: format: " + format + " size: " + width + "x" + height); + } + + // Update frame dimensions and report any changes to `rendererEvents`. + private void updateFrameDimensionsAndReportEvents(VideoFrame frame) { + synchronized (layoutLock) { + if (isRenderingPaused) { + return; + } + if (!isFirstFrameRendered) { + isFirstFrameRendered = true; + logD("Reporting first rendered frame."); + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); + } + } + if (rotatedFrameWidth != frame.getRotatedWidth() + || rotatedFrameHeight != frame.getRotatedHeight() + || frameRotation != frame.getRotation()) { + logD("Reporting frame resolution changed to " + frame.getBuffer().getWidth() + "x" + + frame.getBuffer().getHeight() + " with rotation " + frame.getRotation()); + if (rendererEvents != null) { + rendererEvents.onFrameResolutionChanged( + frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), frame.getRotation()); + } + rotatedFrameWidth = frame.getRotatedWidth(); + rotatedFrameHeight = frame.getRotatedHeight(); + frameRotation = frame.getRotation(); + } + } + } + + private void logD(String string) { + Logging.d(TAG, name + ": " + string); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java new file mode 100644 index 0000000000..3ea22736ea --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java @@ -0,0 +1,390 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.annotation.TargetApi; +import android.graphics.SurfaceTexture; +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; +import androidx.annotation.Nullable; +import java.util.concurrent.Callable; +import org.webrtc.EglBase.Context; +import org.webrtc.TextureBufferImpl.RefCountMonitor; +import org.webrtc.VideoFrame.TextureBuffer; + +/** + * Helper class for using a SurfaceTexture to create WebRTC VideoFrames. In order to create WebRTC + * VideoFrames, render onto the SurfaceTexture. The frames will be delivered to the listener. Only + * one texture frame can be in flight at once, so the frame must be released in order to receive a + * new frame. Call stopListening() to stop receiveing new frames. Call dispose to release all + * resources once the texture frame is released. + */ +public class SurfaceTextureHelper { + /** + * Interface for monitoring texture buffers created from this SurfaceTexture. Since only one + * texture buffer can exist at a time, this can be used to monitor for stuck frames. + */ + public interface FrameRefMonitor { + /** A new frame was created. New frames start with ref count of 1. */ + void onNewBuffer(TextureBuffer textureBuffer); + /** Ref count of the frame was incremented by the calling thread. */ + void onRetainBuffer(TextureBuffer textureBuffer); + /** Ref count of the frame was decremented by the calling thread. */ + void onReleaseBuffer(TextureBuffer textureBuffer); + /** Frame was destroyed (ref count reached 0). */ + void onDestroyBuffer(TextureBuffer textureBuffer); + } + + private static final String TAG = "SurfaceTextureHelper"; + /** + * Construct a new SurfaceTextureHelper sharing OpenGL resources with `sharedContext`. A dedicated + * thread and handler is created for handling the SurfaceTexture. May return null if EGL fails to + * initialize a pixel buffer surface and make it current. If alignTimestamps is true, the frame + * timestamps will be aligned to rtc::TimeNanos(). If frame timestamps are aligned to + * rtc::TimeNanos() there is no need for aligning timestamps again in + * PeerConnectionFactory.createVideoSource(). This makes the timestamps more accurate and + * closer to actual creation time. + */ + public static SurfaceTextureHelper create(final String threadName, + final EglBase.Context sharedContext, boolean alignTimestamps, final YuvConverter yuvConverter, + FrameRefMonitor frameRefMonitor) { + final HandlerThread thread = new HandlerThread(threadName); + thread.start(); + final Handler handler = new Handler(thread.getLooper()); + + // The onFrameAvailable() callback will be executed on the SurfaceTexture ctor thread. See: + // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.android/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. + // Therefore, in order to control the callback thread on API lvl < 21, the SurfaceTextureHelper + // is constructed on the `handler` thread. + return ThreadUtils.invokeAtFrontUninterruptibly(handler, new Callable<SurfaceTextureHelper>() { + @Nullable + @Override + public SurfaceTextureHelper call() { + try { + return new SurfaceTextureHelper( + sharedContext, handler, alignTimestamps, yuvConverter, frameRefMonitor); + } catch (RuntimeException e) { + Logging.e(TAG, threadName + " create failure", e); + return null; + } + } + }); + } + + /** + * Same as above with alignTimestamps set to false and yuvConverter set to new YuvConverter. + * + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) + */ + public static SurfaceTextureHelper create( + final String threadName, final EglBase.Context sharedContext) { + return create(threadName, sharedContext, /* alignTimestamps= */ false, new YuvConverter(), + /*frameRefMonitor=*/null); + } + + /** + * Same as above with yuvConverter set to new YuvConverter. + * + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) + */ + public static SurfaceTextureHelper create( + final String threadName, final EglBase.Context sharedContext, boolean alignTimestamps) { + return create( + threadName, sharedContext, alignTimestamps, new YuvConverter(), /*frameRefMonitor=*/null); + } + + /** + * Create a SurfaceTextureHelper without frame ref monitor. + * + * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor) + */ + public static SurfaceTextureHelper create(final String threadName, + final EglBase.Context sharedContext, boolean alignTimestamps, YuvConverter yuvConverter) { + return create( + threadName, sharedContext, alignTimestamps, yuvConverter, /*frameRefMonitor=*/null); + } + + private final RefCountMonitor textureRefCountMonitor = new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) { + if (frameRefMonitor != null) { + frameRefMonitor.onRetainBuffer(textureBuffer); + } + } + + @Override + public void onRelease(TextureBufferImpl textureBuffer) { + if (frameRefMonitor != null) { + frameRefMonitor.onReleaseBuffer(textureBuffer); + } + } + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + returnTextureFrame(); + if (frameRefMonitor != null) { + frameRefMonitor.onDestroyBuffer(textureBuffer); + } + } + }; + + private final Handler handler; + private final EglBase eglBase; + private final SurfaceTexture surfaceTexture; + private final int oesTextureId; + private final YuvConverter yuvConverter; + @Nullable private final TimestampAligner timestampAligner; + private final FrameRefMonitor frameRefMonitor; + + // These variables are only accessed from the `handler` thread. + @Nullable private VideoSink listener; + // The possible states of this class. + private boolean hasPendingTexture; + private volatile boolean isTextureInUse; + private boolean isQuitting; + private int frameRotation; + private int textureWidth; + private int textureHeight; + // `pendingListener` is set in setListener() and the runnable is posted to the handler thread. + // setListener() is not allowed to be called again before stopListening(), so this is thread safe. + @Nullable private VideoSink pendingListener; + final Runnable setListenerRunnable = new Runnable() { + @Override + public void run() { + Logging.d(TAG, "Setting listener to " + pendingListener); + listener = pendingListener; + pendingListener = null; + // May have a pending frame from the previous capture session - drop it. + if (hasPendingTexture) { + // Calling updateTexImage() is neccessary in order to receive new frames. + updateTexImage(); + hasPendingTexture = false; + } + } + }; + + private SurfaceTextureHelper(Context sharedContext, Handler handler, boolean alignTimestamps, + YuvConverter yuvConverter, FrameRefMonitor frameRefMonitor) { + if (handler.getLooper().getThread() != Thread.currentThread()) { + throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread"); + } + this.handler = handler; + this.timestampAligner = alignTimestamps ? new TimestampAligner() : null; + this.yuvConverter = yuvConverter; + this.frameRefMonitor = frameRefMonitor; + + eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); + try { + // Both these statements have been observed to fail on rare occasions, see BUG=webrtc:5682. + eglBase.createDummyPbufferSurface(); + eglBase.makeCurrent(); + } catch (RuntimeException e) { + // Clean up before rethrowing the exception. + eglBase.release(); + handler.getLooper().quit(); + throw e; + } + + oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); + surfaceTexture = new SurfaceTexture(oesTextureId); + surfaceTexture.setOnFrameAvailableListener(st -> { + if (hasPendingTexture) { + Logging.d(TAG, "A frame is already pending, dropping frame."); + } + + hasPendingTexture = true; + tryDeliverTextureFrame(); + }, handler); + } + + /** + * Start to stream textures to the given `listener`. If you need to change listener, you need to + * call stopListening() first. + */ + public void startListening(final VideoSink listener) { + if (this.listener != null || this.pendingListener != null) { + throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); + } + this.pendingListener = listener; + handler.post(setListenerRunnable); + } + + /** + * Stop listening. The listener set in startListening() is guaranteded to not receive any more + * onFrame() callbacks after this function returns. + */ + public void stopListening() { + Logging.d(TAG, "stopListening()"); + handler.removeCallbacks(setListenerRunnable); + ThreadUtils.invokeAtFrontUninterruptibly(handler, () -> { + listener = null; + pendingListener = null; + }); + } + + /** + * Use this function to set the texture size. Note, do not call setDefaultBufferSize() yourself + * since this class needs to be aware of the texture size. + */ + public void setTextureSize(int textureWidth, int textureHeight) { + if (textureWidth <= 0) { + throw new IllegalArgumentException("Texture width must be positive, but was " + textureWidth); + } + if (textureHeight <= 0) { + throw new IllegalArgumentException( + "Texture height must be positive, but was " + textureHeight); + } + surfaceTexture.setDefaultBufferSize(textureWidth, textureHeight); + handler.post(() -> { + this.textureWidth = textureWidth; + this.textureHeight = textureHeight; + tryDeliverTextureFrame(); + }); + } + + /** + * Forces a frame to be produced. If no new frame is available, the last frame is sent to the + * listener again. + */ + public void forceFrame() { + handler.post(() -> { + hasPendingTexture = true; + tryDeliverTextureFrame(); + }); + } + + /** Set the rotation of the delivered frames. */ + public void setFrameRotation(int rotation) { + handler.post(() -> this.frameRotation = rotation); + } + + /** + * Retrieve the underlying SurfaceTexture. The SurfaceTexture should be passed in to a video + * producer such as a camera or decoder. + */ + public SurfaceTexture getSurfaceTexture() { + return surfaceTexture; + } + + /** Retrieve the handler that calls onFrame(). This handler is valid until dispose() is called. */ + public Handler getHandler() { + return handler; + } + + /** + * This function is called when the texture frame is released. Only one texture frame can be in + * flight at once, so this function must be called before a new frame is delivered. + */ + private void returnTextureFrame() { + handler.post(() -> { + isTextureInUse = false; + if (isQuitting) { + release(); + } else { + tryDeliverTextureFrame(); + } + }); + } + + public boolean isTextureInUse() { + return isTextureInUse; + } + + /** + * Call disconnect() to stop receiving frames. OpenGL resources are released and the handler is + * stopped when the texture frame has been released. You are guaranteed to not receive any more + * onFrame() after this function returns. + */ + public void dispose() { + Logging.d(TAG, "dispose()"); + ThreadUtils.invokeAtFrontUninterruptibly(handler, () -> { + isQuitting = true; + if (!isTextureInUse) { + release(); + } + }); + } + + /** + * Posts to the correct thread to convert `textureBuffer` to I420. + * + * @deprecated Use toI420() instead. + */ + @Deprecated + public VideoFrame.I420Buffer textureToYuv(final TextureBuffer textureBuffer) { + return textureBuffer.toI420(); + } + + private void updateTexImage() { + // SurfaceTexture.updateTexImage apparently can compete and deadlock with eglSwapBuffers, + // as observed on Nexus 5. Therefore, synchronize it with the EGL functions. + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5702 for more info. + synchronized (EglBase.lock) { + surfaceTexture.updateTexImage(); + } + } + + private void tryDeliverTextureFrame() { + if (handler.getLooper().getThread() != Thread.currentThread()) { + throw new IllegalStateException("Wrong thread."); + } + if (isQuitting || !hasPendingTexture || isTextureInUse || listener == null) { + return; + } + if (textureWidth == 0 || textureHeight == 0) { + // Information about the resolution needs to be provided by a call to setTextureSize() before + // frames are produced. + Logging.w(TAG, "Texture size has not been set."); + return; + } + isTextureInUse = true; + hasPendingTexture = false; + + updateTexImage(); + + final float[] transformMatrix = new float[16]; + surfaceTexture.getTransformMatrix(transformMatrix); + long timestampNs = surfaceTexture.getTimestamp(); + if (timestampAligner != null) { + timestampNs = timestampAligner.translateTimestamp(timestampNs); + } + final VideoFrame.TextureBuffer buffer = + new TextureBufferImpl(textureWidth, textureHeight, TextureBuffer.Type.OES, oesTextureId, + RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix), handler, + yuvConverter, textureRefCountMonitor); + if (frameRefMonitor != null) { + frameRefMonitor.onNewBuffer(buffer); + } + final VideoFrame frame = new VideoFrame(buffer, frameRotation, timestampNs); + listener.onFrame(frame); + frame.release(); + } + + private void release() { + if (handler.getLooper().getThread() != Thread.currentThread()) { + throw new IllegalStateException("Wrong thread."); + } + if (isTextureInUse || !isQuitting) { + throw new IllegalStateException("Unexpected release."); + } + yuvConverter.release(); + GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); + surfaceTexture.release(); + eglBase.release(); + handler.getLooper().quit(); + if (timestampAligner != null) { + timestampAligner.dispose(); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java new file mode 100644 index 0000000000..6c9140abbd --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java @@ -0,0 +1,300 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.content.res.Resources.NotFoundException; +import android.graphics.Point; +import android.os.Looper; +import android.util.AttributeSet; +import android.view.SurfaceHolder; +import android.view.SurfaceView; + +/** + * Display the video stream on a SurfaceView. + */ +public class SurfaceViewRenderer extends SurfaceView + implements SurfaceHolder.Callback, VideoSink, RendererCommon.RendererEvents { + private static final String TAG = "SurfaceViewRenderer"; + + // Cached resource name. + private final String resourceName; + private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure = + new RendererCommon.VideoLayoutMeasure(); + private final SurfaceEglRenderer eglRenderer; + + // Callback for reporting renderer events. Read-only after initialization so no lock required. + private RendererCommon.RendererEvents rendererEvents; + + // Accessed only on the main thread. + private int rotatedFrameWidth; + private int rotatedFrameHeight; + private boolean enableFixedSize; + private int surfaceWidth; + private int surfaceHeight; + + /** + * Standard View constructor. In order to render something, you must first call init(). + */ + public SurfaceViewRenderer(Context context) { + super(context); + this.resourceName = getResourceName(); + eglRenderer = new SurfaceEglRenderer(resourceName); + getHolder().addCallback(this); + getHolder().addCallback(eglRenderer); + } + + /** + * Standard View constructor. In order to render something, you must first call init(). + */ + public SurfaceViewRenderer(Context context, AttributeSet attrs) { + super(context, attrs); + this.resourceName = getResourceName(); + eglRenderer = new SurfaceEglRenderer(resourceName); + getHolder().addCallback(this); + getHolder().addCallback(eglRenderer); + } + + /** + * Initialize this class, sharing resources with `sharedContext`. It is allowed to call init() to + * reinitialize the renderer after a previous init()/release() cycle. + */ + public void init(EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents) { + init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawer()); + } + + /** + * Initialize this class, sharing resources with `sharedContext`. The custom `drawer` will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * `drawer`. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. + */ + public void init(final EglBase.Context sharedContext, + RendererCommon.RendererEvents rendererEvents, final int[] configAttributes, + RendererCommon.GlDrawer drawer) { + ThreadUtils.checkIsOnMainThread(); + this.rendererEvents = rendererEvents; + rotatedFrameWidth = 0; + rotatedFrameHeight = 0; + eglRenderer.init(sharedContext, this /* rendererEvents */, configAttributes, drawer); + } + + /** + * Block until any pending frame is returned and all GL resources released, even if an interrupt + * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function + * should be called before the Activity is destroyed and the EGLContext is still valid. If you + * don't call this function, the GL resources might leak. + */ + public void release() { + eglRenderer.release(); + } + + /** + * Register a callback to be invoked when a new video frame has been received. + * + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. + * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is + * required. + * @param drawer Custom drawer to use for this frame listener. + */ + public void addFrameListener( + EglRenderer.FrameListener listener, float scale, RendererCommon.GlDrawer drawerParam) { + eglRenderer.addFrameListener(listener, scale, drawerParam); + } + + /** + * Register a callback to be invoked when a new video frame has been received. This version uses + * the drawer of the EglRenderer that was passed in init. + * + * @param listener The callback to be invoked. The callback will be invoked on the render thread. + * It should be lightweight and must not call removeFrameListener. + * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is + * required. + */ + public void addFrameListener(EglRenderer.FrameListener listener, float scale) { + eglRenderer.addFrameListener(listener, scale); + } + + public void removeFrameListener(EglRenderer.FrameListener listener) { + eglRenderer.removeFrameListener(listener); + } + + /** + * Enables fixed size for the surface. This provides better performance but might be buggy on some + * devices. By default this is turned off. + */ + public void setEnableHardwareScaler(boolean enabled) { + ThreadUtils.checkIsOnMainThread(); + enableFixedSize = enabled; + updateSurfaceSize(); + } + + /** + * Set if the video stream should be mirrored or not. + */ + public void setMirror(final boolean mirror) { + eglRenderer.setMirror(mirror); + } + + /** + * Set how the video will fill the allowed layout area. + */ + public void setScalingType(RendererCommon.ScalingType scalingType) { + ThreadUtils.checkIsOnMainThread(); + videoLayoutMeasure.setScalingType(scalingType); + requestLayout(); + } + + public void setScalingType(RendererCommon.ScalingType scalingTypeMatchOrientation, + RendererCommon.ScalingType scalingTypeMismatchOrientation) { + ThreadUtils.checkIsOnMainThread(); + videoLayoutMeasure.setScalingType(scalingTypeMatchOrientation, scalingTypeMismatchOrientation); + requestLayout(); + } + + /** + * Limit render framerate. + * + * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps + * reduction. + */ + public void setFpsReduction(float fps) { + eglRenderer.setFpsReduction(fps); + } + + public void disableFpsReduction() { + eglRenderer.disableFpsReduction(); + } + + public void pauseVideo() { + eglRenderer.pauseVideo(); + } + + // VideoSink interface. + @Override + public void onFrame(VideoFrame frame) { + eglRenderer.onFrame(frame); + } + + // View layout interface. + @Override + protected void onMeasure(int widthSpec, int heightSpec) { + ThreadUtils.checkIsOnMainThread(); + Point size = + videoLayoutMeasure.measure(widthSpec, heightSpec, rotatedFrameWidth, rotatedFrameHeight); + setMeasuredDimension(size.x, size.y); + logD("onMeasure(). New size: " + size.x + "x" + size.y); + } + + @Override + protected void onLayout(boolean changed, int left, int top, int right, int bottom) { + ThreadUtils.checkIsOnMainThread(); + eglRenderer.setLayoutAspectRatio((right - left) / (float) (bottom - top)); + updateSurfaceSize(); + } + + private void updateSurfaceSize() { + ThreadUtils.checkIsOnMainThread(); + if (enableFixedSize && rotatedFrameWidth != 0 && rotatedFrameHeight != 0 && getWidth() != 0 + && getHeight() != 0) { + final float layoutAspectRatio = getWidth() / (float) getHeight(); + final float frameAspectRatio = rotatedFrameWidth / (float) rotatedFrameHeight; + final int drawnFrameWidth; + final int drawnFrameHeight; + if (frameAspectRatio > layoutAspectRatio) { + drawnFrameWidth = (int) (rotatedFrameHeight * layoutAspectRatio); + drawnFrameHeight = rotatedFrameHeight; + } else { + drawnFrameWidth = rotatedFrameWidth; + drawnFrameHeight = (int) (rotatedFrameWidth / layoutAspectRatio); + } + // Aspect ratio of the drawn frame and the view is the same. + final int width = Math.min(getWidth(), drawnFrameWidth); + final int height = Math.min(getHeight(), drawnFrameHeight); + logD("updateSurfaceSize. Layout size: " + getWidth() + "x" + getHeight() + ", frame size: " + + rotatedFrameWidth + "x" + rotatedFrameHeight + ", requested surface size: " + width + + "x" + height + ", old surface size: " + surfaceWidth + "x" + surfaceHeight); + if (width != surfaceWidth || height != surfaceHeight) { + surfaceWidth = width; + surfaceHeight = height; + getHolder().setFixedSize(width, height); + } + } else { + surfaceWidth = surfaceHeight = 0; + getHolder().setSizeFromLayout(); + } + } + + // SurfaceHolder.Callback interface. + @Override + public void surfaceCreated(final SurfaceHolder holder) { + ThreadUtils.checkIsOnMainThread(); + surfaceWidth = surfaceHeight = 0; + updateSurfaceSize(); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) {} + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {} + + private String getResourceName() { + try { + return getResources().getResourceEntryName(getId()); + } catch (NotFoundException e) { + return ""; + } + } + + /** + * Post a task to clear the SurfaceView to a transparent uniform color. + */ + public void clearImage() { + eglRenderer.clearImage(); + } + + @Override + public void onFirstFrameRendered() { + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); + } + } + + @Override + public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) { + if (rendererEvents != null) { + rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation); + } + int rotatedWidth = rotation == 0 || rotation == 180 ? videoWidth : videoHeight; + int rotatedHeight = rotation == 0 || rotation == 180 ? videoHeight : videoWidth; + // run immediately if possible for ui thread tests + postOrRun(() -> { + rotatedFrameWidth = rotatedWidth; + rotatedFrameHeight = rotatedHeight; + updateSurfaceSize(); + requestLayout(); + }); + } + + private void postOrRun(Runnable r) { + if (Thread.currentThread() == Looper.getMainLooper().getThread()) { + r.run(); + } else { + post(r); + } + } + + private void logD(String string) { + Logging.d(TAG, resourceName + ": " + string); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/TextureBufferImpl.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/TextureBufferImpl.java new file mode 100644 index 0000000000..43470d6e39 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/TextureBufferImpl.java @@ -0,0 +1,197 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.os.Handler; +import androidx.annotation.Nullable; + +/** + * Android texture buffer that glues together the necessary information together with a generic + * release callback. ToI420() is implemented by providing a Handler and a YuvConverter. + */ +public class TextureBufferImpl implements VideoFrame.TextureBuffer { + interface RefCountMonitor { + void onRetain(TextureBufferImpl textureBuffer); + void onRelease(TextureBufferImpl textureBuffer); + void onDestroy(TextureBufferImpl textureBuffer); + } + + // This is the full resolution the texture has in memory after applying the transformation matrix + // that might include cropping. This resolution is useful to know when sampling the texture to + // avoid downscaling artifacts. + private final int unscaledWidth; + private final int unscaledHeight; + // This is the resolution that has been applied after cropAndScale(). + private final int width; + private final int height; + private final Type type; + private final int id; + private final Matrix transformMatrix; + private final Handler toI420Handler; + private final YuvConverter yuvConverter; + private final RefCountDelegate refCountDelegate; + private final RefCountMonitor refCountMonitor; + + public TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix, + Handler toI420Handler, YuvConverter yuvConverter, @Nullable Runnable releaseCallback) { + this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter, + new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) {} + + @Override + public void onRelease(TextureBufferImpl textureBuffer) {} + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + if (releaseCallback != null) { + releaseCallback.run(); + } + } + }); + } + + TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix, + Handler toI420Handler, YuvConverter yuvConverter, RefCountMonitor refCountMonitor) { + this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter, + refCountMonitor); + } + + private TextureBufferImpl(int unscaledWidth, int unscaledHeight, int width, int height, Type type, + int id, Matrix transformMatrix, Handler toI420Handler, YuvConverter yuvConverter, + RefCountMonitor refCountMonitor) { + this.unscaledWidth = unscaledWidth; + this.unscaledHeight = unscaledHeight; + this.width = width; + this.height = height; + this.type = type; + this.id = id; + this.transformMatrix = transformMatrix; + this.toI420Handler = toI420Handler; + this.yuvConverter = yuvConverter; + this.refCountDelegate = new RefCountDelegate(() -> refCountMonitor.onDestroy(this)); + this.refCountMonitor = refCountMonitor; + } + + @Override + public VideoFrame.TextureBuffer.Type getType() { + return type; + } + + @Override + public int getTextureId() { + return id; + } + + @Override + public Matrix getTransformMatrix() { + return transformMatrix; + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public VideoFrame.I420Buffer toI420() { + return ThreadUtils.invokeAtFrontUninterruptibly( + toI420Handler, () -> yuvConverter.convert(this)); + } + + @Override + public void retain() { + refCountMonitor.onRetain(this); + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountMonitor.onRelease(this); + refCountDelegate.release(); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + final Matrix cropAndScaleMatrix = new Matrix(); + // In WebRTC, Y=0 is the top row, while in OpenGL Y=0 is the bottom row. This means that the Y + // direction is effectively reversed. + final int cropYFromBottom = height - (cropY + cropHeight); + cropAndScaleMatrix.preTranslate(cropX / (float) width, cropYFromBottom / (float) height); + cropAndScaleMatrix.preScale(cropWidth / (float) width, cropHeight / (float) height); + + return applyTransformMatrix(cropAndScaleMatrix, + Math.round(unscaledWidth * cropWidth / (float) width), + Math.round(unscaledHeight * cropHeight / (float) height), scaleWidth, scaleHeight); + } + + @Override + public int getUnscaledWidth() { + return unscaledWidth; + } + + @Override + public int getUnscaledHeight() { + return unscaledHeight; + } + + public Handler getToI420Handler() { + return toI420Handler; + } + + public YuvConverter getYuvConverter() { + return yuvConverter; + } + + /** + * Create a new TextureBufferImpl with an applied transform matrix and a new size. The + * existing buffer is unchanged. The given transform matrix is applied first when texture + * coordinates are still in the unmodified [0, 1] range. + */ + @Override + public TextureBufferImpl applyTransformMatrix( + Matrix transformMatrix, int newWidth, int newHeight) { + return applyTransformMatrix(transformMatrix, /* unscaledWidth= */ newWidth, + /* unscaledHeight= */ newHeight, /* scaledWidth= */ newWidth, + /* scaledHeight= */ newHeight); + } + + private TextureBufferImpl applyTransformMatrix(Matrix transformMatrix, int unscaledWidth, + int unscaledHeight, int scaledWidth, int scaledHeight) { + final Matrix newMatrix = new Matrix(this.transformMatrix); + newMatrix.preConcat(transformMatrix); + retain(); + return new TextureBufferImpl(unscaledWidth, unscaledHeight, scaledWidth, scaledHeight, type, id, + newMatrix, toI420Handler, yuvConverter, new RefCountMonitor() { + @Override + public void onRetain(TextureBufferImpl textureBuffer) { + refCountMonitor.onRetain(TextureBufferImpl.this); + } + + @Override + public void onRelease(TextureBufferImpl textureBuffer) { + refCountMonitor.onRelease(TextureBufferImpl.this); + } + + @Override + public void onDestroy(TextureBufferImpl textureBuffer) { + release(); + } + }); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/TimestampAligner.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/TimestampAligner.java new file mode 100644 index 0000000000..d96c939595 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/TimestampAligner.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * The TimestampAligner class helps translating camera timestamps into the same timescale as is + * used by rtc::TimeNanos(). Some cameras have built in timestamping which is more accurate than + * reading the system clock, but using a different epoch and unknown clock drift. Frame timestamps + * in webrtc should use rtc::TimeNanos (system monotonic time), and this class provides a filter + * which lets us use the rtc::TimeNanos timescale, and at the same time take advantage of higher + * accuracy of the camera clock. This class is a wrapper on top of rtc::TimestampAligner. + */ +public class TimestampAligner { + /** + * Wrapper around rtc::TimeNanos(). This is normally same as System.nanoTime(), but call this + * function to be safe. + */ + public static long getRtcTimeNanos() { + return nativeRtcTimeNanos(); + } + + private volatile long nativeTimestampAligner = nativeCreateTimestampAligner(); + + /** + * Translates camera timestamps to the same timescale as is used by rtc::TimeNanos(). + * `cameraTimeNs` is assumed to be accurate, but with an unknown epoch and clock drift. Returns + * the translated timestamp. + */ + public long translateTimestamp(long cameraTimeNs) { + checkNativeAlignerExists(); + return nativeTranslateTimestamp(nativeTimestampAligner, cameraTimeNs); + } + + /** Dispose native timestamp aligner. */ + public void dispose() { + checkNativeAlignerExists(); + nativeReleaseTimestampAligner(nativeTimestampAligner); + nativeTimestampAligner = 0; + } + + private void checkNativeAlignerExists() { + if (nativeTimestampAligner == 0) { + throw new IllegalStateException("TimestampAligner has been disposed."); + } + } + + private static native long nativeRtcTimeNanos(); + private static native long nativeCreateTimestampAligner(); + private static native void nativeReleaseTimestampAligner(long timestampAligner); + private static native long nativeTranslateTimestamp(long timestampAligner, long cameraTimeNs); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/TurnCustomizer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/TurnCustomizer.java new file mode 100644 index 0000000000..41bedb7dcb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/TurnCustomizer.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Java wrapper for a C++ TurnCustomizer. */ +public class TurnCustomizer { + private long nativeTurnCustomizer; + + public TurnCustomizer(long nativeTurnCustomizer) { + this.nativeTurnCustomizer = nativeTurnCustomizer; + } + + public void dispose() { + checkTurnCustomizerExists(); + nativeFreeTurnCustomizer(nativeTurnCustomizer); + nativeTurnCustomizer = 0; + } + + private static native void nativeFreeTurnCustomizer(long turnCustomizer); + + /** Return a pointer to webrtc::TurnCustomizer. */ + @CalledByNative + long getNativeTurnCustomizer() { + checkTurnCustomizerExists(); + return nativeTurnCustomizer; + } + + private void checkTurnCustomizerExists() { + if (nativeTurnCustomizer == 0) { + throw new IllegalStateException("TurnCustomizer has been disposed."); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCapturer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCapturer.java new file mode 100644 index 0000000000..67eb7ab086 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCapturer.java @@ -0,0 +1,53 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; + +// Base interface for all VideoCapturers to implement. +public interface VideoCapturer { + /** + * This function is used to initialize the camera thread, the android application context, and the + * capture observer. It will be called only once and before any startCapture() request. The + * camera thread is guaranteed to be valid until dispose() is called. If the VideoCapturer wants + * to deliver texture frames, it should do this by rendering on the SurfaceTexture in + * {@code surfaceTextureHelper}, register itself as a listener, and forward the frames to + * CapturerObserver.onFrameCaptured(). The caller still has ownership of {@code + * surfaceTextureHelper} and is responsible for making sure surfaceTextureHelper.dispose() is + * called. This also means that the caller can reuse the SurfaceTextureHelper to initialize a new + * VideoCapturer once the previous VideoCapturer has been disposed. + */ + void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext, + CapturerObserver capturerObserver); + + /** + * Start capturing frames in a format that is as close as possible to {@code width x height} and + * {@code framerate}. + */ + void startCapture(int width, int height, int framerate); + + /** + * Stop capturing. This function should block until capture is actually stopped. + */ + void stopCapture() throws InterruptedException; + + void changeCaptureFormat(int width, int height, int framerate); + + /** + * Perform any final cleanup here. No more capturing will be done after this call. + */ + void dispose(); + + /** + * @return true if-and-only-if this is a screen capturer. + */ + boolean isScreencast(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java new file mode 100644 index 0000000000..363be347b5 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -0,0 +1,86 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; + +/** + * Represent a video codec as encoded in SDP. + */ +public class VideoCodecInfo { + // Keys for H264 VideoCodecInfo properties. + public static final String H264_FMTP_PROFILE_LEVEL_ID = "profile-level-id"; + public static final String H264_FMTP_LEVEL_ASYMMETRY_ALLOWED = "level-asymmetry-allowed"; + public static final String H264_FMTP_PACKETIZATION_MODE = "packetization-mode"; + + public static final String H264_PROFILE_CONSTRAINED_BASELINE = "42e0"; + public static final String H264_PROFILE_CONSTRAINED_HIGH = "640c"; + public static final String H264_LEVEL_3_1 = "1f"; // 31 in hex. + public static final String H264_CONSTRAINED_HIGH_3_1 = + H264_PROFILE_CONSTRAINED_HIGH + H264_LEVEL_3_1; + public static final String H264_CONSTRAINED_BASELINE_3_1 = + H264_PROFILE_CONSTRAINED_BASELINE + H264_LEVEL_3_1; + + public final String name; + public final Map<String, String> params; + @Deprecated public final int payload; + + @CalledByNative + public VideoCodecInfo(String name, Map<String, String> params) { + this.payload = 0; + this.name = name; + this.params = params; + } + + @Deprecated + public VideoCodecInfo(int payload, String name, Map<String, String> params) { + this.payload = payload; + this.name = name; + this.params = params; + } + + @Override + public boolean equals(@Nullable Object obj) { + if (obj == null) + return false; + if (obj == this) + return true; + if (!(obj instanceof VideoCodecInfo)) + return false; + + VideoCodecInfo otherInfo = (VideoCodecInfo) obj; + return name.equalsIgnoreCase(otherInfo.name) && params.equals(otherInfo.params); + } + + @Override + public int hashCode() { + Object[] values = {name.toUpperCase(Locale.ROOT), params}; + return Arrays.hashCode(values); + } + + @Override + public String toString() { + return "VideoCodec{" + name + " " + params + "}"; + } + + @CalledByNative + String getName() { + return name; + } + + @CalledByNative + Map<String, String> getParams() { + return params; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecStatus.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecStatus.java new file mode 100644 index 0000000000..670d255880 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoCodecStatus.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Status codes reported by video encoding/decoding components. This should be kept in sync with + * video_error_codes.h. + */ +public enum VideoCodecStatus { + TARGET_BITRATE_OVERSHOOT(5), + REQUEST_SLI(2), + NO_OUTPUT(1), + OK(0), + ERROR(-1), + LEVEL_EXCEEDED(-2), + MEMORY(-3), + ERR_PARAMETER(-4), + ERR_SIZE(-5), + TIMEOUT(-6), + UNINITIALIZED(-7), + ERR_REQUEST_SLI(-12), + FALLBACK_SOFTWARE(-13); + + private final int number; + + private VideoCodecStatus(int number) { + this.number = number; + } + + @CalledByNative + public int getNumber() { + return number; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoder.java new file mode 100644 index 0000000000..a80fa4fef2 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoder.java @@ -0,0 +1,94 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Interface for a video decoder that can be used in WebRTC. All calls to the class will be made on + * a single decoding thread. + */ +public interface VideoDecoder { + /** Settings passed to the decoder by WebRTC. */ + public class Settings { + public final int numberOfCores; + public final int width; + public final int height; + + @CalledByNative("Settings") + public Settings(int numberOfCores, int width, int height) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + } + } + + /** Additional info for decoding. */ + public class DecodeInfo { + public final boolean isMissingFrames; + public final long renderTimeMs; + + public DecodeInfo(boolean isMissingFrames, long renderTimeMs) { + this.isMissingFrames = isMissingFrames; + this.renderTimeMs = renderTimeMs; + } + } + + public interface Callback { + /** + * Call to return a decoded frame. Can be called on any thread. + * + * @param frame Decoded frame + * @param decodeTimeMs Time it took to decode the frame in milliseconds or null if not available + * @param qp QP value of the decoded frame or null if not available + */ + void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp); + } + + /** + * The decoder implementation backing this interface is either 1) a Java + * decoder (e.g., an Android platform decoder), or alternatively 2) a native + * decoder (e.g., a software decoder or a C++ decoder adapter). + * + * For case 1), createNativeVideoDecoder() should return zero. + * In this case, we expect the native library to call the decoder through + * JNI using the Java interface declared below. + * + * For case 2), createNativeVideoDecoder() should return a non-zero value. + * In this case, we expect the native library to treat the returned value as + * a raw pointer of type webrtc::VideoDecoder* (ownership is transferred to + * the caller). The native library should then directly call the + * webrtc::VideoDecoder interface without going through JNI. All calls to + * the Java interface methods declared below should thus throw an + * UnsupportedOperationException. + */ + @CalledByNative + default long createNativeVideoDecoder() { + return 0; + } + + /** + * Initializes the decoding process with specified settings. Will be called on the decoding thread + * before any decode calls. + */ + @CalledByNative VideoCodecStatus initDecode(Settings settings, Callback decodeCallback); + /** + * Called when the decoder is no longer needed. Any more calls to decode will not be made. + */ + @CalledByNative VideoCodecStatus release(); + /** + * Request the decoder to decode a frame. + */ + @CalledByNative VideoCodecStatus decode(EncodedImage frame, DecodeInfo info); + /** + * Should return a descriptive name for the implementation. Gets called once and cached. May be + * called from arbitrary thread. + */ + @CalledByNative String getImplementationName(); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFactory.java new file mode 100644 index 0000000000..8b25516e99 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFactory.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +/** Factory for creating VideoDecoders. */ +public interface VideoDecoderFactory { + /** + * Creates a VideoDecoder for the given codec. Supports the same codecs supported by + * VideoEncoderFactory. + */ + @Nullable @CalledByNative VideoDecoder createDecoder(VideoCodecInfo info); + + /** + * Enumerates the list of supported video codecs. + */ + @CalledByNative + default VideoCodecInfo[] getSupportedCodecs() { + return new VideoCodecInfo[0]; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFallback.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFallback.java new file mode 100644 index 0000000000..ddfa3ecd40 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoDecoderFallback.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * A combined video decoder that falls back on a secondary decoder if the primary decoder fails. + */ +public class VideoDecoderFallback extends WrappedNativeVideoDecoder { + private final VideoDecoder fallback; + private final VideoDecoder primary; + + public VideoDecoderFallback(VideoDecoder fallback, VideoDecoder primary) { + this.fallback = fallback; + this.primary = primary; + } + + @Override + public long createNativeVideoDecoder() { + return nativeCreateDecoder(fallback, primary); + } + + private static native long nativeCreateDecoder(VideoDecoder fallback, VideoDecoder primary); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoder.java new file mode 100644 index 0000000000..0d8cf830ae --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoder.java @@ -0,0 +1,385 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import org.webrtc.EncodedImage; + +/** + * Interface for a video encoder that can be used with WebRTC. All calls will be made on the + * encoding thread. The encoder may be constructed on a different thread and changing thread after + * calling release is allowed. + */ +public interface VideoEncoder { + /** Settings passed to the encoder by WebRTC. */ + public class Settings { + public final int numberOfCores; + public final int width; + public final int height; + public final int startBitrate; // Kilobits per second. + public final int maxFramerate; + public final int numberOfSimulcastStreams; + public final boolean automaticResizeOn; + public final Capabilities capabilities; + + // TODO(bugs.webrtc.org/10720): Remove. + @Deprecated + public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate, + int numberOfSimulcastStreams, boolean automaticResizeOn) { + this(numberOfCores, width, height, startBitrate, maxFramerate, numberOfSimulcastStreams, + automaticResizeOn, new VideoEncoder.Capabilities(false /* lossNotification */)); + } + + @CalledByNative("Settings") + public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate, + int numberOfSimulcastStreams, boolean automaticResizeOn, Capabilities capabilities) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + this.startBitrate = startBitrate; + this.maxFramerate = maxFramerate; + this.numberOfSimulcastStreams = numberOfSimulcastStreams; + this.automaticResizeOn = automaticResizeOn; + this.capabilities = capabilities; + } + } + + /** Capabilities (loss notification, etc.) passed to the encoder by WebRTC. */ + public class Capabilities { + /** + * The remote side has support for the loss notification RTCP feedback message format, and will + * be sending these feedback messages if necessary. + */ + public final boolean lossNotification; + + @CalledByNative("Capabilities") + public Capabilities(boolean lossNotification) { + this.lossNotification = lossNotification; + } + } + + /** Additional info for encoding. */ + public class EncodeInfo { + public final EncodedImage.FrameType[] frameTypes; + + @CalledByNative("EncodeInfo") + public EncodeInfo(EncodedImage.FrameType[] frameTypes) { + this.frameTypes = frameTypes; + } + } + + // TODO(sakal): Add values to these classes as necessary. + /** Codec specific information about the encoded frame. */ + public class CodecSpecificInfo {} + + public class CodecSpecificInfoVP8 extends CodecSpecificInfo {} + + public class CodecSpecificInfoVP9 extends CodecSpecificInfo {} + + public class CodecSpecificInfoH264 extends CodecSpecificInfo {} + + public class CodecSpecificInfoAV1 extends CodecSpecificInfo {} + + /** + * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between + * spatial and temporal layers. + */ + public class BitrateAllocation { + // First index is the spatial layer and second the temporal layer. + public final int[][] bitratesBbs; + + /** + * Initializes the allocation with a two dimensional array of bitrates. The first index of the + * array is the spatial layer and the second index in the temporal layer. + */ + @CalledByNative("BitrateAllocation") + public BitrateAllocation(int[][] bitratesBbs) { + this.bitratesBbs = bitratesBbs; + } + + /** + * Gets the total bitrate allocated for all layers. + */ + public int getSum() { + int sum = 0; + for (int[] spatialLayer : bitratesBbs) { + for (int bitrate : spatialLayer) { + sum += bitrate; + } + } + return sum; + } + } + + /** Settings for WebRTC quality based scaling. */ + public class ScalingSettings { + public final boolean on; + @Nullable public final Integer low; + @Nullable public final Integer high; + + /** + * Settings to disable quality based scaling. + */ + public static final ScalingSettings OFF = new ScalingSettings(); + + /** + * Creates settings to enable quality based scaling. + * + * @param low Average QP at which to scale up the resolution. + * @param high Average QP at which to scale down the resolution. + */ + public ScalingSettings(int low, int high) { + this.on = true; + this.low = low; + this.high = high; + } + + private ScalingSettings() { + this.on = false; + this.low = null; + this.high = null; + } + + // TODO(bugs.webrtc.org/8830): Below constructors are deprecated. + // Default thresholds are going away, so thresholds have to be set + // when scaling is on. + /** + * Creates quality based scaling setting. + * + * @param on True if quality scaling is turned on. + */ + @Deprecated + public ScalingSettings(boolean on) { + this.on = on; + this.low = null; + this.high = null; + } + + /** + * Creates quality based scaling settings with custom thresholds. + * + * @param on True if quality scaling is turned on. + * @param low Average QP at which to scale up the resolution. + * @param high Average QP at which to scale down the resolution. + */ + @Deprecated + public ScalingSettings(boolean on, int low, int high) { + this.on = on; + this.low = low; + this.high = high; + } + + @Override + public String toString() { + return on ? "[ " + low + ", " + high + " ]" : "OFF"; + } + } + + /** + * Bitrate limits for resolution. + */ + public class ResolutionBitrateLimits { + /** + * Maximum size of video frame, in pixels, the bitrate limits are intended for. + */ + public final int frameSizePixels; + + /** + * Recommended minimum bitrate to start encoding. + */ + public final int minStartBitrateBps; + + /** + * Recommended minimum bitrate. + */ + public final int minBitrateBps; + + /** + * Recommended maximum bitrate. + */ + public final int maxBitrateBps; + + public ResolutionBitrateLimits( + int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) { + this.frameSizePixels = frameSizePixels; + this.minStartBitrateBps = minStartBitrateBps; + this.minBitrateBps = minBitrateBps; + this.maxBitrateBps = maxBitrateBps; + } + + @CalledByNative("ResolutionBitrateLimits") + public int getFrameSizePixels() { + return frameSizePixels; + } + + @CalledByNative("ResolutionBitrateLimits") + public int getMinStartBitrateBps() { + return minStartBitrateBps; + } + + @CalledByNative("ResolutionBitrateLimits") + public int getMinBitrateBps() { + return minBitrateBps; + } + + @CalledByNative("ResolutionBitrateLimits") + public int getMaxBitrateBps() { + return maxBitrateBps; + } + } + + /** Rate control parameters. */ + public class RateControlParameters { + /** + * Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target + * depending on encoder behaviour. + */ + public final BitrateAllocation bitrate; + + /** + * Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate + * target not available. In this case the encoder should fall back to the max framerate + * specified in `codec_settings` of the last InitEncode() call. + */ + public final double framerateFps; + + @CalledByNative("RateControlParameters") + public RateControlParameters(BitrateAllocation bitrate, double framerateFps) { + this.bitrate = bitrate; + this.framerateFps = framerateFps; + } + } + + /** + * Metadata about the Encoder. + */ + public class EncoderInfo { + /** + * The width and height of the incoming video frames should be divisible by + * |requested_resolution_alignment| + */ + public final int requestedResolutionAlignment; + + /** + * Same as above but if true, each simulcast layer should also be divisible by + * |requested_resolution_alignment|. + */ + public final boolean applyAlignmentToAllSimulcastLayers; + + public EncoderInfo( + int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) { + this.requestedResolutionAlignment = requestedResolutionAlignment; + this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers; + } + + @CalledByNative("EncoderInfo") + public int getRequestedResolutionAlignment() { + return requestedResolutionAlignment; + } + + @CalledByNative("EncoderInfo") + public boolean getApplyAlignmentToAllSimulcastLayers() { + return applyAlignmentToAllSimulcastLayers; + } + } + + public interface Callback { + /** + * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to + * this method returns. If the pipeline downstream needs to hold on to the buffer, it then has + * to make its own copy. We want to move to a model where no copying is needed, and instead use + * retain()/release() to signal to the encoder when it is safe to reuse the buffer. + * + * Over the transition, implementations of this class should use the maybeRetain() method if + * they want to keep a reference to the buffer, and fall back to copying if that method returns + * false. + */ + void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info); + } + + /** + * The encoder implementation backing this interface is either 1) a Java + * encoder (e.g., an Android platform encoder), or alternatively 2) a native + * encoder (e.g., a software encoder or a C++ encoder adapter). + * + * For case 1), createNativeVideoEncoder() should return zero. + * In this case, we expect the native library to call the encoder through + * JNI using the Java interface declared below. + * + * For case 2), createNativeVideoEncoder() should return a non-zero value. + * In this case, we expect the native library to treat the returned value as + * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to + * the caller). The native library should then directly call the + * webrtc::VideoEncoder interface without going through JNI. All calls to + * the Java interface methods declared below should thus throw an + * UnsupportedOperationException. + */ + @CalledByNative + default long createNativeVideoEncoder() { + return 0; + } + + /** + * Returns true if the encoder is backed by hardware. + */ + @CalledByNative + default boolean isHardwareEncoder() { + return true; + } + + /** + * Initializes the encoding process. Call before any calls to encode. + */ + @CalledByNative VideoCodecStatus initEncode(Settings settings, Callback encodeCallback); + + /** + * Releases the encoder. No more calls to encode will be made after this call. + */ + @CalledByNative VideoCodecStatus release(); + + /** + * Requests the encoder to encode a frame. + */ + @CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info); + + /** Sets the bitrate allocation and the target framerate for the encoder. */ + VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate); + + /** Sets the bitrate allocation and the target framerate for the encoder. */ + default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) { + // Round frame rate up to avoid overshoots. + int framerateFps = (int) Math.ceil(rcParameters.framerateFps); + return setRateAllocation(rcParameters.bitrate, framerateFps); + } + + /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */ + @CalledByNative ScalingSettings getScalingSettings(); + + /** Returns the list of bitrate limits. */ + @CalledByNative + default ResolutionBitrateLimits[] getResolutionBitrateLimits() { + // TODO(ssilkin): Update downstream projects and remove default implementation. + ResolutionBitrateLimits bitrate_limits[] = {}; + return bitrate_limits; + } + + /** + * Should return a descriptive name for the implementation. Gets called once and cached. May be + * called from arbitrary thread. + */ + @CalledByNative String getImplementationName(); + + @CalledByNative + default EncoderInfo getEncoderInfo() { + return new EncoderInfo( + /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFactory.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFactory.java new file mode 100644 index 0000000000..2a46662d14 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFactory.java @@ -0,0 +1,72 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +/** Factory for creating VideoEncoders. */ +public interface VideoEncoderFactory { + public interface VideoEncoderSelector { + /** Called with the VideoCodecInfo of the currently used encoder. */ + @CalledByNative("VideoEncoderSelector") void onCurrentEncoder(VideoCodecInfo info); + + /** + * Called with the current available bitrate. Returns null if the encoder selector prefers to + * keep the current encoder or a VideoCodecInfo if a new encoder is preferred. + */ + @Nullable @CalledByNative("VideoEncoderSelector") VideoCodecInfo onAvailableBitrate(int kbps); + + /** + * Called every time the encoder input resolution change. Returns null if the encoder selector + * prefers to keep the current encoder or a VideoCodecInfo if a new encoder is preferred. + */ + @Nullable + @CalledByNative("VideoEncoderSelector") + default VideoCodecInfo onResolutionChange(int widht, int height) { + return null; + } + + /** + * Called when the currently used encoder signal itself as broken. Returns null if the encoder + * selector prefers to keep the current encoder or a VideoCodecInfo if a new encoder is + * preferred. + */ + @Nullable @CalledByNative("VideoEncoderSelector") VideoCodecInfo onEncoderBroken(); + } + + /** Creates an encoder for the given video codec. */ + @Nullable @CalledByNative VideoEncoder createEncoder(VideoCodecInfo info); + + /** + * Enumerates the list of supported video codecs. This method will only be called once and the + * result will be cached. + */ + @CalledByNative VideoCodecInfo[] getSupportedCodecs(); + + /** + * Enumerates the list of supported video codecs that can also be tagged with + * implementation information. This method will only be called once and the + * result will be cached. + */ + @CalledByNative + default VideoCodecInfo[] getImplementations() { + return getSupportedCodecs(); + } + + /** + * Returns a VideoEncoderSelector if implemented by the VideoEncoderFactory, + * null otherwise. + */ + @CalledByNative + default VideoEncoderSelector getEncoderSelector() { + return null; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFallback.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFallback.java new file mode 100644 index 0000000000..fa36b7c989 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoEncoderFallback.java @@ -0,0 +1,36 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * A combined video encoder that falls back on a secondary encoder if the primary encoder fails. + */ +public class VideoEncoderFallback extends WrappedNativeVideoEncoder { + private final VideoEncoder fallback; + private final VideoEncoder primary; + + public VideoEncoderFallback(VideoEncoder fallback, VideoEncoder primary) { + this.fallback = fallback; + this.primary = primary; + } + + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(fallback, primary); + } + + @Override + public boolean isHardwareEncoder() { + return primary.isHardwareEncoder(); + } + + private static native long nativeCreateEncoder(VideoEncoder fallback, VideoEncoder primary); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFileRenderer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFileRenderer.java new file mode 100644 index 0000000000..aef8030459 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFileRenderer.java @@ -0,0 +1,162 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.os.Handler; +import android.os.HandlerThread; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.concurrent.CountDownLatch; + +/** + * Can be used to save the video frames to file. + */ +public class VideoFileRenderer implements VideoSink { + private static final String TAG = "VideoFileRenderer"; + + private final HandlerThread renderThread; + private final Handler renderThreadHandler; + private final HandlerThread fileThread; + private final Handler fileThreadHandler; + private final FileOutputStream videoOutFile; + private final String outputFileName; + private final int outputFileWidth; + private final int outputFileHeight; + private final int outputFrameSize; + private final ByteBuffer outputFrameBuffer; + private EglBase eglBase; + private YuvConverter yuvConverter; + private int frameCount; + + public VideoFileRenderer(String outputFile, int outputFileWidth, int outputFileHeight, + final EglBase.Context sharedContext) throws IOException { + if ((outputFileWidth % 2) == 1 || (outputFileHeight % 2) == 1) { + throw new IllegalArgumentException("Does not support uneven width or height"); + } + + this.outputFileName = outputFile; + this.outputFileWidth = outputFileWidth; + this.outputFileHeight = outputFileHeight; + + outputFrameSize = outputFileWidth * outputFileHeight * 3 / 2; + outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); + + videoOutFile = new FileOutputStream(outputFile); + videoOutFile.write( + ("YUV4MPEG2 C420 W" + outputFileWidth + " H" + outputFileHeight + " Ip F30:1 A1:1\n") + .getBytes(Charset.forName("US-ASCII"))); + + renderThread = new HandlerThread(TAG + "RenderThread"); + renderThread.start(); + renderThreadHandler = new Handler(renderThread.getLooper()); + + fileThread = new HandlerThread(TAG + "FileThread"); + fileThread.start(); + fileThreadHandler = new Handler(fileThread.getLooper()); + + ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, new Runnable() { + @Override + public void run() { + eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); + eglBase.createDummyPbufferSurface(); + eglBase.makeCurrent(); + yuvConverter = new YuvConverter(); + } + }); + } + + @Override + public void onFrame(VideoFrame frame) { + frame.retain(); + renderThreadHandler.post(() -> renderFrameOnRenderThread(frame)); + } + + private void renderFrameOnRenderThread(VideoFrame frame) { + final VideoFrame.Buffer buffer = frame.getBuffer(); + + // If the frame is rotated, it will be applied after cropAndScale. Therefore, if the frame is + // rotated by 90 degrees, swap width and height. + final int targetWidth = frame.getRotation() % 180 == 0 ? outputFileWidth : outputFileHeight; + final int targetHeight = frame.getRotation() % 180 == 0 ? outputFileHeight : outputFileWidth; + + final float frameAspectRatio = (float) buffer.getWidth() / (float) buffer.getHeight(); + final float fileAspectRatio = (float) targetWidth / (float) targetHeight; + + // Calculate cropping to equalize the aspect ratio. + int cropWidth = buffer.getWidth(); + int cropHeight = buffer.getHeight(); + if (fileAspectRatio > frameAspectRatio) { + cropHeight = (int) (cropHeight * (frameAspectRatio / fileAspectRatio)); + } else { + cropWidth = (int) (cropWidth * (fileAspectRatio / frameAspectRatio)); + } + + final int cropX = (buffer.getWidth() - cropWidth) / 2; + final int cropY = (buffer.getHeight() - cropHeight) / 2; + + final VideoFrame.Buffer scaledBuffer = + buffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, targetWidth, targetHeight); + frame.release(); + + final VideoFrame.I420Buffer i420 = scaledBuffer.toI420(); + scaledBuffer.release(); + + fileThreadHandler.post(() -> { + YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), + i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), + frame.getRotation()); + i420.release(); + + try { + videoOutFile.write("FRAME\n".getBytes(Charset.forName("US-ASCII"))); + videoOutFile.write( + outputFrameBuffer.array(), outputFrameBuffer.arrayOffset(), outputFrameSize); + } catch (IOException e) { + throw new RuntimeException("Error writing video to disk", e); + } + frameCount++; + }); + } + + /** + * Release all resources. All already posted frames will be rendered first. + */ + public void release() { + final CountDownLatch cleanupBarrier = new CountDownLatch(1); + renderThreadHandler.post(() -> { + yuvConverter.release(); + eglBase.release(); + renderThread.quit(); + cleanupBarrier.countDown(); + }); + ThreadUtils.awaitUninterruptibly(cleanupBarrier); + fileThreadHandler.post(() -> { + try { + videoOutFile.close(); + Logging.d(TAG, + "Video written to disk as " + outputFileName + ". The number of frames is " + frameCount + + " and the dimensions of the frames are " + outputFileWidth + "x" + + outputFileHeight + "."); + } catch (IOException e) { + throw new RuntimeException("Error closing output file", e); + } + fileThread.quit(); + }); + try { + fileThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Logging.e(TAG, "Interrupted while waiting for the write to disk to complete.", e); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrame.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrame.java new file mode 100644 index 0000000000..443a0315af --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrame.java @@ -0,0 +1,234 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * Java version of webrtc::VideoFrame and webrtc::VideoFrameBuffer. A difference from the C++ + * version is that no explicit tag is used, and clients are expected to use 'instanceof' to find the + * right subclass of the buffer. This allows clients to create custom VideoFrame.Buffer in + * arbitrary format in their custom VideoSources, and then cast it back to the correct subclass in + * their custom VideoSinks. All implementations must also implement the toI420() function, + * converting from the underlying representation if necessary. I420 is the most widely accepted + * format and serves as a fallback for video sinks that can only handle I420, e.g. the internal + * WebRTC software encoders. + */ +public class VideoFrame implements RefCounted { + /** + * Implements image storage medium. Might be for example an OpenGL texture or a memory region + * containing I420-data. + * + * <p>Reference counting is needed since a video buffer can be shared between multiple VideoSinks, + * and the buffer needs to be returned to the VideoSource as soon as all references are gone. + */ + public interface Buffer extends RefCounted { + /** + * Representation of the underlying buffer. Currently, only NATIVE and I420 are supported. + */ + @CalledByNative("Buffer") + @VideoFrameBufferType + default int getBufferType() { + return VideoFrameBufferType.NATIVE; + } + + /** + * Resolution of the buffer in pixels. + */ + @CalledByNative("Buffer") int getWidth(); + @CalledByNative("Buffer") int getHeight(); + + /** + * Returns a memory-backed frame in I420 format. If the pixel data is in another format, a + * conversion will take place. All implementations must provide a fallback to I420 for + * compatibility with e.g. the internal WebRTC software encoders. + * + * <p> Conversion may fail, for example if reading the pixel data from a texture fails. If the + * conversion fails, null is returned. + */ + @Nullable @CalledByNative("Buffer") I420Buffer toI420(); + + @Override @CalledByNative("Buffer") void retain(); + @Override @CalledByNative("Buffer") void release(); + + /** + * Crops a region defined by `cropx`, `cropY`, `cropWidth` and `cropHeight`. Scales it to size + * `scaleWidth` x `scaleHeight`. + */ + @CalledByNative("Buffer") + Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight); + } + + /** + * Interface for I420 buffers. + */ + public interface I420Buffer extends Buffer { + @Override + default int getBufferType() { + return VideoFrameBufferType.I420; + } + + /** + * Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least + * getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must + * be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so + * implementations must return a new ByteBuffer or slice for each call. + */ + @CalledByNative("I420Buffer") ByteBuffer getDataY(); + /** + * Returns a direct ByteBuffer containing U-plane data. The buffer capacity is at least + * getStrideU() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored + * and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so + * implementations must return a new ByteBuffer or slice for each call. + */ + @CalledByNative("I420Buffer") ByteBuffer getDataU(); + /** + * Returns a direct ByteBuffer containing V-plane data. The buffer capacity is at least + * getStrideV() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored + * and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so + * implementations must return a new ByteBuffer or slice for each call. + */ + @CalledByNative("I420Buffer") ByteBuffer getDataV(); + + @CalledByNative("I420Buffer") int getStrideY(); + @CalledByNative("I420Buffer") int getStrideU(); + @CalledByNative("I420Buffer") int getStrideV(); + } + + /** + * Interface for buffers that are stored as a single texture, either in OES or RGB format. + */ + public interface TextureBuffer extends Buffer { + enum Type { + OES(GLES11Ext.GL_TEXTURE_EXTERNAL_OES), + RGB(GLES20.GL_TEXTURE_2D); + + private final int glTarget; + + private Type(final int glTarget) { + this.glTarget = glTarget; + } + + public int getGlTarget() { + return glTarget; + } + } + + Type getType(); + int getTextureId(); + + /** + * Retrieve the transform matrix associated with the frame. This transform matrix maps 2D + * homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to + * the coordinate that should be used to sample that location from the buffer. + */ + Matrix getTransformMatrix(); + + /** + * Create a new TextureBufferImpl with an applied transform matrix and a new size. The existing + * buffer is unchanged. The given transform matrix is applied first when texture coordinates are + * still in the unmodified [0, 1] range. + */ + default TextureBuffer applyTransformMatrix( + Matrix transformMatrix, int newWidth, int newHeight) { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Returns the width of the texture in memory. This should only be used for downscaling, and you + * should still respect the width from getWidth(). + */ + default public int getUnscaledWidth() { + return getWidth(); + } + + /** + * Returns the height of the texture in memory. This should only be used for downscaling, and + * you should still respect the height from getHeight(). + */ + default public int getUnscaledHeight() { + return getHeight(); + } + } + + private final Buffer buffer; + private final int rotation; + private final long timestampNs; + + /** + * Constructs a new VideoFrame backed by the given {@code buffer}. + * + * @note Ownership of the buffer object is tranferred to the new VideoFrame. + */ + @CalledByNative + public VideoFrame(Buffer buffer, int rotation, long timestampNs) { + if (buffer == null) { + throw new IllegalArgumentException("buffer not allowed to be null"); + } + if (rotation % 90 != 0) { + throw new IllegalArgumentException("rotation must be a multiple of 90"); + } + this.buffer = buffer; + this.rotation = rotation; + this.timestampNs = timestampNs; + } + + @CalledByNative + public Buffer getBuffer() { + return buffer; + } + + /** + * Rotation of the frame in degrees. + */ + @CalledByNative + public int getRotation() { + return rotation; + } + + /** + * Timestamp of the frame in nano seconds. + */ + @CalledByNative + public long getTimestampNs() { + return timestampNs; + } + + public int getRotatedWidth() { + if (rotation % 180 == 0) { + return buffer.getWidth(); + } + return buffer.getHeight(); + } + + public int getRotatedHeight() { + if (rotation % 180 == 0) { + return buffer.getHeight(); + } + return buffer.getWidth(); + } + + @Override + public void retain() { + buffer.retain(); + } + + @Override + @CalledByNative + public void release() { + buffer.release(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameBufferType.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameBufferType.java new file mode 100644 index 0000000000..7b05b88cba --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameBufferType.java @@ -0,0 +1,33 @@ + +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file is autogenerated by +// java_cpp_enum.py +// From +// ../../api/video/video_frame_buffer.h + +package org.webrtc; + +import androidx.annotation.IntDef; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@IntDef({ + VideoFrameBufferType.NATIVE, VideoFrameBufferType.I420, VideoFrameBufferType.I420A, + VideoFrameBufferType.I422, VideoFrameBufferType.I444, VideoFrameBufferType.I010, + VideoFrameBufferType.I210, VideoFrameBufferType.NV12 +}) +@Retention(RetentionPolicy.SOURCE) +public @interface VideoFrameBufferType { + int NATIVE = 0; + int I420 = 1; + int I420A = 2; + int I422 = 3; + int I444 = 4; + int I010 = 5; + int I210 = 6; + int NV12 = 7; +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java new file mode 100644 index 0000000000..af32587886 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoFrameDrawer.java @@ -0,0 +1,241 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.graphics.Point; +import android.opengl.GLES20; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * Helper class to draw VideoFrames. Calls either drawer.drawOes, drawer.drawRgb, or + * drawer.drawYuv depending on the type of the buffer. The frame will be rendered with rotation + * taken into account. You can supply an additional render matrix for custom transformations. + */ +public class VideoFrameDrawer { + public static final String TAG = "VideoFrameDrawer"; + /** + * Draws a VideoFrame.TextureBuffer. Calls either drawer.drawOes or drawer.drawRgb + * depending on the type of the buffer. You can supply an additional render matrix. This is + * used multiplied together with the transformation matrix of the frame. (M = renderMatrix * + * transformationMatrix) + */ + public static void drawTexture(RendererCommon.GlDrawer drawer, VideoFrame.TextureBuffer buffer, + Matrix renderMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY, + int viewportWidth, int viewportHeight) { + Matrix finalMatrix = new Matrix(buffer.getTransformMatrix()); + finalMatrix.preConcat(renderMatrix); + float[] finalGlMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(finalMatrix); + switch (buffer.getType()) { + case OES: + drawer.drawOes(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, + viewportY, viewportWidth, viewportHeight); + break; + case RGB: + drawer.drawRgb(buffer.getTextureId(), finalGlMatrix, frameWidth, frameHeight, viewportX, + viewportY, viewportWidth, viewportHeight); + break; + default: + throw new RuntimeException("Unknown texture type."); + } + } + + /** + * Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This + * class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies. + */ + private static class YuvUploader { + // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. + // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader + // that handles stride and compare performance with intermediate copy. + @Nullable private ByteBuffer copyBuffer; + @Nullable private int[] yuvTextures; + + /** + * Upload `planes` into OpenGL textures, taking stride into consideration. + * + * @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively. + */ + @Nullable + public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) { + final int[] planeWidths = new int[] {width, width / 2, width / 2}; + final int[] planeHeights = new int[] {height, height / 2, height / 2}; + // Make a first pass to see if we need a temporary copy buffer. + int copyCapacityNeeded = 0; + for (int i = 0; i < 3; ++i) { + if (strides[i] > planeWidths[i]) { + copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]); + } + } + // Allocate copy buffer if necessary. + if (copyCapacityNeeded > 0 + && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { + copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); + } + // Make sure YUV textures are allocated. + if (yuvTextures == null) { + yuvTextures = new int[3]; + for (int i = 0; i < 3; i++) { + yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); + } + } + // Upload each plane. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); + // GLES only accepts packed data, i.e. stride == planeWidth. + final ByteBuffer packedByteBuffer; + if (strides[i] == planeWidths[i]) { + // Input is packed already. + packedByteBuffer = planes[i]; + } else { + YuvHelper.copyPlane( + planes[i], strides[i], copyBuffer, planeWidths[i], planeWidths[i], planeHeights[i]); + packedByteBuffer = copyBuffer; + } + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i], + planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); + } + return yuvTextures; + } + + @Nullable + public int[] uploadFromBuffer(VideoFrame.I420Buffer buffer) { + int[] strides = {buffer.getStrideY(), buffer.getStrideU(), buffer.getStrideV()}; + ByteBuffer[] planes = {buffer.getDataY(), buffer.getDataU(), buffer.getDataV()}; + return uploadYuvData(buffer.getWidth(), buffer.getHeight(), strides, planes); + } + + @Nullable + public int[] getYuvTextures() { + return yuvTextures; + } + + /** + * Releases cached resources. Uploader can still be used and the resources will be reallocated + * on first use. + */ + public void release() { + copyBuffer = null; + if (yuvTextures != null) { + GLES20.glDeleteTextures(3, yuvTextures, 0); + yuvTextures = null; + } + } + } + + private static int distance(float x0, float y0, float x1, float y1) { + return (int) Math.round(Math.hypot(x1 - x0, y1 - y0)); + } + + // These points are used to calculate the size of the part of the frame we are rendering. + final static float[] srcPoints = + new float[] {0f /* x0 */, 0f /* y0 */, 1f /* x1 */, 0f /* y1 */, 0f /* x2 */, 1f /* y2 */}; + private final float[] dstPoints = new float[6]; + private final Point renderSize = new Point(); + private int renderWidth; + private int renderHeight; + + // Calculate the frame size after `renderMatrix` is applied. Stores the output in member variables + // `renderWidth` and `renderHeight` to avoid allocations since this function is called for every + // frame. + private void calculateTransformedRenderSize( + int frameWidth, int frameHeight, @Nullable Matrix renderMatrix) { + if (renderMatrix == null) { + renderWidth = frameWidth; + renderHeight = frameHeight; + return; + } + // Transform the texture coordinates (in the range [0, 1]) according to `renderMatrix`. + renderMatrix.mapPoints(dstPoints, srcPoints); + + // Multiply with the width and height to get the positions in terms of pixels. + for (int i = 0; i < 3; ++i) { + dstPoints[i * 2 + 0] *= frameWidth; + dstPoints[i * 2 + 1] *= frameHeight; + } + + // Get the length of the sides of the transformed rectangle in terms of pixels. + renderWidth = distance(dstPoints[0], dstPoints[1], dstPoints[2], dstPoints[3]); + renderHeight = distance(dstPoints[0], dstPoints[1], dstPoints[4], dstPoints[5]); + } + + private final YuvUploader yuvUploader = new YuvUploader(); + // This variable will only be used for checking reference equality and is used for caching I420 + // textures. + @Nullable private VideoFrame lastI420Frame; + private final Matrix renderMatrix = new Matrix(); + + public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer) { + drawFrame(frame, drawer, null /* additionalRenderMatrix */); + } + + public void drawFrame( + VideoFrame frame, RendererCommon.GlDrawer drawer, Matrix additionalRenderMatrix) { + drawFrame(frame, drawer, additionalRenderMatrix, 0 /* viewportX */, 0 /* viewportY */, + frame.getRotatedWidth(), frame.getRotatedHeight()); + } + + public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer, + @Nullable Matrix additionalRenderMatrix, int viewportX, int viewportY, int viewportWidth, + int viewportHeight) { + final int width = frame.getRotatedWidth(); + final int height = frame.getRotatedHeight(); + calculateTransformedRenderSize(width, height, additionalRenderMatrix); + if (renderWidth <= 0 || renderHeight <= 0) { + Logging.w(TAG, "Illegal frame size: " + renderWidth + "x" + renderHeight); + return; + } + + final boolean isTextureFrame = frame.getBuffer() instanceof VideoFrame.TextureBuffer; + renderMatrix.reset(); + renderMatrix.preTranslate(0.5f, 0.5f); + if (!isTextureFrame) { + renderMatrix.preScale(1f, -1f); // I420-frames are upside down + } + renderMatrix.preRotate(frame.getRotation()); + renderMatrix.preTranslate(-0.5f, -0.5f); + if (additionalRenderMatrix != null) { + renderMatrix.preConcat(additionalRenderMatrix); + } + + if (isTextureFrame) { + lastI420Frame = null; + drawTexture(drawer, (VideoFrame.TextureBuffer) frame.getBuffer(), renderMatrix, renderWidth, + renderHeight, viewportX, viewportY, viewportWidth, viewportHeight); + } else { + // Only upload the I420 data to textures once per frame, if we are called multiple times + // with the same frame. + if (frame != lastI420Frame) { + lastI420Frame = frame; + final VideoFrame.I420Buffer i420Buffer = frame.getBuffer().toI420(); + yuvUploader.uploadFromBuffer(i420Buffer); + i420Buffer.release(); + } + + drawer.drawYuv(yuvUploader.getYuvTextures(), + RendererCommon.convertMatrixFromAndroidGraphicsMatrix(renderMatrix), renderWidth, + renderHeight, viewportX, viewportY, viewportWidth, viewportHeight); + } + } + + public VideoFrame.Buffer prepareBufferForViewportSize( + VideoFrame.Buffer buffer, int width, int height) { + buffer.retain(); + return buffer; + } + + public void release() { + yuvUploader.release(); + lastI420Frame = null; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoProcessor.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoProcessor.java new file mode 100644 index 0000000000..c39a55c27e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoProcessor.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +/** + * Lightweight abstraction for an object that can receive video frames, process them, and pass them + * on to another object. This object is also allowed to observe capturer start/stop. + */ +public interface VideoProcessor extends CapturerObserver { + public static class FrameAdaptationParameters { + public final int cropX; + public final int cropY; + public final int cropWidth; + public final int cropHeight; + public final int scaleWidth; + public final int scaleHeight; + public final long timestampNs; + public final boolean drop; + + public FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, + int scaleWidth, int scaleHeight, long timestampNs, boolean drop) { + this.cropX = cropX; + this.cropY = cropY; + this.cropWidth = cropWidth; + this.cropHeight = cropHeight; + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; + this.timestampNs = timestampNs; + this.drop = drop; + } + } + + /** + * This is a chance to access an unadapted frame. The default implementation applies the + * adaptation and forwards the frame to {@link #onFrameCaptured(VideoFrame)}. + */ + default void onFrameCaptured(VideoFrame frame, FrameAdaptationParameters parameters) { + VideoFrame adaptedFrame = applyFrameAdaptationParameters(frame, parameters); + if (adaptedFrame != null) { + onFrameCaptured(adaptedFrame); + adaptedFrame.release(); + } + } + + /** + * Set the sink that receives the output from this processor. Null can be passed in to unregister + * a sink. + */ + void setSink(@Nullable VideoSink sink); + + /** + * Applies the frame adaptation parameters to a frame. Returns null if the frame is meant to be + * dropped. Returns a new frame. The caller is responsible for releasing the returned frame. + */ + public static @Nullable VideoFrame applyFrameAdaptationParameters( + VideoFrame frame, FrameAdaptationParameters parameters) { + if (parameters.drop) { + return null; + } + + final VideoFrame.Buffer adaptedBuffer = + frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, + parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); + return new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSink.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSink.java new file mode 100644 index 0000000000..5a0a6c719c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSink.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Java version of rtc::VideoSinkInterface. + */ +public interface VideoSink { + /** + * Implementations should call frame.retain() if they need to hold a reference to the frame after + * this function returns. Each call to retain() should be followed by a call to frame.release() + * when the reference is no longer needed. + */ + @CalledByNative void onFrame(VideoFrame frame); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSource.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSource.java new file mode 100644 index 0000000000..2e22d1a2db --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoSource.java @@ -0,0 +1,162 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; + +/** + * Java wrapper of native AndroidVideoTrackSource. + */ +public class VideoSource extends MediaSource { + /** Simple aspect ratio clas for use in constraining output format. */ + public static class AspectRatio { + public static final AspectRatio UNDEFINED = new AspectRatio(/* width= */ 0, /* height= */ 0); + + public final int width; + public final int height; + + public AspectRatio(int width, int height) { + this.width = width; + this.height = height; + } + } + + private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource; + private final Object videoProcessorLock = new Object(); + @Nullable private VideoProcessor videoProcessor; + private boolean isCapturerRunning; + + private final CapturerObserver capturerObserver = new CapturerObserver() { + @Override + public void onCapturerStarted(boolean success) { + nativeAndroidVideoTrackSource.setState(success); + synchronized (videoProcessorLock) { + isCapturerRunning = success; + if (videoProcessor != null) { + videoProcessor.onCapturerStarted(success); + } + } + } + + @Override + public void onCapturerStopped() { + nativeAndroidVideoTrackSource.setState(/* isLive= */ false); + synchronized (videoProcessorLock) { + isCapturerRunning = false; + if (videoProcessor != null) { + videoProcessor.onCapturerStopped(); + } + } + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + final VideoProcessor.FrameAdaptationParameters parameters = + nativeAndroidVideoTrackSource.adaptFrame(frame); + synchronized (videoProcessorLock) { + if (videoProcessor != null) { + videoProcessor.onFrameCaptured(frame, parameters); + return; + } + } + + VideoFrame adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters); + if (adaptedFrame != null) { + nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame); + adaptedFrame.release(); + } + } + }; + + public VideoSource(long nativeSource) { + super(nativeSource); + this.nativeAndroidVideoTrackSource = new NativeAndroidVideoTrackSource(nativeSource); + } + + /** + * Calling this function will cause frames to be scaled down to the requested resolution. Also, + * frames will be cropped to match the requested aspect ratio, and frames will be dropped to match + * the requested fps. The requested aspect ratio is orientation agnostic and will be adjusted to + * maintain the input orientation, so it doesn't matter if e.g. 1280x720 or 720x1280 is requested. + */ + public void adaptOutputFormat(int width, int height, int fps) { + final int maxSide = Math.max(width, height); + final int minSide = Math.min(width, height); + adaptOutputFormat(maxSide, minSide, minSide, maxSide, fps); + } + + /** + * Same as above, but allows setting two different target resolutions depending on incoming + * frame orientation. This gives more fine-grained control and can e.g. be used to force landscape + * video to be cropped to portrait video. + */ + public void adaptOutputFormat( + int landscapeWidth, int landscapeHeight, int portraitWidth, int portraitHeight, int fps) { + adaptOutputFormat(new AspectRatio(landscapeWidth, landscapeHeight), + /* maxLandscapePixelCount= */ landscapeWidth * landscapeHeight, + new AspectRatio(portraitWidth, portraitHeight), + /* maxPortraitPixelCount= */ portraitWidth * portraitHeight, fps); + } + + /** Same as above, with even more control as each constraint is optional. */ + public void adaptOutputFormat(AspectRatio targetLandscapeAspectRatio, + @Nullable Integer maxLandscapePixelCount, AspectRatio targetPortraitAspectRatio, + @Nullable Integer maxPortraitPixelCount, @Nullable Integer maxFps) { + nativeAndroidVideoTrackSource.adaptOutputFormat(targetLandscapeAspectRatio, + maxLandscapePixelCount, targetPortraitAspectRatio, maxPortraitPixelCount, maxFps); + } + + public void setIsScreencast(boolean isScreencast) { + nativeAndroidVideoTrackSource.setIsScreencast(isScreencast); + } + + /** + * Hook for injecting a custom video processor before frames are passed onto WebRTC. The frames + * will be cropped and scaled depending on CPU and network conditions before they are passed to + * the video processor. Frames will be delivered to the video processor on the same thread they + * are passed to this object. The video processor is allowed to deliver the processed frames + * back on any thread. + */ + public void setVideoProcessor(@Nullable VideoProcessor newVideoProcessor) { + synchronized (videoProcessorLock) { + if (videoProcessor != null) { + videoProcessor.setSink(/* sink= */ null); + if (isCapturerRunning) { + videoProcessor.onCapturerStopped(); + } + } + videoProcessor = newVideoProcessor; + if (newVideoProcessor != null) { + newVideoProcessor.setSink( + (frame) + -> runWithReference(() -> nativeAndroidVideoTrackSource.onFrameCaptured(frame))); + if (isCapturerRunning) { + newVideoProcessor.onCapturerStarted(/* success= */ true); + } + } + } + } + + public CapturerObserver getCapturerObserver() { + return capturerObserver; + } + + /** Returns a pointer to webrtc::VideoTrackSourceInterface. */ + long getNativeVideoTrackSource() { + return getNativeMediaSource(); + } + + @Override + public void dispose() { + setVideoProcessor(/* newVideoProcessor= */ null); + super.dispose(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoTrack.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoTrack.java new file mode 100644 index 0000000000..512e46c26e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/VideoTrack.java @@ -0,0 +1,76 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.IdentityHashMap; + +/** Java version of VideoTrackInterface. */ +public class VideoTrack extends MediaStreamTrack { + private final IdentityHashMap<VideoSink, Long> sinks = new IdentityHashMap<VideoSink, Long>(); + + public VideoTrack(long nativeTrack) { + super(nativeTrack); + } + + /** + * Adds a VideoSink to the track. + * + * A track can have any number of VideoSinks. VideoSinks will replace + * renderers. However, converting old style texture frames will involve costly + * conversion to I420 so it is not recommended to upgrade before all your + * sources produce VideoFrames. + */ + public void addSink(VideoSink sink) { + if (sink == null) { + throw new IllegalArgumentException("The VideoSink is not allowed to be null"); + } + // We allow calling addSink() with the same sink multiple times. This is similar to the C++ + // VideoTrack::AddOrUpdateSink(). + if (!sinks.containsKey(sink)) { + final long nativeSink = nativeWrapSink(sink); + sinks.put(sink, nativeSink); + nativeAddSink(getNativeMediaStreamTrack(), nativeSink); + } + } + + /** + * Removes a VideoSink from the track. + * + * If the VideoSink was not attached to the track, this is a no-op. + */ + public void removeSink(VideoSink sink) { + final Long nativeSink = sinks.remove(sink); + if (nativeSink != null) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + } + + @Override + public void dispose() { + for (long nativeSink : sinks.values()) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + sinks.clear(); + super.dispose(); + } + + /** Returns a pointer to webrtc::VideoTrackInterface. */ + public long getNativeVideoTrack() { + return getNativeMediaStreamTrack(); + } + + private static native void nativeAddSink(long track, long nativeSink); + private static native void nativeRemoveSink(long track, long nativeSink); + private static native long nativeWrapSink(VideoSink sink); + private static native void nativeFreeSink(long sink); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java new file mode 100644 index 0000000000..027120e48e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Wraps a native webrtc::VideoDecoder. + */ +public abstract class WrappedNativeVideoDecoder implements VideoDecoder { + @Override public abstract long createNativeVideoDecoder(); + + @Override + public final VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final VideoCodecStatus release() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final VideoCodecStatus decode(EncodedImage frame, DecodeInfo info) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final String getImplementationName() { + throw new UnsupportedOperationException("Not implemented."); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java new file mode 100644 index 0000000000..7d0908a6ac --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Wraps a native webrtc::VideoEncoder. + */ +public abstract class WrappedNativeVideoEncoder implements VideoEncoder { + @Override public abstract long createNativeVideoEncoder(); + @Override public abstract boolean isHardwareEncoder(); + + @Override + public final VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final VideoCodecStatus release() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate) { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final ScalingSettings getScalingSettings() { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public final String getImplementationName() { + throw new UnsupportedOperationException("Not implemented."); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java new file mode 100644 index 0000000000..c855d4be41 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvConverter.java @@ -0,0 +1,252 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Matrix; +import android.opengl.GLES20; +import android.opengl.GLException; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import org.webrtc.VideoFrame.I420Buffer; +import org.webrtc.VideoFrame.TextureBuffer; + +/** + * Class for converting OES textures to a YUV ByteBuffer. It can be constructed on any thread, but + * should only be operated from a single thread with an active EGL context. + */ +public final class YuvConverter { + private static final String TAG = "YuvConverter"; + + private static final String FRAGMENT_SHADER = + // Difference in texture coordinate corresponding to one + // sub-pixel in the x direction. + "uniform vec2 xUnit;\n" + // Color conversion coefficients, including constant term + + "uniform vec4 coeffs;\n" + + "\n" + + "void main() {\n" + // Since the alpha read from the texture is always 1, this could + // be written as a mat4 x vec4 multiply. However, that seems to + // give a worse framerate, possibly because the additional + // multiplies by 1.0 consume resources. + + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc - 1.5 * xUnit).rgb);\n" + + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc - 0.5 * xUnit).rgb);\n" + + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc + 0.5 * xUnit).rgb);\n" + + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" + + " sample(tc + 1.5 * xUnit).rgb);\n" + + "}\n"; + + private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks { + // Y'UV444 to RGB888, see https://en.wikipedia.org/wiki/YUV#Y%E2%80%B2UV444_to_RGB888_conversion + // We use the ITU-R BT.601 coefficients for Y, U and V. + // The values in Wikipedia are inaccurate, the accurate values derived from the spec are: + // Y = 0.299 * R + 0.587 * G + 0.114 * B + // U = -0.168736 * R - 0.331264 * G + 0.5 * B + 0.5 + // V = 0.5 * R - 0.418688 * G - 0.0813124 * B + 0.5 + // To map the Y-values to range [16-235] and U- and V-values to range [16-240], the matrix has + // been multiplied with matrix: + // {{219 / 255, 0, 0, 16 / 255}, + // {0, 224 / 255, 0, 16 / 255}, + // {0, 0, 224 / 255, 16 / 255}, + // {0, 0, 0, 1}} + private static final float[] yCoeffs = + new float[] {0.256788f, 0.504129f, 0.0979059f, 0.0627451f}; + private static final float[] uCoeffs = + new float[] {-0.148223f, -0.290993f, 0.439216f, 0.501961f}; + private static final float[] vCoeffs = + new float[] {0.439216f, -0.367788f, -0.0714274f, 0.501961f}; + + private int xUnitLoc; + private int coeffsLoc; + + private float[] coeffs; + private float stepSize; + + public void setPlaneY() { + coeffs = yCoeffs; + stepSize = 1.0f; + } + + public void setPlaneU() { + coeffs = uCoeffs; + stepSize = 2.0f; + } + + public void setPlaneV() { + coeffs = vCoeffs; + stepSize = 2.0f; + } + + @Override + public void onNewShader(GlShader shader) { + xUnitLoc = shader.getUniformLocation("xUnit"); + coeffsLoc = shader.getUniformLocation("coeffs"); + } + + @Override + public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, + int viewportWidth, int viewportHeight) { + GLES20.glUniform4fv(coeffsLoc, /* count= */ 1, coeffs, /* offset= */ 0); + // Matrix * (1;0;0;0) / (width / stepSize). Note that OpenGL uses column major order. + GLES20.glUniform2f( + xUnitLoc, stepSize * texMatrix[0] / frameWidth, stepSize * texMatrix[1] / frameWidth); + } + } + + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + private final GlTextureFrameBuffer i420TextureFrameBuffer = + new GlTextureFrameBuffer(GLES20.GL_RGBA); + private final ShaderCallbacks shaderCallbacks = new ShaderCallbacks(); + private final GlGenericDrawer drawer = new GlGenericDrawer(FRAGMENT_SHADER, shaderCallbacks); + private final VideoFrameDrawer videoFrameDrawer; + + /** + * This class should be constructed on a thread that has an active EGL context. + */ + public YuvConverter() { + this(new VideoFrameDrawer()); + } + + public YuvConverter(VideoFrameDrawer videoFrameDrawer) { + this.videoFrameDrawer = videoFrameDrawer; + threadChecker.detachThread(); + } + + /** Converts the texture buffer to I420. */ + @Nullable + public I420Buffer convert(TextureBuffer inputTextureBuffer) { + try { + return convertInternal(inputTextureBuffer); + } catch (GLException e) { + Logging.w(TAG, "Failed to convert TextureBuffer", e); + } + return null; + } + + private I420Buffer convertInternal(TextureBuffer inputTextureBuffer) { + TextureBuffer preparedBuffer = (TextureBuffer) videoFrameDrawer.prepareBufferForViewportSize( + inputTextureBuffer, inputTextureBuffer.getWidth(), inputTextureBuffer.getHeight()); + + // We draw into a buffer laid out like + // + // +---------+ + // | | + // | Y | + // | | + // | | + // +----+----+ + // | U | V | + // | | | + // +----+----+ + // + // In memory, we use the same stride for all of Y, U and V. The + // U data starts at offset `height` * `stride` from the Y data, + // and the V data starts at at offset |stride/2| from the U + // data, with rows of U and V data alternating. + // + // Now, it would have made sense to allocate a pixel buffer with + // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, + // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be + // unsupported by devices. So do the following hack: Allocate an + // RGBA buffer, of width `stride`/4. To render each of these + // large pixels, sample the texture at 4 different x coordinates + // and store the results in the four components. + // + // Since the V data needs to start on a boundary of such a + // larger pixel, it is not sufficient that `stride` is even, it + // has to be a multiple of 8 pixels. + final int frameWidth = preparedBuffer.getWidth(); + final int frameHeight = preparedBuffer.getHeight(); + final int stride = ((frameWidth + 7) / 8) * 8; + final int uvHeight = (frameHeight + 1) / 2; + // Total height of the combined memory layout. + final int totalHeight = frameHeight + uvHeight; + final ByteBuffer i420ByteBuffer = JniCommon.nativeAllocateByteBuffer(stride * totalHeight); + // Viewport width is divided by four since we are squeezing in four color bytes in each RGBA + // pixel. + final int viewportWidth = stride / 4; + + // Produce a frame buffer starting at top-left corner, not bottom-left. + final Matrix renderMatrix = new Matrix(); + renderMatrix.preTranslate(0.5f, 0.5f); + renderMatrix.preScale(1f, -1f); + renderMatrix.preTranslate(-0.5f, -0.5f); + + i420TextureFrameBuffer.setSize(viewportWidth, totalHeight); + + // Bind our framebuffer. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, i420TextureFrameBuffer.getFrameBufferId()); + GlUtil.checkNoGLES2Error("glBindFramebuffer"); + + // Draw Y. + shaderCallbacks.setPlaneY(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ 0, /* viewportY= */ 0, viewportWidth, + /* viewportHeight= */ frameHeight); + + // Draw U. + shaderCallbacks.setPlaneU(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ 0, /* viewportY= */ frameHeight, viewportWidth / 2, + /* viewportHeight= */ uvHeight); + + // Draw V. + shaderCallbacks.setPlaneV(); + VideoFrameDrawer.drawTexture(drawer, preparedBuffer, renderMatrix, frameWidth, frameHeight, + /* viewportX= */ viewportWidth / 2, /* viewportY= */ frameHeight, viewportWidth / 2, + /* viewportHeight= */ uvHeight); + + GLES20.glReadPixels(0, 0, i420TextureFrameBuffer.getWidth(), i420TextureFrameBuffer.getHeight(), + GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, i420ByteBuffer); + + GlUtil.checkNoGLES2Error("YuvConverter.convert"); + + // Restore normal framebuffer. + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + + // Prepare Y, U, and V ByteBuffer slices. + final int yPos = 0; + final int uPos = yPos + stride * frameHeight; + // Rows of U and V alternate in the buffer, so V data starts after the first row of U. + final int vPos = uPos + stride / 2; + + i420ByteBuffer.position(yPos); + i420ByteBuffer.limit(yPos + stride * frameHeight); + final ByteBuffer dataY = i420ByteBuffer.slice(); + + i420ByteBuffer.position(uPos); + // The last row does not have padding. + final int uvSize = stride * (uvHeight - 1) + stride / 2; + i420ByteBuffer.limit(uPos + uvSize); + final ByteBuffer dataU = i420ByteBuffer.slice(); + + i420ByteBuffer.position(vPos); + i420ByteBuffer.limit(vPos + uvSize); + final ByteBuffer dataV = i420ByteBuffer.slice(); + + preparedBuffer.release(); + + return JavaI420Buffer.wrap(frameWidth, frameHeight, dataY, stride, dataU, stride, dataV, stride, + () -> { JniCommon.nativeFreeByteBuffer(i420ByteBuffer); }); + } + + public void release() { + threadChecker.checkIsOnValidThread(); + drawer.release(); + i420TextureFrameBuffer.release(); + videoFrameDrawer.release(); + // Allow this class to be reused. + threadChecker.detachThread(); + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvHelper.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvHelper.java new file mode 100644 index 0000000000..8a46a57162 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/YuvHelper.java @@ -0,0 +1,236 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** Wraps libyuv methods to Java. All passed byte buffers must be direct byte buffers. */ +public class YuvHelper { + /** + * Copy I420 Buffer to a contiguously allocated buffer. + * <p> In Android, MediaCodec can request a buffer of a specific layout with the stride and + * slice-height (or plane height), and this function is used in this case. + * <p> For more information, see + * https://cs.android.com/android/platform/superproject/+/64fea7e5726daebc40f46890100837c01091100d:frameworks/base/media/java/android/media/MediaFormat.java;l=568 + * @param dstStrideY the stride of output buffers' Y plane. + * @param dstSliceHeightY the slice-height of output buffer's Y plane. + * @param dstStrideU the stride of output buffers' U (and V) plane. + * @param dstSliceHeightU the slice-height of output buffer's U (and V) plane + */ + public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int dstWidth, int dstHeight, int dstStrideY, + int dstSliceHeightY, int dstStrideU, int dstSliceHeightU) { + final int chromaWidth = (dstWidth + 1) / 2; + final int chromaHeight = (dstHeight + 1) / 2; + + final int dstStartY = 0; + final int dstEndY = dstStartY + dstStrideY * dstHeight; + final int dstStartU = dstStartY + dstStrideY * dstSliceHeightY; + final int dstEndU = dstStartU + dstStrideU * chromaHeight; + final int dstStartV = dstStartU + dstStrideU * dstSliceHeightU; + // The last line doesn't need any padding, so use chromaWidth to calculate the exact end + // position. + final int dstEndV = dstStartV + dstStrideU * (chromaHeight - 1) + chromaWidth; + if (dst.capacity() < dstEndV) { + throw new IllegalArgumentException("Expected destination buffer capacity to be at least " + + dstEndV + " was " + dst.capacity()); + } + + dst.limit(dstEndY); + dst.position(dstStartY); + final ByteBuffer dstY = dst.slice(); + dst.limit(dstEndU); + dst.position(dstStartU); + final ByteBuffer dstU = dst.slice(); + dst.limit(dstEndV); + dst.position(dstStartV); + final ByteBuffer dstV = dst.slice(); + + I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU, + dstStrideU, dstV, dstStrideU, dstWidth, dstHeight); + } + + /** Helper method for copying I420 to tightly packed destination buffer. */ + public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int dstWidth, int dstHeight) { + I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dst, dstWidth, dstHeight, + dstWidth, dstHeight, (dstWidth + 1) / 2, (dstHeight + 1) / 2); + } + + /** Helper method for copying I420 to buffer with the given stride and slice height. */ + public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int dstWidth, int dstHeight, int dstStride, + int dstSliceHeight) { + I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dst, dstWidth, dstHeight, + dstStride, dstSliceHeight, (dstStride + 1) / 2, (dstSliceHeight + 1) / 2); + } + + /** + * Copy I420 Buffer to a contiguously allocated buffer. + * @param dstStrideY the stride of output buffers' Y plane. + * @param dstSliceHeightY the slice-height of output buffer's Y plane. + */ + public static void I420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int dstWidth, int dstHeight, int dstStrideY, + int dstSliceHeightY) { + final int chromaHeight = (dstHeight + 1) / 2; + final int chromaWidth = (dstWidth + 1) / 2; + + final int dstStartY = 0; + final int dstEndY = dstStartY + dstStrideY * dstHeight; + final int dstStartUV = dstStartY + dstStrideY * dstSliceHeightY; + final int dstEndUV = dstStartUV + chromaWidth * chromaHeight * 2; + if (dst.capacity() < dstEndUV) { + throw new IllegalArgumentException("Expected destination buffer capacity to be at least " + + dstEndUV + " was " + dst.capacity()); + } + + dst.limit(dstEndY); + dst.position(dstStartY); + final ByteBuffer dstY = dst.slice(); + dst.limit(dstEndUV); + dst.position(dstStartUV); + final ByteBuffer dstUV = dst.slice(); + + I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstUV, + chromaWidth * 2, dstWidth, dstHeight); + } + + /** Helper method for copying I420 to tightly packed NV12 destination buffer. */ + public static void I420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int dstWidth, int dstHeight) { + I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dst, dstWidth, dstHeight, + dstWidth, dstHeight); + } + + /** Helper method for rotating I420 to tightly packed destination buffer. */ + public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int srcWidth, int srcHeight, + int rotationMode) { + checkNotNull(srcY, "srcY"); + checkNotNull(srcU, "srcU"); + checkNotNull(srcV, "srcV"); + checkNotNull(dst, "dst"); + final int dstWidth = rotationMode % 180 == 0 ? srcWidth : srcHeight; + final int dstHeight = rotationMode % 180 == 0 ? srcHeight : srcWidth; + + final int dstChromaHeight = (dstHeight + 1) / 2; + final int dstChromaWidth = (dstWidth + 1) / 2; + + final int minSize = dstWidth * dstHeight + dstChromaWidth * dstChromaHeight * 2; + if (dst.capacity() < minSize) { + throw new IllegalArgumentException("Expected destination buffer capacity to be at least " + + minSize + " was " + dst.capacity()); + } + + final int startY = 0; + final int startU = dstHeight * dstWidth; + final int startV = startU + dstChromaHeight * dstChromaWidth; + + dst.position(startY); + final ByteBuffer dstY = dst.slice(); + dst.position(startU); + final ByteBuffer dstU = dst.slice(); + dst.position(startV); + final ByteBuffer dstV = dst.slice(); + + nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstWidth, dstU, + dstChromaWidth, dstV, dstChromaWidth, srcWidth, srcHeight, rotationMode); + } + + /** Helper method for copying a single colour plane. */ + public static void copyPlane( + ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) { + nativeCopyPlane( + checkNotNull(src, "src"), srcStride, checkNotNull(dst, "dst"), dstStride, width, height); + } + + /** Converts ABGR little endian (rgba in memory) to I420. */ + public static void ABGRToI420(ByteBuffer src, int srcStride, ByteBuffer dstY, int dstStrideY, + ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height) { + nativeABGRToI420(checkNotNull(src, "src"), srcStride, checkNotNull(dstY, "dstY"), dstStrideY, + checkNotNull(dstU, "dstU"), dstStrideU, checkNotNull(dstV, "dstV"), dstStrideV, width, + height); + } + + /** + * Copies I420 to the I420 dst buffer. + * <p> Unlike `libyuv::I420Copy`, this function checks if the height <= 0, so flipping is not + * supported. + */ + public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU, + int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height) { + checkNotNull(srcY, "srcY"); + checkNotNull(srcU, "srcU"); + checkNotNull(srcV, "srcV"); + checkNotNull(dstY, "dstY"); + checkNotNull(dstU, "dstU"); + checkNotNull(dstV, "dstV"); + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("I420Copy: width and height should not be negative"); + } + nativeI420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU, + dstStrideU, dstV, dstStrideV, width, height); + } + + public static void I420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstUV, + int dstStrideUV, int width, int height) { + checkNotNull(srcY, "srcY"); + checkNotNull(srcU, "srcU"); + checkNotNull(srcV, "srcV"); + checkNotNull(dstY, "dstY"); + checkNotNull(dstUV, "dstUV"); + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("I420ToNV12: width and height should not be negative"); + } + nativeI420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstUV, + dstStrideUV, width, height); + } + + public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU, + ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU, + int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight, + int rotationMode) { + checkNotNull(srcY, "srcY"); + checkNotNull(srcU, "srcU"); + checkNotNull(srcV, "srcV"); + checkNotNull(dstY, "dstY"); + checkNotNull(dstU, "dstU"); + checkNotNull(dstV, "dstV"); + nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU, + dstStrideU, dstV, dstStrideV, srcWidth, srcHeight, rotationMode); + } + + private static <T> T checkNotNull(T obj, String description) { + if (obj == null) { + throw new NullPointerException(description + " should not be null"); + } + return obj; + } + + private static native void nativeCopyPlane( + ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height); + private static native void nativeI420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, + int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, + ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height); + private static native void nativeI420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, + int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, + ByteBuffer dstUV, int dstStrideUV, int width, int height); + private static native void nativeI420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, + int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, + ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight, + int rotationMode); + private static native void nativeABGRToI420(ByteBuffer src, int srcStride, ByteBuffer dstY, + int dstStrideY, ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, + int height); +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/AudioDeviceModule.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/AudioDeviceModule.java new file mode 100644 index 0000000000..5a0bf5c74d --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/AudioDeviceModule.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +/** + * This interface is a thin wrapper on top of a native C++ webrtc::AudioDeviceModule (ADM). The + * reason for basing it on a native ADM instead of a pure Java interface is that we have two native + * Android implementations (OpenSLES and AAudio) that does not make sense to wrap through JNI. + * + * <p>Note: This class is still under development and may change without notice. + */ +public interface AudioDeviceModule { + /** + * Returns a C++ pointer to a webrtc::AudioDeviceModule. Caller does _not_ take ownership and + * lifetime is handled through the release() call. + */ + long getNativeAudioDeviceModulePointer(); + + /** + * Release resources for this AudioDeviceModule, including native resources. The object should not + * be used after this call. + */ + void release(); + + /** Control muting/unmuting the speaker. */ + void setSpeakerMute(boolean mute); + + /** Control muting/unmuting the microphone. */ + void setMicrophoneMute(boolean mute); + + /** + * Enable or disable built in noise suppressor. Returns true if the enabling was successful, + * otherwise false is returned. + */ + default boolean setNoiseSuppressorEnabled(boolean enabled) { + return false; + } + + /** + * Sets the preferred field dimension for the built-in microphone. Returns + * true if setting was successful, otherwise false is returned. + * This functionality can be implemented with + * {@code android.media.MicrophoneDirection.setPreferredMicrophoneFieldDimension}. + */ + default boolean setPreferredMicrophoneFieldDimension(float dimension) { + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java new file mode 100644 index 0000000000..b118843ea0 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -0,0 +1,442 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.content.Context; +import android.media.AudioAttributes; +import android.media.AudioDeviceInfo; +import android.media.AudioManager; +import android.os.Build; +import androidx.annotation.RequiresApi; +import java.util.concurrent.ScheduledExecutorService; +import org.webrtc.JniCommon; +import org.webrtc.Logging; + +/** + * AudioDeviceModule implemented using android.media.AudioRecord as input and + * android.media.AudioTrack as output. + */ +public class JavaAudioDeviceModule implements AudioDeviceModule { + private static final String TAG = "JavaAudioDeviceModule"; + + public static Builder builder(Context context) { + return new Builder(context); + } + + public static class Builder { + private final Context context; + private ScheduledExecutorService scheduler; + private final AudioManager audioManager; + private int inputSampleRate; + private int outputSampleRate; + private int audioSource = WebRtcAudioRecord.DEFAULT_AUDIO_SOURCE; + private int audioFormat = WebRtcAudioRecord.DEFAULT_AUDIO_FORMAT; + private AudioTrackErrorCallback audioTrackErrorCallback; + private AudioRecordErrorCallback audioRecordErrorCallback; + private SamplesReadyCallback samplesReadyCallback; + private AudioTrackStateCallback audioTrackStateCallback; + private AudioRecordStateCallback audioRecordStateCallback; + private boolean useHardwareAcousticEchoCanceler = isBuiltInAcousticEchoCancelerSupported(); + private boolean useHardwareNoiseSuppressor = isBuiltInNoiseSuppressorSupported(); + private boolean useStereoInput; + private boolean useStereoOutput; + private AudioAttributes audioAttributes; + private boolean useLowLatency; + private boolean enableVolumeLogger; + + private Builder(Context context) { + this.context = context; + this.audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + this.inputSampleRate = WebRtcAudioManager.getSampleRate(audioManager); + this.outputSampleRate = WebRtcAudioManager.getSampleRate(audioManager); + this.useLowLatency = false; + this.enableVolumeLogger = true; + } + + public Builder setScheduler(ScheduledExecutorService scheduler) { + this.scheduler = scheduler; + return this; + } + + /** + * Call this method if the default handling of querying the native sample rate shall be + * overridden. Can be useful on some devices where the available Android APIs are known to + * return invalid results. + */ + public Builder setSampleRate(int sampleRate) { + Logging.d(TAG, "Input/Output sample rate overridden to: " + sampleRate); + this.inputSampleRate = sampleRate; + this.outputSampleRate = sampleRate; + return this; + } + + /** + * Call this method to specifically override input sample rate. + */ + public Builder setInputSampleRate(int inputSampleRate) { + Logging.d(TAG, "Input sample rate overridden to: " + inputSampleRate); + this.inputSampleRate = inputSampleRate; + return this; + } + + /** + * Call this method to specifically override output sample rate. + */ + public Builder setOutputSampleRate(int outputSampleRate) { + Logging.d(TAG, "Output sample rate overridden to: " + outputSampleRate); + this.outputSampleRate = outputSampleRate; + return this; + } + + /** + * Call this to change the audio source. The argument should be one of the values from + * android.media.MediaRecorder.AudioSource. The default is AudioSource.VOICE_COMMUNICATION. + */ + public Builder setAudioSource(int audioSource) { + this.audioSource = audioSource; + return this; + } + + /** + * Call this to change the audio format. The argument should be one of the values from + * android.media.AudioFormat ENCODING_PCM_8BIT, ENCODING_PCM_16BIT or ENCODING_PCM_FLOAT. + * Default audio data format is PCM 16 bit per sample. + * Guaranteed to be supported by all devices. + */ + public Builder setAudioFormat(int audioFormat) { + this.audioFormat = audioFormat; + return this; + } + + /** + * Set a callback to retrieve errors from the AudioTrack. + */ + public Builder setAudioTrackErrorCallback(AudioTrackErrorCallback audioTrackErrorCallback) { + this.audioTrackErrorCallback = audioTrackErrorCallback; + return this; + } + + /** + * Set a callback to retrieve errors from the AudioRecord. + */ + public Builder setAudioRecordErrorCallback(AudioRecordErrorCallback audioRecordErrorCallback) { + this.audioRecordErrorCallback = audioRecordErrorCallback; + return this; + } + + /** + * Set a callback to listen to the raw audio input from the AudioRecord. + */ + public Builder setSamplesReadyCallback(SamplesReadyCallback samplesReadyCallback) { + this.samplesReadyCallback = samplesReadyCallback; + return this; + } + + /** + * Set a callback to retrieve information from the AudioTrack on when audio starts and stop. + */ + public Builder setAudioTrackStateCallback(AudioTrackStateCallback audioTrackStateCallback) { + this.audioTrackStateCallback = audioTrackStateCallback; + return this; + } + + /** + * Set a callback to retrieve information from the AudioRecord on when audio starts and stops. + */ + public Builder setAudioRecordStateCallback(AudioRecordStateCallback audioRecordStateCallback) { + this.audioRecordStateCallback = audioRecordStateCallback; + return this; + } + + /** + * Control if the built-in HW noise suppressor should be used or not. The default is on if it is + * supported. It is possible to query support by calling isBuiltInNoiseSuppressorSupported(). + */ + public Builder setUseHardwareNoiseSuppressor(boolean useHardwareNoiseSuppressor) { + if (useHardwareNoiseSuppressor && !isBuiltInNoiseSuppressorSupported()) { + Logging.e(TAG, "HW NS not supported"); + useHardwareNoiseSuppressor = false; + } + this.useHardwareNoiseSuppressor = useHardwareNoiseSuppressor; + return this; + } + + /** + * Control if the built-in HW acoustic echo canceler should be used or not. The default is on if + * it is supported. It is possible to query support by calling + * isBuiltInAcousticEchoCancelerSupported(). + */ + public Builder setUseHardwareAcousticEchoCanceler(boolean useHardwareAcousticEchoCanceler) { + if (useHardwareAcousticEchoCanceler && !isBuiltInAcousticEchoCancelerSupported()) { + Logging.e(TAG, "HW AEC not supported"); + useHardwareAcousticEchoCanceler = false; + } + this.useHardwareAcousticEchoCanceler = useHardwareAcousticEchoCanceler; + return this; + } + + /** + * Control if stereo input should be used or not. The default is mono. + */ + public Builder setUseStereoInput(boolean useStereoInput) { + this.useStereoInput = useStereoInput; + return this; + } + + /** + * Control if stereo output should be used or not. The default is mono. + */ + public Builder setUseStereoOutput(boolean useStereoOutput) { + this.useStereoOutput = useStereoOutput; + return this; + } + + /** + * Control if the low-latency mode should be used. The default is disabled. + */ + public Builder setUseLowLatency(boolean useLowLatency) { + this.useLowLatency = useLowLatency; + return this; + } + + /** + * Set custom {@link AudioAttributes} to use. + */ + public Builder setAudioAttributes(AudioAttributes audioAttributes) { + this.audioAttributes = audioAttributes; + return this; + } + + /** Disables the volume logger on the audio output track. */ + public Builder setEnableVolumeLogger(boolean enableVolumeLogger) { + this.enableVolumeLogger = enableVolumeLogger; + return this; + } + + /** + * Construct an AudioDeviceModule based on the supplied arguments. The caller takes ownership + * and is responsible for calling release(). + */ + public JavaAudioDeviceModule createAudioDeviceModule() { + Logging.d(TAG, "createAudioDeviceModule"); + if (useHardwareNoiseSuppressor) { + Logging.d(TAG, "HW NS will be used."); + } else { + if (isBuiltInNoiseSuppressorSupported()) { + Logging.d(TAG, "Overriding default behavior; now using WebRTC NS!"); + } + Logging.d(TAG, "HW NS will not be used."); + } + if (useHardwareAcousticEchoCanceler) { + Logging.d(TAG, "HW AEC will be used."); + } else { + if (isBuiltInAcousticEchoCancelerSupported()) { + Logging.d(TAG, "Overriding default behavior; now using WebRTC AEC!"); + } + Logging.d(TAG, "HW AEC will not be used."); + } + // Low-latency mode was introduced in API version 26, see + // https://developer.android.com/reference/android/media/AudioTrack#PERFORMANCE_MODE_LOW_LATENCY + final int MIN_LOW_LATENCY_SDK_VERSION = 26; + if (useLowLatency && Build.VERSION.SDK_INT >= MIN_LOW_LATENCY_SDK_VERSION) { + Logging.d(TAG, "Low latency mode will be used."); + } + ScheduledExecutorService executor = this.scheduler; + if (executor == null) { + executor = WebRtcAudioRecord.newDefaultScheduler(); + } + final WebRtcAudioRecord audioInput = new WebRtcAudioRecord(context, executor, audioManager, + audioSource, audioFormat, audioRecordErrorCallback, audioRecordStateCallback, + samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); + final WebRtcAudioTrack audioOutput = + new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, + audioTrackStateCallback, useLowLatency, enableVolumeLogger); + return new JavaAudioDeviceModule(context, audioManager, audioInput, audioOutput, + inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput); + } + } + + /* AudioRecord */ + // Audio recording error handler functions. + public enum AudioRecordStartErrorCode { + AUDIO_RECORD_START_EXCEPTION, + AUDIO_RECORD_START_STATE_MISMATCH, + } + + public static interface AudioRecordErrorCallback { + void onWebRtcAudioRecordInitError(String errorMessage); + void onWebRtcAudioRecordStartError(AudioRecordStartErrorCode errorCode, String errorMessage); + void onWebRtcAudioRecordError(String errorMessage); + } + + /** Called when audio recording starts and stops. */ + public static interface AudioRecordStateCallback { + void onWebRtcAudioRecordStart(); + void onWebRtcAudioRecordStop(); + } + + /** + * Contains audio sample information. + */ + public static class AudioSamples { + /** See {@link AudioRecord#getAudioFormat()} */ + private final int audioFormat; + /** See {@link AudioRecord#getChannelCount()} */ + private final int channelCount; + /** See {@link AudioRecord#getSampleRate()} */ + private final int sampleRate; + + private final byte[] data; + + public AudioSamples(int audioFormat, int channelCount, int sampleRate, byte[] data) { + this.audioFormat = audioFormat; + this.channelCount = channelCount; + this.sampleRate = sampleRate; + this.data = data; + } + + public int getAudioFormat() { + return audioFormat; + } + + public int getChannelCount() { + return channelCount; + } + + public int getSampleRate() { + return sampleRate; + } + + public byte[] getData() { + return data; + } + } + + /** Called when new audio samples are ready. This should only be set for debug purposes */ + public static interface SamplesReadyCallback { + void onWebRtcAudioRecordSamplesReady(AudioSamples samples); + } + + /* AudioTrack */ + // Audio playout/track error handler functions. + public enum AudioTrackStartErrorCode { + AUDIO_TRACK_START_EXCEPTION, + AUDIO_TRACK_START_STATE_MISMATCH, + } + + public static interface AudioTrackErrorCallback { + void onWebRtcAudioTrackInitError(String errorMessage); + void onWebRtcAudioTrackStartError(AudioTrackStartErrorCode errorCode, String errorMessage); + void onWebRtcAudioTrackError(String errorMessage); + } + + /** Called when audio playout starts and stops. */ + public static interface AudioTrackStateCallback { + void onWebRtcAudioTrackStart(); + void onWebRtcAudioTrackStop(); + } + + /** + * Returns true if the device supports built-in HW AEC, and the UUID is approved (some UUIDs can + * be excluded). + */ + public static boolean isBuiltInAcousticEchoCancelerSupported() { + return WebRtcAudioEffects.isAcousticEchoCancelerSupported(); + } + + /** + * Returns true if the device supports built-in HW NS, and the UUID is approved (some UUIDs can be + * excluded). + */ + public static boolean isBuiltInNoiseSuppressorSupported() { + return WebRtcAudioEffects.isNoiseSuppressorSupported(); + } + + private final Context context; + private final AudioManager audioManager; + private final WebRtcAudioRecord audioInput; + private final WebRtcAudioTrack audioOutput; + private final int inputSampleRate; + private final int outputSampleRate; + private final boolean useStereoInput; + private final boolean useStereoOutput; + + private final Object nativeLock = new Object(); + private long nativeAudioDeviceModule; + + private JavaAudioDeviceModule(Context context, AudioManager audioManager, + WebRtcAudioRecord audioInput, WebRtcAudioTrack audioOutput, int inputSampleRate, + int outputSampleRate, boolean useStereoInput, boolean useStereoOutput) { + this.context = context; + this.audioManager = audioManager; + this.audioInput = audioInput; + this.audioOutput = audioOutput; + this.inputSampleRate = inputSampleRate; + this.outputSampleRate = outputSampleRate; + this.useStereoInput = useStereoInput; + this.useStereoOutput = useStereoOutput; + } + + @Override + public long getNativeAudioDeviceModulePointer() { + synchronized (nativeLock) { + if (nativeAudioDeviceModule == 0) { + nativeAudioDeviceModule = nativeCreateAudioDeviceModule(context, audioManager, audioInput, + audioOutput, inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput); + } + return nativeAudioDeviceModule; + } + } + + @Override + public void release() { + synchronized (nativeLock) { + if (nativeAudioDeviceModule != 0) { + JniCommon.nativeReleaseRef(nativeAudioDeviceModule); + nativeAudioDeviceModule = 0; + } + } + } + + @Override + public void setSpeakerMute(boolean mute) { + Logging.d(TAG, "setSpeakerMute: " + mute); + audioOutput.setSpeakerMute(mute); + } + + @Override + public void setMicrophoneMute(boolean mute) { + Logging.d(TAG, "setMicrophoneMute: " + mute); + audioInput.setMicrophoneMute(mute); + } + + @Override + public boolean setNoiseSuppressorEnabled(boolean enabled) { + Logging.d(TAG, "setNoiseSuppressorEnabled: " + enabled); + return audioInput.setNoiseSuppressorEnabled(enabled); + } + + /** + * Start to prefer a specific {@link AudioDeviceInfo} device for recording. Typically this should + * only be used if a client gives an explicit option for choosing a physical device to record + * from. Otherwise the best-matching device for other parameters will be used. Calling after + * recording is started may cause a temporary interruption if the audio routing changes. + */ + @RequiresApi(Build.VERSION_CODES.M) + public void setPreferredInputDevice(AudioDeviceInfo preferredInputDevice) { + Logging.d(TAG, "setPreferredInputDevice: " + preferredInputDevice); + audioInput.setPreferredDevice(preferredInputDevice); + } + + private static native long nativeCreateAudioDeviceModule(Context context, + AudioManager audioManager, WebRtcAudioRecord audioInput, WebRtcAudioTrack audioOutput, + int inputSampleRate, int outputSampleRate, boolean useStereoInput, boolean useStereoOutput); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java new file mode 100644 index 0000000000..47cb5689ce --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java @@ -0,0 +1,673 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaFormat; +import android.os.SystemClock; +import android.view.Surface; +import androidx.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import org.webrtc.ThreadUtils.ThreadChecker; + +/** + * Android hardware video decoder. + */ +class AndroidVideoDecoder implements VideoDecoder, VideoSink { + private static final String TAG = "AndroidVideoDecoder"; + + // MediaCodec.release() occasionally hangs. Release stops waiting and reports failure after + // this timeout. + private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; + + // WebRTC queues input frames quickly in the beginning on the call. Wait for input buffers with a + // long timeout (500 ms) to prevent this from causing the codec to return an error. + private static final int DEQUEUE_INPUT_TIMEOUT_US = 500000; + + // Dequeuing an output buffer will block until a buffer is available (up to 100 milliseconds). + // If this timeout is exceeded, the output thread will unblock and check if the decoder is still + // running. If it is, it will block on dequeue again. Otherwise, it will stop and release the + // MediaCodec. + private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; + + private final MediaCodecWrapperFactory mediaCodecWrapperFactory; + private final String codecName; + private final VideoCodecMimeType codecType; + + private static class FrameInfo { + final long decodeStartTimeMs; + final int rotation; + + FrameInfo(long decodeStartTimeMs, int rotation) { + this.decodeStartTimeMs = decodeStartTimeMs; + this.rotation = rotation; + } + } + + private final BlockingDeque<FrameInfo> frameInfos; + private int colorFormat; + + // Output thread runs a loop which polls MediaCodec for decoded output buffers. It reformats + // those buffers into VideoFrames and delivers them to the callback. Variable is set on decoder + // thread and is immutable while the codec is running. + @Nullable private Thread outputThread; + + // Checker that ensures work is run on the output thread. + private ThreadChecker outputThreadChecker; + + // Checker that ensures work is run on the decoder thread. The decoder thread is owned by the + // caller and must be used to call initDecode, decode, and release. + private ThreadChecker decoderThreadChecker; + + private volatile boolean running; + @Nullable private volatile Exception shutdownException; + + // Dimensions (width, height, stride, and sliceHeight) may be accessed by either the decode thread + // or the output thread. Accesses should be protected with this lock. + private final Object dimensionLock = new Object(); + private int width; + private int height; + private int stride; + private int sliceHeight; + + // Whether the decoder has finished the first frame. The codec may not change output dimensions + // after delivering the first frame. Only accessed on the output thread while the decoder is + // running. + private boolean hasDecodedFirstFrame; + // Whether the decoder has seen a key frame. The first frame must be a key frame. Only accessed + // on the decoder thread. + private boolean keyFrameRequired; + + private final @Nullable EglBase.Context sharedContext; + // Valid and immutable while the decoder is running. + @Nullable private SurfaceTextureHelper surfaceTextureHelper; + @Nullable private Surface surface; + + private static class DecodedTextureMetadata { + final long presentationTimestampUs; + final Integer decodeTimeMs; + + DecodedTextureMetadata(long presentationTimestampUs, Integer decodeTimeMs) { + this.presentationTimestampUs = presentationTimestampUs; + this.decodeTimeMs = decodeTimeMs; + } + } + + // Metadata for the last frame rendered to the texture. + private final Object renderedTextureMetadataLock = new Object(); + @Nullable private DecodedTextureMetadata renderedTextureMetadata; + + // Decoding proceeds asynchronously. This callback returns decoded frames to the caller. Valid + // and immutable while the decoder is running. + @Nullable private Callback callback; + + // Valid and immutable while the decoder is running. + @Nullable private MediaCodecWrapper codec; + + AndroidVideoDecoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, + VideoCodecMimeType codecType, int colorFormat, @Nullable EglBase.Context sharedContext) { + if (!isSupportedColorFormat(colorFormat)) { + throw new IllegalArgumentException("Unsupported color format: " + colorFormat); + } + Logging.d(TAG, + "ctor name: " + codecName + " type: " + codecType + " color format: " + colorFormat + + " context: " + sharedContext); + this.mediaCodecWrapperFactory = mediaCodecWrapperFactory; + this.codecName = codecName; + this.codecType = codecType; + this.colorFormat = colorFormat; + this.sharedContext = sharedContext; + this.frameInfos = new LinkedBlockingDeque<>(); + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.decoderThreadChecker = new ThreadChecker(); + + this.callback = callback; + if (sharedContext != null) { + surfaceTextureHelper = createSurfaceTextureHelper(); + surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); + surfaceTextureHelper.startListening(this); + } + return initDecodeInternal(settings.width, settings.height); + } + + // Internal variant is used when restarting the codec due to reconfiguration. + private VideoCodecStatus initDecodeInternal(int width, int height) { + decoderThreadChecker.checkIsOnValidThread(); + Logging.d(TAG, + "initDecodeInternal name: " + codecName + " type: " + codecType + " width: " + width + + " height: " + height + " color format: " + colorFormat); + if (outputThread != null) { + Logging.e(TAG, "initDecodeInternal called while the codec is already running"); + return VideoCodecStatus.FALLBACK_SOFTWARE; + } + + // Note: it is not necessary to initialize dimensions under the lock, since the output thread + // is not running. + this.width = width; + this.height = height; + + stride = width; + sliceHeight = height; + hasDecodedFirstFrame = false; + keyFrameRequired = true; + + try { + codec = mediaCodecWrapperFactory.createByCodecName(codecName); + } catch (IOException | IllegalArgumentException | IllegalStateException e) { + Logging.e(TAG, "Cannot create media decoder " + codecName); + return VideoCodecStatus.FALLBACK_SOFTWARE; + } + try { + MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height); + if (sharedContext == null) { + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); + } + codec.configure(format, surface, null, 0); + codec.start(); + } catch (IllegalStateException | IllegalArgumentException e) { + Logging.e(TAG, "initDecode failed", e); + release(); + return VideoCodecStatus.FALLBACK_SOFTWARE; + } + running = true; + outputThread = createOutputThread(); + outputThread.start(); + + Logging.d(TAG, "initDecodeInternal done"); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage frame, DecodeInfo info) { + decoderThreadChecker.checkIsOnValidThread(); + if (codec == null || callback == null) { + Logging.d(TAG, "decode uninitalized, codec: " + (codec != null) + ", callback: " + callback); + return VideoCodecStatus.UNINITIALIZED; + } + + if (frame.buffer == null) { + Logging.e(TAG, "decode() - no input data"); + return VideoCodecStatus.ERR_PARAMETER; + } + + int size = frame.buffer.remaining(); + if (size == 0) { + Logging.e(TAG, "decode() - input buffer empty"); + return VideoCodecStatus.ERR_PARAMETER; + } + + // Load dimensions from shared memory under the dimension lock. + final int width; + final int height; + synchronized (dimensionLock) { + width = this.width; + height = this.height; + } + + // Check if the resolution changed and reset the codec if necessary. + if (frame.encodedWidth * frame.encodedHeight > 0 + && (frame.encodedWidth != width || frame.encodedHeight != height)) { + VideoCodecStatus status = reinitDecode(frame.encodedWidth, frame.encodedHeight); + if (status != VideoCodecStatus.OK) { + return status; + } + } + + if (keyFrameRequired) { + // Need to process a key frame first. + if (frame.frameType != EncodedImage.FrameType.VideoFrameKey) { + Logging.e(TAG, "decode() - key frame required first"); + return VideoCodecStatus.NO_OUTPUT; + } + } + + int index; + try { + index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US); + } catch (IllegalStateException e) { + Logging.e(TAG, "dequeueInputBuffer failed", e); + return VideoCodecStatus.ERROR; + } + if (index < 0) { + // Decoder is falling behind. No input buffers available. + // The decoder can't simply drop frames; it might lose a key frame. + Logging.e(TAG, "decode() - no HW buffers available; decoder falling behind"); + return VideoCodecStatus.ERROR; + } + + ByteBuffer buffer; + try { + buffer = codec.getInputBuffer(index); + } catch (IllegalStateException e) { + Logging.e(TAG, "getInputBuffer with index=" + index + " failed", e); + return VideoCodecStatus.ERROR; + } + + if (buffer.capacity() < size) { + Logging.e(TAG, "decode() - HW buffer too small"); + return VideoCodecStatus.ERROR; + } + buffer.put(frame.buffer); + + frameInfos.offer(new FrameInfo(SystemClock.elapsedRealtime(), frame.rotation)); + try { + codec.queueInputBuffer(index, 0 /* offset */, size, + TimeUnit.NANOSECONDS.toMicros(frame.captureTimeNs), 0 /* flags */); + } catch (IllegalStateException e) { + Logging.e(TAG, "queueInputBuffer failed", e); + frameInfos.pollLast(); + return VideoCodecStatus.ERROR; + } + if (keyFrameRequired) { + keyFrameRequired = false; + } + return VideoCodecStatus.OK; + } + + @Override + public String getImplementationName() { + return codecName; + } + + @Override + public VideoCodecStatus release() { + // TODO(sakal): This is not called on the correct thread but is still called synchronously. + // Re-enable the check once this is called on the correct thread. + // decoderThreadChecker.checkIsOnValidThread(); + Logging.d(TAG, "release"); + VideoCodecStatus status = releaseInternal(); + if (surface != null) { + releaseSurface(); + surface = null; + surfaceTextureHelper.stopListening(); + surfaceTextureHelper.dispose(); + surfaceTextureHelper = null; + } + synchronized (renderedTextureMetadataLock) { + renderedTextureMetadata = null; + } + callback = null; + frameInfos.clear(); + return status; + } + + // Internal variant is used when restarting the codec due to reconfiguration. + private VideoCodecStatus releaseInternal() { + if (!running) { + Logging.d(TAG, "release: Decoder is not running."); + return VideoCodecStatus.OK; + } + try { + // The outputThread actually stops and releases the codec once running is false. + running = false; + if (!ThreadUtils.joinUninterruptibly(outputThread, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { + // Log an exception to capture the stack trace and turn it into a TIMEOUT error. + Logging.e(TAG, "Media decoder release timeout", new RuntimeException()); + return VideoCodecStatus.TIMEOUT; + } + if (shutdownException != null) { + // Log the exception and turn it into an error. Wrap the exception in a new exception to + // capture both the output thread's stack trace and this thread's stack trace. + Logging.e(TAG, "Media decoder release error", new RuntimeException(shutdownException)); + shutdownException = null; + return VideoCodecStatus.ERROR; + } + } finally { + codec = null; + outputThread = null; + } + return VideoCodecStatus.OK; + } + + private VideoCodecStatus reinitDecode(int newWidth, int newHeight) { + decoderThreadChecker.checkIsOnValidThread(); + VideoCodecStatus status = releaseInternal(); + if (status != VideoCodecStatus.OK) { + return status; + } + return initDecodeInternal(newWidth, newHeight); + } + + private Thread createOutputThread() { + return new Thread("AndroidVideoDecoder.outputThread") { + @Override + public void run() { + outputThreadChecker = new ThreadChecker(); + while (running) { + deliverDecodedFrame(); + } + releaseCodecOnOutputThread(); + } + }; + } + + // Visible for testing. + protected void deliverDecodedFrame() { + outputThreadChecker.checkIsOnValidThread(); + try { + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + // Block until an output buffer is available (up to 100 milliseconds). If the timeout is + // exceeded, deliverDecodedFrame() will be called again on the next iteration of the output + // thread's loop. Blocking here prevents the output thread from busy-waiting while the codec + // is idle. + int index = codec.dequeueOutputBuffer(info, DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US); + if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + reformat(codec.getOutputFormat()); + return; + } + + if (index < 0) { + Logging.v(TAG, "dequeueOutputBuffer returned " + index); + return; + } + + FrameInfo frameInfo = frameInfos.poll(); + Integer decodeTimeMs = null; + int rotation = 0; + if (frameInfo != null) { + decodeTimeMs = (int) (SystemClock.elapsedRealtime() - frameInfo.decodeStartTimeMs); + rotation = frameInfo.rotation; + } + + hasDecodedFirstFrame = true; + + if (surfaceTextureHelper != null) { + deliverTextureFrame(index, info, rotation, decodeTimeMs); + } else { + deliverByteFrame(index, info, rotation, decodeTimeMs); + } + + } catch (IllegalStateException e) { + Logging.e(TAG, "deliverDecodedFrame failed", e); + } + } + + private void deliverTextureFrame(final int index, final MediaCodec.BufferInfo info, + final int rotation, final Integer decodeTimeMs) { + // Load dimensions from shared memory under the dimension lock. + final int width; + final int height; + synchronized (dimensionLock) { + width = this.width; + height = this.height; + } + + synchronized (renderedTextureMetadataLock) { + if (renderedTextureMetadata != null) { + codec.releaseOutputBuffer(index, false); + return; // We are still waiting for texture for the previous frame, drop this one. + } + surfaceTextureHelper.setTextureSize(width, height); + surfaceTextureHelper.setFrameRotation(rotation); + renderedTextureMetadata = new DecodedTextureMetadata(info.presentationTimeUs, decodeTimeMs); + codec.releaseOutputBuffer(index, /* render= */ true); + } + } + + @Override + public void onFrame(VideoFrame frame) { + final VideoFrame newFrame; + final Integer decodeTimeMs; + final long timestampNs; + synchronized (renderedTextureMetadataLock) { + if (renderedTextureMetadata == null) { + throw new IllegalStateException( + "Rendered texture metadata was null in onTextureFrameAvailable."); + } + timestampNs = renderedTextureMetadata.presentationTimestampUs * 1000; + decodeTimeMs = renderedTextureMetadata.decodeTimeMs; + renderedTextureMetadata = null; + } + // Change timestamp of frame. + final VideoFrame frameWithModifiedTimeStamp = + new VideoFrame(frame.getBuffer(), frame.getRotation(), timestampNs); + callback.onDecodedFrame(frameWithModifiedTimeStamp, decodeTimeMs, null /* qp */); + } + + private void deliverByteFrame( + int index, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) { + // Load dimensions from shared memory under the dimension lock. + int width; + int height; + int stride; + int sliceHeight; + synchronized (dimensionLock) { + width = this.width; + height = this.height; + stride = this.stride; + sliceHeight = this.sliceHeight; + } + + // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2) + // bytes for each of the U and V channels. + if (info.size < width * height * 3 / 2) { + Logging.e(TAG, "Insufficient output buffer size: " + info.size); + return; + } + + if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) { + // Some codecs (Exynos) report an incorrect stride. Correct it here. + // Expected size == stride * height * 3 / 2. A bit of algebra gives the correct stride as + // 2 * size / (3 * height). + stride = info.size * 2 / (height * 3); + } + + ByteBuffer buffer = codec.getOutputBuffer(index); + buffer.position(info.offset); + buffer.limit(info.offset + info.size); + buffer = buffer.slice(); + + final VideoFrame.Buffer frameBuffer; + if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) { + frameBuffer = copyI420Buffer(buffer, stride, sliceHeight, width, height); + } else { + // All other supported color formats are NV12. + frameBuffer = copyNV12ToI420Buffer(buffer, stride, sliceHeight, width, height); + } + codec.releaseOutputBuffer(index, /* render= */ false); + + long presentationTimeNs = info.presentationTimeUs * 1000; + VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs); + + // Note that qp is parsed on the C++ side. + callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */); + frame.release(); + } + + private VideoFrame.Buffer copyNV12ToI420Buffer( + ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { + // toI420 copies the buffer. + return new NV12Buffer(width, height, stride, sliceHeight, buffer, null /* releaseCallback */) + .toI420(); + } + + private VideoFrame.Buffer copyI420Buffer( + ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { + if (stride % 2 != 0) { + throw new AssertionError("Stride is not divisible by two: " + stride); + } + + // Note that the case with odd `sliceHeight` is handled in a special way. + // The chroma height contained in the payload is rounded down instead of + // up, making it one row less than what we expect in WebRTC. Therefore, we + // have to duplicate the last chroma rows for this case. Also, the offset + // between the Y plane and the U plane is unintuitive for this case. See + // http://bugs.webrtc.org/6651 for more info. + final int chromaWidth = (width + 1) / 2; + final int chromaHeight = (sliceHeight % 2 == 0) ? (height + 1) / 2 : height / 2; + + final int uvStride = stride / 2; + + final int yPos = 0; + final int yEnd = yPos + stride * height; + final int uPos = yPos + stride * sliceHeight; + final int uEnd = uPos + uvStride * chromaHeight; + final int vPos = uPos + uvStride * sliceHeight / 2; + final int vEnd = vPos + uvStride * chromaHeight; + + VideoFrame.I420Buffer frameBuffer = allocateI420Buffer(width, height); + + buffer.limit(yEnd); + buffer.position(yPos); + copyPlane( + buffer.slice(), stride, frameBuffer.getDataY(), frameBuffer.getStrideY(), width, height); + + buffer.limit(uEnd); + buffer.position(uPos); + copyPlane(buffer.slice(), uvStride, frameBuffer.getDataU(), frameBuffer.getStrideU(), + chromaWidth, chromaHeight); + if (sliceHeight % 2 == 1) { + buffer.position(uPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. + + ByteBuffer dataU = frameBuffer.getDataU(); + dataU.position(frameBuffer.getStrideU() * chromaHeight); // Seek to beginning of last row. + dataU.put(buffer); // Copy the last row. + } + + buffer.limit(vEnd); + buffer.position(vPos); + copyPlane(buffer.slice(), uvStride, frameBuffer.getDataV(), frameBuffer.getStrideV(), + chromaWidth, chromaHeight); + if (sliceHeight % 2 == 1) { + buffer.position(vPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. + + ByteBuffer dataV = frameBuffer.getDataV(); + dataV.position(frameBuffer.getStrideV() * chromaHeight); // Seek to beginning of last row. + dataV.put(buffer); // Copy the last row. + } + + return frameBuffer; + } + + private void reformat(MediaFormat format) { + outputThreadChecker.checkIsOnValidThread(); + Logging.d(TAG, "Decoder format changed: " + format); + final int newWidth; + final int newHeight; + if (format.containsKey(MediaFormat.KEY_CROP_LEFT) + && format.containsKey(MediaFormat.KEY_CROP_RIGHT) + && format.containsKey(MediaFormat.KEY_CROP_BOTTOM) + && format.containsKey(MediaFormat.KEY_CROP_TOP)) { + newWidth = 1 + format.getInteger(MediaFormat.KEY_CROP_RIGHT) + - format.getInteger(MediaFormat.KEY_CROP_LEFT); + newHeight = 1 + format.getInteger(MediaFormat.KEY_CROP_BOTTOM) + - format.getInteger(MediaFormat.KEY_CROP_TOP); + } else { + newWidth = format.getInteger(MediaFormat.KEY_WIDTH); + newHeight = format.getInteger(MediaFormat.KEY_HEIGHT); + } + // Compare to existing width, height, and save values under the dimension lock. + synchronized (dimensionLock) { + if (newWidth != width || newHeight != height) { + if (hasDecodedFirstFrame) { + stopOnOutputThread(new RuntimeException("Unexpected size change. " + + "Configured " + width + "*" + height + ". " + + "New " + newWidth + "*" + newHeight)); + return; + } else if (newWidth <= 0 || newHeight <= 0) { + Logging.w(TAG, + "Unexpected format dimensions. Configured " + width + "*" + height + ". " + + "New " + newWidth + "*" + newHeight + ". Skip it"); + return; + } + width = newWidth; + height = newHeight; + } + } + + // Note: texture mode ignores colorFormat. Hence, if the texture helper is non-null, skip + // color format updates. + if (surfaceTextureHelper == null && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { + colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); + Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); + if (!isSupportedColorFormat(colorFormat)) { + stopOnOutputThread(new IllegalStateException("Unsupported color format: " + colorFormat)); + return; + } + } + + // Save stride and sliceHeight under the dimension lock. + synchronized (dimensionLock) { + if (format.containsKey(MediaFormat.KEY_STRIDE)) { + stride = format.getInteger(MediaFormat.KEY_STRIDE); + } + if (format.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + sliceHeight = format.getInteger(MediaFormat.KEY_SLICE_HEIGHT); + } + Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); + stride = Math.max(width, stride); + sliceHeight = Math.max(height, sliceHeight); + } + } + + private void releaseCodecOnOutputThread() { + outputThreadChecker.checkIsOnValidThread(); + Logging.d(TAG, "Releasing MediaCodec on output thread"); + try { + codec.stop(); + } catch (Exception e) { + Logging.e(TAG, "Media decoder stop failed", e); + } + try { + codec.release(); + } catch (Exception e) { + Logging.e(TAG, "Media decoder release failed", e); + // Propagate exceptions caught during release back to the main thread. + shutdownException = e; + } + Logging.d(TAG, "Release on output thread done"); + } + + private void stopOnOutputThread(Exception e) { + outputThreadChecker.checkIsOnValidThread(); + running = false; + shutdownException = e; + } + + private boolean isSupportedColorFormat(int colorFormat) { + for (int supported : MediaCodecUtils.DECODER_COLOR_FORMATS) { + if (supported == colorFormat) { + return true; + } + } + return false; + } + + // Visible for testing. + protected SurfaceTextureHelper createSurfaceTextureHelper() { + return SurfaceTextureHelper.create("decoder-texture-thread", sharedContext); + } + + // Visible for testing. + // TODO(sakal): Remove once Robolectric commit fa991a0 has been rolled to WebRTC. + protected void releaseSurface() { + surface.release(); + } + + // Visible for testing. + protected VideoFrame.I420Buffer allocateI420Buffer(int width, int height) { + return JavaI420Buffer.allocate(width, height); + } + + // Visible for testing. + protected void copyPlane( + ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) { + YuvHelper.copyPlane(src, srcStride, dst, dstStride, width, height); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java new file mode 100644 index 0000000000..3b5f5d2931 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** BitrateAdjuster that tracks bitrate and framerate but does not adjust them. */ +class BaseBitrateAdjuster implements BitrateAdjuster { + protected int targetBitrateBps; + protected double targetFramerateFps; + + @Override + public void setTargets(int targetBitrateBps, double targetFramerateFps) { + this.targetBitrateBps = targetBitrateBps; + this.targetFramerateFps = targetFramerateFps; + } + + @Override + public void reportEncodedFrame(int size) { + // No op. + } + + @Override + public int getAdjustedBitrateBps() { + return targetBitrateBps; + } + + @Override + public double getAdjustedFramerateFps() { + return targetFramerateFps; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BitrateAdjuster.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BitrateAdjuster.java new file mode 100644 index 0000000000..bfa08bad89 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/BitrateAdjuster.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Object that adjusts the bitrate of a hardware codec. */ +interface BitrateAdjuster { + /** + * Sets the target bitrate in bits per second and framerate in frames per second. + */ + void setTargets(int targetBitrateBps, double targetFramerateFps); + + /** + * Should be used to report the size of an encoded frame to the bitrate adjuster. Use + * getAdjustedBitrateBps to get the updated bitrate after calling this method. + */ + void reportEncodedFrame(int size); + + /** Gets the current bitrate. */ + int getAdjustedBitrateBps(); + + /** Gets the current framerate. */ + double getAdjustedFramerateFps(); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNative.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNative.java new file mode 100644 index 0000000000..9b410ceaef --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNative.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @CalledByNative is used by the JNI generator to create the necessary JNI + * bindings and expose this method to native code. + */ +@Target({ElementType.CONSTRUCTOR, ElementType.METHOD}) +@Retention(RetentionPolicy.CLASS) +public @interface CalledByNative { + /* + * If present, tells which inner class the method belongs to. + */ + public String value() default ""; +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java new file mode 100644 index 0000000000..8a00a7fadb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @CalledByNativeUnchecked is used to generate JNI bindings that do not check for exceptions. + * It only makes sense to use this annotation on methods that declare a throws... spec. + * However, note that the exception received native side maybe an 'unchecked' (RuntimeExpception) + * such as NullPointerException, so the native code should differentiate these cases. + * Usage of this should be very rare; where possible handle exceptions in the Java side and use a + * return value to indicate success / failure. + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.CLASS) +public @interface CalledByNativeUnchecked { + /* + * If present, tells which inner class the method belongs to. + */ + public String value() default ""; +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera1Session.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera1Session.java new file mode 100644 index 0000000000..a54f7201b2 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera1Session.java @@ -0,0 +1,340 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.hardware.Camera; +import android.os.Handler; +import android.os.SystemClock; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; + +@SuppressWarnings("deprecation") +class Camera1Session implements CameraSession { + private static final String TAG = "Camera1Session"; + private static final int NUMBER_OF_CAPTURE_BUFFERS = 3; + + private static final Histogram camera1StartTimeMsHistogram = + Histogram.createCounts("WebRTC.Android.Camera1.StartTimeMs", 1, 10000, 50); + private static final Histogram camera1StopTimeMsHistogram = + Histogram.createCounts("WebRTC.Android.Camera1.StopTimeMs", 1, 10000, 50); + private static final Histogram camera1ResolutionHistogram = Histogram.createEnumeration( + "WebRTC.Android.Camera1.Resolution", CameraEnumerationAndroid.COMMON_RESOLUTIONS.size()); + + private static enum SessionState { RUNNING, STOPPED } + + private final Handler cameraThreadHandler; + private final Events events; + private final boolean captureToTexture; + private final Context applicationContext; + private final SurfaceTextureHelper surfaceTextureHelper; + private final int cameraId; + private final Camera camera; + private final Camera.CameraInfo info; + private final CaptureFormat captureFormat; + // Used only for stats. Only used on the camera thread. + private final long constructionTimeNs; // Construction time of this class. + + private SessionState state; + private boolean firstFrameReported; + + // TODO(titovartem) make correct fix during webrtc:9175 + @SuppressWarnings("ByteBufferBackingArray") + public static void create(final CreateSessionCallback callback, final Events events, + final boolean captureToTexture, final Context applicationContext, + final SurfaceTextureHelper surfaceTextureHelper, final String cameraName, + final int width, final int height, final int framerate) { + final long constructionTimeNs = System.nanoTime(); + Logging.d(TAG, "Open camera " + cameraName); + events.onCameraOpening(); + + final int cameraId; + try { + cameraId = Camera1Enumerator.getCameraIndex(cameraName); + } catch (IllegalArgumentException e) { + callback.onFailure(FailureType.ERROR, e.getMessage()); + return; + } + + final Camera camera; + try { + camera = Camera.open(cameraId); + } catch (RuntimeException e) { + callback.onFailure(FailureType.ERROR, e.getMessage()); + return; + } + + if (camera == null) { + callback.onFailure( + FailureType.ERROR, "Camera.open returned null for camera id = " + cameraId); + return; + } + + try { + camera.setPreviewTexture(surfaceTextureHelper.getSurfaceTexture()); + } catch (IOException | RuntimeException e) { + camera.release(); + callback.onFailure(FailureType.ERROR, e.getMessage()); + return; + } + + final Camera.CameraInfo info = new Camera.CameraInfo(); + Camera.getCameraInfo(cameraId, info); + + final CaptureFormat captureFormat; + try { + final Camera.Parameters parameters = camera.getParameters(); + captureFormat = findClosestCaptureFormat(parameters, width, height, framerate); + final Size pictureSize = findClosestPictureSize(parameters, width, height); + updateCameraParameters(camera, parameters, captureFormat, pictureSize, captureToTexture); + } catch (RuntimeException e) { + camera.release(); + callback.onFailure(FailureType.ERROR, e.getMessage()); + return; + } + + if (!captureToTexture) { + final int frameSize = captureFormat.frameSize(); + for (int i = 0; i < NUMBER_OF_CAPTURE_BUFFERS; ++i) { + final ByteBuffer buffer = ByteBuffer.allocateDirect(frameSize); + camera.addCallbackBuffer(buffer.array()); + } + } + + // Calculate orientation manually and send it as CVO instead. + try { + camera.setDisplayOrientation(0 /* degrees */); + } catch (RuntimeException e) { + camera.release(); + callback.onFailure(FailureType.ERROR, e.getMessage()); + return; + } + + callback.onDone(new Camera1Session(events, captureToTexture, applicationContext, + surfaceTextureHelper, cameraId, camera, info, captureFormat, constructionTimeNs)); + } + + private static void updateCameraParameters(Camera camera, Camera.Parameters parameters, + CaptureFormat captureFormat, Size pictureSize, boolean captureToTexture) { + final List<String> focusModes = parameters.getSupportedFocusModes(); + + parameters.setPreviewFpsRange(captureFormat.framerate.min, captureFormat.framerate.max); + parameters.setPreviewSize(captureFormat.width, captureFormat.height); + parameters.setPictureSize(pictureSize.width, pictureSize.height); + if (!captureToTexture) { + parameters.setPreviewFormat(captureFormat.imageFormat); + } + + if (parameters.isVideoStabilizationSupported()) { + parameters.setVideoStabilization(true); + } + if (focusModes != null && focusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { + parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + } + camera.setParameters(parameters); + } + + private static CaptureFormat findClosestCaptureFormat( + Camera.Parameters parameters, int width, int height, int framerate) { + // Find closest supported format for `width` x `height` @ `framerate`. + final List<CaptureFormat.FramerateRange> supportedFramerates = + Camera1Enumerator.convertFramerates(parameters.getSupportedPreviewFpsRange()); + Logging.d(TAG, "Available fps ranges: " + supportedFramerates); + + final CaptureFormat.FramerateRange fpsRange = + CameraEnumerationAndroid.getClosestSupportedFramerateRange(supportedFramerates, framerate); + + final Size previewSize = CameraEnumerationAndroid.getClosestSupportedSize( + Camera1Enumerator.convertSizes(parameters.getSupportedPreviewSizes()), width, height); + CameraEnumerationAndroid.reportCameraResolution(camera1ResolutionHistogram, previewSize); + + return new CaptureFormat(previewSize.width, previewSize.height, fpsRange); + } + + private static Size findClosestPictureSize(Camera.Parameters parameters, int width, int height) { + return CameraEnumerationAndroid.getClosestSupportedSize( + Camera1Enumerator.convertSizes(parameters.getSupportedPictureSizes()), width, height); + } + + private Camera1Session(Events events, boolean captureToTexture, Context applicationContext, + SurfaceTextureHelper surfaceTextureHelper, int cameraId, Camera camera, + Camera.CameraInfo info, CaptureFormat captureFormat, long constructionTimeNs) { + Logging.d(TAG, "Create new camera1 session on camera " + cameraId); + + this.cameraThreadHandler = new Handler(); + this.events = events; + this.captureToTexture = captureToTexture; + this.applicationContext = applicationContext; + this.surfaceTextureHelper = surfaceTextureHelper; + this.cameraId = cameraId; + this.camera = camera; + this.info = info; + this.captureFormat = captureFormat; + this.constructionTimeNs = constructionTimeNs; + + surfaceTextureHelper.setTextureSize(captureFormat.width, captureFormat.height); + + startCapturing(); + } + + @Override + public void stop() { + Logging.d(TAG, "Stop camera1 session on camera " + cameraId); + checkIsOnCameraThread(); + if (state != SessionState.STOPPED) { + final long stopStartTime = System.nanoTime(); + stopInternal(); + final int stopTimeMs = (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - stopStartTime); + camera1StopTimeMsHistogram.addSample(stopTimeMs); + } + } + + private void startCapturing() { + Logging.d(TAG, "Start capturing"); + checkIsOnCameraThread(); + + state = SessionState.RUNNING; + + camera.setErrorCallback(new Camera.ErrorCallback() { + @Override + public void onError(int error, Camera camera) { + String errorMessage; + if (error == Camera.CAMERA_ERROR_SERVER_DIED) { + errorMessage = "Camera server died!"; + } else { + errorMessage = "Camera error: " + error; + } + Logging.e(TAG, errorMessage); + stopInternal(); + if (error == Camera.CAMERA_ERROR_EVICTED) { + events.onCameraDisconnected(Camera1Session.this); + } else { + events.onCameraError(Camera1Session.this, errorMessage); + } + } + }); + + if (captureToTexture) { + listenForTextureFrames(); + } else { + listenForBytebufferFrames(); + } + try { + camera.startPreview(); + } catch (RuntimeException e) { + stopInternal(); + events.onCameraError(this, e.getMessage()); + } + } + + private void stopInternal() { + Logging.d(TAG, "Stop internal"); + checkIsOnCameraThread(); + if (state == SessionState.STOPPED) { + Logging.d(TAG, "Camera is already stopped"); + return; + } + + state = SessionState.STOPPED; + surfaceTextureHelper.stopListening(); + // Note: stopPreview or other driver code might deadlock. Deadlock in + // Camera._stopPreview(Native Method) has been observed on + // Nexus 5 (hammerhead), OS version LMY48I. + camera.stopPreview(); + camera.release(); + events.onCameraClosed(this); + Logging.d(TAG, "Stop done"); + } + + private void listenForTextureFrames() { + surfaceTextureHelper.startListening((VideoFrame frame) -> { + checkIsOnCameraThread(); + + if (state != SessionState.RUNNING) { + Logging.d(TAG, "Texture frame captured but camera is no longer running."); + return; + } + + if (!firstFrameReported) { + final int startTimeMs = + (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - constructionTimeNs); + camera1StartTimeMsHistogram.addSample(startTimeMs); + firstFrameReported = true; + } + + // Undo the mirror that the OS "helps" us with. + // http://developer.android.com/reference/android/hardware/Camera.html#setDisplayOrientation(int) + final VideoFrame modifiedFrame = + new VideoFrame(CameraSession.createTextureBufferWithModifiedTransformMatrix( + (TextureBufferImpl) frame.getBuffer(), + /* mirror= */ info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT, + /* rotation= */ 0), + /* rotation= */ getFrameOrientation(), frame.getTimestampNs()); + events.onFrameCaptured(Camera1Session.this, modifiedFrame); + modifiedFrame.release(); + }); + } + + private void listenForBytebufferFrames() { + camera.setPreviewCallbackWithBuffer(new Camera.PreviewCallback() { + @Override + public void onPreviewFrame(final byte[] data, Camera callbackCamera) { + checkIsOnCameraThread(); + + if (callbackCamera != camera) { + Logging.e(TAG, "Callback from a different camera. This should never happen."); + return; + } + + if (state != SessionState.RUNNING) { + Logging.d(TAG, "Bytebuffer frame captured but camera is no longer running."); + return; + } + + final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); + + if (!firstFrameReported) { + final int startTimeMs = + (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - constructionTimeNs); + camera1StartTimeMsHistogram.addSample(startTimeMs); + firstFrameReported = true; + } + + VideoFrame.Buffer frameBuffer = new NV21Buffer( + data, captureFormat.width, captureFormat.height, () -> cameraThreadHandler.post(() -> { + if (state == SessionState.RUNNING) { + camera.addCallbackBuffer(data); + } + })); + final VideoFrame frame = new VideoFrame(frameBuffer, getFrameOrientation(), captureTimeNs); + events.onFrameCaptured(Camera1Session.this, frame); + frame.release(); + } + }); + } + + private int getFrameOrientation() { + int rotation = CameraSession.getDeviceOrientation(applicationContext); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + rotation = 360 - rotation; + } + return (info.orientation + rotation) % 360; + } + + private void checkIsOnCameraThread() { + if (Thread.currentThread() != cameraThreadHandler.getLooper().getThread()) { + throw new IllegalStateException("Wrong thread"); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera2Session.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera2Session.java new file mode 100644 index 0000000000..d5ee80c73e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Camera2Session.java @@ -0,0 +1,428 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CameraMetadata; +import android.hardware.camera2.CaptureFailure; +import android.hardware.camera2.CaptureRequest; +import android.os.Handler; +import android.util.Range; +import android.view.Surface; +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; + +class Camera2Session implements CameraSession { + private static final String TAG = "Camera2Session"; + + private static final Histogram camera2StartTimeMsHistogram = + Histogram.createCounts("WebRTC.Android.Camera2.StartTimeMs", 1, 10000, 50); + private static final Histogram camera2StopTimeMsHistogram = + Histogram.createCounts("WebRTC.Android.Camera2.StopTimeMs", 1, 10000, 50); + private static final Histogram camera2ResolutionHistogram = Histogram.createEnumeration( + "WebRTC.Android.Camera2.Resolution", CameraEnumerationAndroid.COMMON_RESOLUTIONS.size()); + + private static enum SessionState { RUNNING, STOPPED } + + private final Handler cameraThreadHandler; + private final CreateSessionCallback callback; + private final Events events; + private final Context applicationContext; + private final CameraManager cameraManager; + private final SurfaceTextureHelper surfaceTextureHelper; + private final String cameraId; + private final int width; + private final int height; + private final int framerate; + + // Initialized at start + private CameraCharacteristics cameraCharacteristics; + private int cameraOrientation; + private boolean isCameraFrontFacing; + private int fpsUnitFactor; + private CaptureFormat captureFormat; + + // Initialized when camera opens + @Nullable private CameraDevice cameraDevice; + @Nullable private Surface surface; + + // Initialized when capture session is created + @Nullable private CameraCaptureSession captureSession; + + // State + private SessionState state = SessionState.RUNNING; + private boolean firstFrameReported; + + // Used only for stats. Only used on the camera thread. + private final long constructionTimeNs; // Construction time of this class. + + private class CameraStateCallback extends CameraDevice.StateCallback { + private String getErrorDescription(int errorCode) { + switch (errorCode) { + case CameraDevice.StateCallback.ERROR_CAMERA_DEVICE: + return "Camera device has encountered a fatal error."; + case CameraDevice.StateCallback.ERROR_CAMERA_DISABLED: + return "Camera device could not be opened due to a device policy."; + case CameraDevice.StateCallback.ERROR_CAMERA_IN_USE: + return "Camera device is in use already."; + case CameraDevice.StateCallback.ERROR_CAMERA_SERVICE: + return "Camera service has encountered a fatal error."; + case CameraDevice.StateCallback.ERROR_MAX_CAMERAS_IN_USE: + return "Camera device could not be opened because" + + " there are too many other open camera devices."; + default: + return "Unknown camera error: " + errorCode; + } + } + + @Override + public void onDisconnected(CameraDevice camera) { + checkIsOnCameraThread(); + final boolean startFailure = (captureSession == null) && (state != SessionState.STOPPED); + state = SessionState.STOPPED; + stopInternal(); + if (startFailure) { + callback.onFailure(FailureType.DISCONNECTED, "Camera disconnected / evicted."); + } else { + events.onCameraDisconnected(Camera2Session.this); + } + } + + @Override + public void onError(CameraDevice camera, int errorCode) { + checkIsOnCameraThread(); + reportError(getErrorDescription(errorCode)); + } + + @Override + public void onOpened(CameraDevice camera) { + checkIsOnCameraThread(); + + Logging.d(TAG, "Camera opened."); + cameraDevice = camera; + + surfaceTextureHelper.setTextureSize(captureFormat.width, captureFormat.height); + surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); + try { + camera.createCaptureSession( + Arrays.asList(surface), new CaptureSessionCallback(), cameraThreadHandler); + } catch (CameraAccessException e) { + reportError("Failed to create capture session. " + e); + return; + } + } + + @Override + public void onClosed(CameraDevice camera) { + checkIsOnCameraThread(); + + Logging.d(TAG, "Camera device closed."); + events.onCameraClosed(Camera2Session.this); + } + } + + private class CaptureSessionCallback extends CameraCaptureSession.StateCallback { + @Override + public void onConfigureFailed(CameraCaptureSession session) { + checkIsOnCameraThread(); + session.close(); + reportError("Failed to configure capture session."); + } + + @Override + public void onConfigured(CameraCaptureSession session) { + checkIsOnCameraThread(); + Logging.d(TAG, "Camera capture session configured."); + captureSession = session; + try { + /* + * The viable options for video capture requests are: + * TEMPLATE_PREVIEW: High frame rate is given priority over the highest-quality + * post-processing. + * TEMPLATE_RECORD: Stable frame rate is used, and post-processing is set for recording + * quality. + */ + final CaptureRequest.Builder captureRequestBuilder = + cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); + // Set auto exposure fps range. + captureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, + new Range<Integer>(captureFormat.framerate.min / fpsUnitFactor, + captureFormat.framerate.max / fpsUnitFactor)); + captureRequestBuilder.set( + CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON); + captureRequestBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false); + chooseStabilizationMode(captureRequestBuilder); + chooseFocusMode(captureRequestBuilder); + + captureRequestBuilder.addTarget(surface); + session.setRepeatingRequest( + captureRequestBuilder.build(), new CameraCaptureCallback(), cameraThreadHandler); + } catch (CameraAccessException e) { + reportError("Failed to start capture request. " + e); + return; + } + + surfaceTextureHelper.startListening((VideoFrame frame) -> { + checkIsOnCameraThread(); + + if (state != SessionState.RUNNING) { + Logging.d(TAG, "Texture frame captured but camera is no longer running."); + return; + } + + if (!firstFrameReported) { + firstFrameReported = true; + final int startTimeMs = + (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - constructionTimeNs); + camera2StartTimeMsHistogram.addSample(startTimeMs); + } + + // Undo the mirror that the OS "helps" us with. + // http://developer.android.com/reference/android/hardware/Camera.html#setDisplayOrientation(int) + // Also, undo camera orientation, we report it as rotation instead. + final VideoFrame modifiedFrame = + new VideoFrame(CameraSession.createTextureBufferWithModifiedTransformMatrix( + (TextureBufferImpl) frame.getBuffer(), + /* mirror= */ isCameraFrontFacing, + /* rotation= */ -cameraOrientation), + /* rotation= */ getFrameOrientation(), frame.getTimestampNs()); + events.onFrameCaptured(Camera2Session.this, modifiedFrame); + modifiedFrame.release(); + }); + Logging.d(TAG, "Camera device successfully started."); + callback.onDone(Camera2Session.this); + } + + // Prefers optical stabilization over software stabilization if available. Only enables one of + // the stabilization modes at a time because having both enabled can cause strange results. + private void chooseStabilizationMode(CaptureRequest.Builder captureRequestBuilder) { + final int[] availableOpticalStabilization = cameraCharacteristics.get( + CameraCharacteristics.LENS_INFO_AVAILABLE_OPTICAL_STABILIZATION); + if (availableOpticalStabilization != null) { + for (int mode : availableOpticalStabilization) { + if (mode == CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_ON) { + captureRequestBuilder.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, + CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_ON); + captureRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, + CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_OFF); + Logging.d(TAG, "Using optical stabilization."); + return; + } + } + } + // If no optical mode is available, try software. + final int[] availableVideoStabilization = cameraCharacteristics.get( + CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES); + if (availableVideoStabilization != null) { + for (int mode : availableVideoStabilization) { + if (mode == CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON) { + captureRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, + CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON); + captureRequestBuilder.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, + CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_OFF); + Logging.d(TAG, "Using video stabilization."); + return; + } + } + } + Logging.d(TAG, "Stabilization not available."); + } + + private void chooseFocusMode(CaptureRequest.Builder captureRequestBuilder) { + final int[] availableFocusModes = + cameraCharacteristics.get(CameraCharacteristics.CONTROL_AF_AVAILABLE_MODES); + for (int mode : availableFocusModes) { + if (mode == CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO) { + captureRequestBuilder.set( + CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO); + Logging.d(TAG, "Using continuous video auto-focus."); + return; + } + } + Logging.d(TAG, "Auto-focus is not available."); + } + } + + private static class CameraCaptureCallback extends CameraCaptureSession.CaptureCallback { + @Override + public void onCaptureFailed( + CameraCaptureSession session, CaptureRequest request, CaptureFailure failure) { + Logging.d(TAG, "Capture failed: " + failure); + } + } + + public static void create(CreateSessionCallback callback, Events events, + Context applicationContext, CameraManager cameraManager, + SurfaceTextureHelper surfaceTextureHelper, String cameraId, int width, int height, + int framerate) { + new Camera2Session(callback, events, applicationContext, cameraManager, surfaceTextureHelper, + cameraId, width, height, framerate); + } + + private Camera2Session(CreateSessionCallback callback, Events events, Context applicationContext, + CameraManager cameraManager, SurfaceTextureHelper surfaceTextureHelper, String cameraId, + int width, int height, int framerate) { + Logging.d(TAG, "Create new camera2 session on camera " + cameraId); + + constructionTimeNs = System.nanoTime(); + + this.cameraThreadHandler = new Handler(); + this.callback = callback; + this.events = events; + this.applicationContext = applicationContext; + this.cameraManager = cameraManager; + this.surfaceTextureHelper = surfaceTextureHelper; + this.cameraId = cameraId; + this.width = width; + this.height = height; + this.framerate = framerate; + + start(); + } + + private void start() { + checkIsOnCameraThread(); + Logging.d(TAG, "start"); + + try { + cameraCharacteristics = cameraManager.getCameraCharacteristics(cameraId); + } catch (CameraAccessException | IllegalArgumentException e) { + reportError("getCameraCharacteristics(): " + e.getMessage()); + return; + } + cameraOrientation = cameraCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); + isCameraFrontFacing = cameraCharacteristics.get(CameraCharacteristics.LENS_FACING) + == CameraMetadata.LENS_FACING_FRONT; + + findCaptureFormat(); + + if (captureFormat == null) { + // findCaptureFormat reports an error already. + return; + } + + openCamera(); + } + + private void findCaptureFormat() { + checkIsOnCameraThread(); + + Range<Integer>[] fpsRanges = + cameraCharacteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); + fpsUnitFactor = Camera2Enumerator.getFpsUnitFactor(fpsRanges); + List<CaptureFormat.FramerateRange> framerateRanges = + Camera2Enumerator.convertFramerates(fpsRanges, fpsUnitFactor); + List<Size> sizes = Camera2Enumerator.getSupportedSizes(cameraCharacteristics); + Logging.d(TAG, "Available preview sizes: " + sizes); + Logging.d(TAG, "Available fps ranges: " + framerateRanges); + + if (framerateRanges.isEmpty() || sizes.isEmpty()) { + reportError("No supported capture formats."); + return; + } + + final CaptureFormat.FramerateRange bestFpsRange = + CameraEnumerationAndroid.getClosestSupportedFramerateRange(framerateRanges, framerate); + + final Size bestSize = CameraEnumerationAndroid.getClosestSupportedSize(sizes, width, height); + CameraEnumerationAndroid.reportCameraResolution(camera2ResolutionHistogram, bestSize); + + captureFormat = new CaptureFormat(bestSize.width, bestSize.height, bestFpsRange); + Logging.d(TAG, "Using capture format: " + captureFormat); + } + + @SuppressLint("MissingPermission") + private void openCamera() { + checkIsOnCameraThread(); + + Logging.d(TAG, "Opening camera " + cameraId); + events.onCameraOpening(); + + try { + cameraManager.openCamera(cameraId, new CameraStateCallback(), cameraThreadHandler); + } catch (CameraAccessException | IllegalArgumentException | SecurityException e) { + reportError("Failed to open camera: " + e); + return; + } + } + + @Override + public void stop() { + Logging.d(TAG, "Stop camera2 session on camera " + cameraId); + checkIsOnCameraThread(); + if (state != SessionState.STOPPED) { + final long stopStartTime = System.nanoTime(); + state = SessionState.STOPPED; + stopInternal(); + final int stopTimeMs = (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - stopStartTime); + camera2StopTimeMsHistogram.addSample(stopTimeMs); + } + } + + private void stopInternal() { + Logging.d(TAG, "Stop internal"); + checkIsOnCameraThread(); + + surfaceTextureHelper.stopListening(); + + if (captureSession != null) { + captureSession.close(); + captureSession = null; + } + if (surface != null) { + surface.release(); + surface = null; + } + if (cameraDevice != null) { + cameraDevice.close(); + cameraDevice = null; + } + + Logging.d(TAG, "Stop done"); + } + + private void reportError(String error) { + checkIsOnCameraThread(); + Logging.e(TAG, "Error: " + error); + + final boolean startFailure = (captureSession == null) && (state != SessionState.STOPPED); + state = SessionState.STOPPED; + stopInternal(); + if (startFailure) { + callback.onFailure(FailureType.ERROR, error); + } else { + events.onCameraError(this, error); + } + } + + private int getFrameOrientation() { + int rotation = CameraSession.getDeviceOrientation(applicationContext); + if (!isCameraFrontFacing) { + rotation = 360 - rotation; + } + return (cameraOrientation + rotation) % 360; + } + + private void checkIsOnCameraThread() { + if (Thread.currentThread() != cameraThreadHandler.getLooper().getThread()) { + throw new IllegalStateException("Wrong thread"); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraCapturer.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraCapturer.java new file mode 100644 index 0000000000..1922a529e2 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraCapturer.java @@ -0,0 +1,458 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.os.Handler; +import android.os.Looper; +import androidx.annotation.Nullable; +import java.util.Arrays; +import java.util.List; + +@SuppressWarnings("deprecation") +abstract class CameraCapturer implements CameraVideoCapturer { + enum SwitchState { + IDLE, // No switch requested. + PENDING, // Waiting for previous capture session to open. + IN_PROGRESS, // Waiting for new switched capture session to start. + } + + private static final String TAG = "CameraCapturer"; + private final static int MAX_OPEN_CAMERA_ATTEMPTS = 3; + private final static int OPEN_CAMERA_DELAY_MS = 500; + private final static int OPEN_CAMERA_TIMEOUT = 10000; + + private final CameraEnumerator cameraEnumerator; + private final CameraEventsHandler eventsHandler; + private final Handler uiThreadHandler; + + @Nullable + private final CameraSession.CreateSessionCallback createSessionCallback = + new CameraSession.CreateSessionCallback() { + @Override + public void onDone(CameraSession session) { + checkIsOnCameraThread(); + Logging.d(TAG, "Create session done. Switch state: " + switchState); + uiThreadHandler.removeCallbacks(openCameraTimeoutRunnable); + synchronized (stateLock) { + capturerObserver.onCapturerStarted(true /* success */); + sessionOpening = false; + currentSession = session; + cameraStatistics = new CameraStatistics(surfaceHelper, eventsHandler); + firstFrameObserved = false; + stateLock.notifyAll(); + + if (switchState == SwitchState.IN_PROGRESS) { + switchState = SwitchState.IDLE; + if (switchEventsHandler != null) { + switchEventsHandler.onCameraSwitchDone(cameraEnumerator.isFrontFacing(cameraName)); + switchEventsHandler = null; + } + } else if (switchState == SwitchState.PENDING) { + String selectedCameraName = pendingCameraName; + pendingCameraName = null; + switchState = SwitchState.IDLE; + switchCameraInternal(switchEventsHandler, selectedCameraName); + } + } + } + + @Override + public void onFailure(CameraSession.FailureType failureType, String error) { + checkIsOnCameraThread(); + uiThreadHandler.removeCallbacks(openCameraTimeoutRunnable); + synchronized (stateLock) { + capturerObserver.onCapturerStarted(false /* success */); + openAttemptsRemaining--; + + if (openAttemptsRemaining <= 0) { + Logging.w(TAG, "Opening camera failed, passing: " + error); + sessionOpening = false; + stateLock.notifyAll(); + + if (switchState != SwitchState.IDLE) { + if (switchEventsHandler != null) { + switchEventsHandler.onCameraSwitchError(error); + switchEventsHandler = null; + } + switchState = SwitchState.IDLE; + } + + if (failureType == CameraSession.FailureType.DISCONNECTED) { + eventsHandler.onCameraDisconnected(); + } else { + eventsHandler.onCameraError(error); + } + } else { + Logging.w(TAG, "Opening camera failed, retry: " + error); + createSessionInternal(OPEN_CAMERA_DELAY_MS); + } + } + } + }; + + @Nullable + private final CameraSession.Events cameraSessionEventsHandler = new CameraSession.Events() { + @Override + public void onCameraOpening() { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (currentSession != null) { + Logging.w(TAG, "onCameraOpening while session was open."); + return; + } + eventsHandler.onCameraOpening(cameraName); + } + } + + @Override + public void onCameraError(CameraSession session, String error) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onCameraError from another session: " + error); + return; + } + eventsHandler.onCameraError(error); + stopCapture(); + } + } + + @Override + public void onCameraDisconnected(CameraSession session) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onCameraDisconnected from another session."); + return; + } + eventsHandler.onCameraDisconnected(); + stopCapture(); + } + } + + @Override + public void onCameraClosed(CameraSession session) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession && currentSession != null) { + Logging.d(TAG, "onCameraClosed from another session."); + return; + } + eventsHandler.onCameraClosed(); + } + } + + @Override + public void onFrameCaptured(CameraSession session, VideoFrame frame) { + checkIsOnCameraThread(); + synchronized (stateLock) { + if (session != currentSession) { + Logging.w(TAG, "onFrameCaptured from another session."); + return; + } + if (!firstFrameObserved) { + eventsHandler.onFirstFrameAvailable(); + firstFrameObserved = true; + } + cameraStatistics.addFrame(); + capturerObserver.onFrameCaptured(frame); + } + } + }; + + private final Runnable openCameraTimeoutRunnable = new Runnable() { + @Override + public void run() { + eventsHandler.onCameraError("Camera failed to start within timeout."); + } + }; + + // Initialized on initialize + // ------------------------- + private Handler cameraThreadHandler; + private Context applicationContext; + private org.webrtc.CapturerObserver capturerObserver; + private SurfaceTextureHelper surfaceHelper; + + private final Object stateLock = new Object(); + private boolean sessionOpening; /* guarded by stateLock */ + @Nullable private CameraSession currentSession; /* guarded by stateLock */ + private String cameraName; /* guarded by stateLock */ + private String pendingCameraName; /* guarded by stateLock */ + private int width; /* guarded by stateLock */ + private int height; /* guarded by stateLock */ + private int framerate; /* guarded by stateLock */ + private int openAttemptsRemaining; /* guarded by stateLock */ + private SwitchState switchState = SwitchState.IDLE; /* guarded by stateLock */ + @Nullable private CameraSwitchHandler switchEventsHandler; /* guarded by stateLock */ + // Valid from onDone call until stopCapture, otherwise null. + @Nullable private CameraStatistics cameraStatistics; /* guarded by stateLock */ + private boolean firstFrameObserved; /* guarded by stateLock */ + + public CameraCapturer(String cameraName, @Nullable CameraEventsHandler eventsHandler, + CameraEnumerator cameraEnumerator) { + if (eventsHandler == null) { + eventsHandler = new CameraEventsHandler() { + @Override + public void onCameraError(String errorDescription) {} + @Override + public void onCameraDisconnected() {} + @Override + public void onCameraFreezed(String errorDescription) {} + @Override + public void onCameraOpening(String cameraName) {} + @Override + public void onFirstFrameAvailable() {} + @Override + public void onCameraClosed() {} + }; + } + + this.eventsHandler = eventsHandler; + this.cameraEnumerator = cameraEnumerator; + this.cameraName = cameraName; + List<String> deviceNames = Arrays.asList(cameraEnumerator.getDeviceNames()); + uiThreadHandler = new Handler(Looper.getMainLooper()); + + if (deviceNames.isEmpty()) { + throw new RuntimeException("No cameras attached."); + } + if (!deviceNames.contains(this.cameraName)) { + throw new IllegalArgumentException( + "Camera name " + this.cameraName + " does not match any known camera device."); + } + } + + @Override + public void initialize(SurfaceTextureHelper surfaceTextureHelper, Context applicationContext, + org.webrtc.CapturerObserver capturerObserver) { + this.applicationContext = applicationContext; + this.capturerObserver = capturerObserver; + this.surfaceHelper = surfaceTextureHelper; + this.cameraThreadHandler = surfaceTextureHelper.getHandler(); + } + + @Override + public void startCapture(int width, int height, int framerate) { + Logging.d(TAG, "startCapture: " + width + "x" + height + "@" + framerate); + if (applicationContext == null) { + throw new RuntimeException("CameraCapturer must be initialized before calling startCapture."); + } + + synchronized (stateLock) { + if (sessionOpening || currentSession != null) { + Logging.w(TAG, "Session already open"); + return; + } + + this.width = width; + this.height = height; + this.framerate = framerate; + + sessionOpening = true; + openAttemptsRemaining = MAX_OPEN_CAMERA_ATTEMPTS; + createSessionInternal(0); + } + } + + private void createSessionInternal(int delayMs) { + uiThreadHandler.postDelayed(openCameraTimeoutRunnable, delayMs + OPEN_CAMERA_TIMEOUT); + cameraThreadHandler.postDelayed(new Runnable() { + @Override + public void run() { + createCameraSession(createSessionCallback, cameraSessionEventsHandler, applicationContext, + surfaceHelper, cameraName, width, height, framerate); + } + }, delayMs); + } + + @Override + public void stopCapture() { + Logging.d(TAG, "Stop capture"); + + synchronized (stateLock) { + while (sessionOpening) { + Logging.d(TAG, "Stop capture: Waiting for session to open"); + try { + stateLock.wait(); + } catch (InterruptedException e) { + Logging.w(TAG, "Stop capture interrupted while waiting for the session to open."); + Thread.currentThread().interrupt(); + return; + } + } + + if (currentSession != null) { + Logging.d(TAG, "Stop capture: Nulling session"); + cameraStatistics.release(); + cameraStatistics = null; + final CameraSession oldSession = currentSession; + cameraThreadHandler.post(new Runnable() { + @Override + public void run() { + oldSession.stop(); + } + }); + currentSession = null; + capturerObserver.onCapturerStopped(); + } else { + Logging.d(TAG, "Stop capture: No session open"); + } + } + + Logging.d(TAG, "Stop capture done"); + } + + @Override + public void changeCaptureFormat(int width, int height, int framerate) { + Logging.d(TAG, "changeCaptureFormat: " + width + "x" + height + "@" + framerate); + synchronized (stateLock) { + stopCapture(); + startCapture(width, height, framerate); + } + } + + @Override + public void dispose() { + Logging.d(TAG, "dispose"); + stopCapture(); + } + + @Override + public void switchCamera(final CameraSwitchHandler switchEventsHandler) { + Logging.d(TAG, "switchCamera"); + cameraThreadHandler.post(new Runnable() { + @Override + public void run() { + List<String> deviceNames = Arrays.asList(cameraEnumerator.getDeviceNames()); + + if (deviceNames.size() < 2) { + reportCameraSwitchError("No camera to switch to.", switchEventsHandler); + return; + } + + int cameraNameIndex = deviceNames.indexOf(cameraName); + String cameraName = deviceNames.get((cameraNameIndex + 1) % deviceNames.size()); + switchCameraInternal(switchEventsHandler, cameraName); + } + }); + } + + @Override + public void switchCamera(final CameraSwitchHandler switchEventsHandler, final String cameraName) { + Logging.d(TAG, "switchCamera"); + cameraThreadHandler.post(new Runnable() { + @Override + public void run() { + switchCameraInternal(switchEventsHandler, cameraName); + } + }); + } + + @Override + public boolean isScreencast() { + return false; + } + + public void printStackTrace() { + Thread cameraThread = null; + if (cameraThreadHandler != null) { + cameraThread = cameraThreadHandler.getLooper().getThread(); + } + if (cameraThread != null) { + StackTraceElement[] cameraStackTrace = cameraThread.getStackTrace(); + if (cameraStackTrace.length > 0) { + Logging.d(TAG, "CameraCapturer stack trace:"); + for (StackTraceElement traceElem : cameraStackTrace) { + Logging.d(TAG, traceElem.toString()); + } + } + } + } + + private void reportCameraSwitchError( + String error, @Nullable CameraSwitchHandler switchEventsHandler) { + Logging.e(TAG, error); + if (switchEventsHandler != null) { + switchEventsHandler.onCameraSwitchError(error); + } + } + + private void switchCameraInternal( + @Nullable final CameraSwitchHandler switchEventsHandler, final String selectedCameraName) { + Logging.d(TAG, "switchCamera internal"); + List<String> deviceNames = Arrays.asList(cameraEnumerator.getDeviceNames()); + + if (!deviceNames.contains(selectedCameraName)) { + reportCameraSwitchError("Attempted to switch to unknown camera device " + selectedCameraName, + switchEventsHandler); + return; + } + + synchronized (stateLock) { + if (switchState != SwitchState.IDLE) { + reportCameraSwitchError("Camera switch already in progress.", switchEventsHandler); + return; + } + if (!sessionOpening && currentSession == null) { + reportCameraSwitchError("switchCamera: camera is not running.", switchEventsHandler); + return; + } + + this.switchEventsHandler = switchEventsHandler; + if (sessionOpening) { + switchState = SwitchState.PENDING; + pendingCameraName = selectedCameraName; + return; + } else { + switchState = SwitchState.IN_PROGRESS; + } + + Logging.d(TAG, "switchCamera: Stopping session"); + cameraStatistics.release(); + cameraStatistics = null; + final CameraSession oldSession = currentSession; + cameraThreadHandler.post(new Runnable() { + @Override + public void run() { + oldSession.stop(); + } + }); + currentSession = null; + + cameraName = selectedCameraName; + + sessionOpening = true; + openAttemptsRemaining = 1; + createSessionInternal(0); + } + Logging.d(TAG, "switchCamera done"); + } + + private void checkIsOnCameraThread() { + if (Thread.currentThread() != cameraThreadHandler.getLooper().getThread()) { + Logging.e(TAG, "Check is on camera thread failed."); + throw new RuntimeException("Not on camera thread."); + } + } + + protected String getCameraName() { + synchronized (stateLock) { + return cameraName; + } + } + + abstract protected void createCameraSession( + CameraSession.CreateSessionCallback createSessionCallback, CameraSession.Events events, + Context applicationContext, SurfaceTextureHelper surfaceTextureHelper, String cameraName, + int width, int height, int framerate); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraSession.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraSession.java new file mode 100644 index 0000000000..8d137854d8 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/CameraSession.java @@ -0,0 +1,72 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.content.Context; +import android.graphics.Matrix; +import android.view.WindowManager; +import android.view.Surface; + +interface CameraSession { + enum FailureType { ERROR, DISCONNECTED } + + // Callbacks are fired on the camera thread. + interface CreateSessionCallback { + void onDone(CameraSession session); + void onFailure(FailureType failureType, String error); + } + + // Events are fired on the camera thread. + interface Events { + void onCameraOpening(); + void onCameraError(CameraSession session, String error); + void onCameraDisconnected(CameraSession session); + void onCameraClosed(CameraSession session); + void onFrameCaptured(CameraSession session, VideoFrame frame); + } + + /** + * Stops the capture. Waits until no more calls to capture observer will be made. + * If waitCameraStop is true, also waits for the camera to stop. + */ + void stop(); + + static int getDeviceOrientation(Context context) { + final WindowManager wm = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + switch (wm.getDefaultDisplay().getRotation()) { + case Surface.ROTATION_90: + return 90; + case Surface.ROTATION_180: + return 180; + case Surface.ROTATION_270: + return 270; + case Surface.ROTATION_0: + default: + return 0; + } + } + + static VideoFrame.TextureBuffer createTextureBufferWithModifiedTransformMatrix( + TextureBufferImpl buffer, boolean mirror, int rotation) { + final Matrix transformMatrix = new Matrix(); + // Perform mirror and rotation around (0.5, 0.5) since that is the center of the texture. + transformMatrix.preTranslate(/* dx= */ 0.5f, /* dy= */ 0.5f); + if (mirror) { + transformMatrix.preScale(/* sx= */ -1f, /* sy= */ 1f); + } + transformMatrix.preRotate(rotation); + transformMatrix.preTranslate(/* dx= */ -0.5f, /* dy= */ -0.5f); + + // The width and height are not affected by rotation since Camera2Session has set them to the + // value they should be after undoing the rotation. + return buffer.applyTransformMatrix(transformMatrix, buffer.getWidth(), buffer.getHeight()); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java new file mode 100644 index 0000000000..96a15bbfe1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java @@ -0,0 +1,98 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * BitrateAdjuster that tracks the bandwidth produced by an encoder and dynamically adjusts the + * bitrate. Used for hardware codecs that pay attention to framerate but still deviate from the + * target bitrate by unacceptable margins. + */ +class DynamicBitrateAdjuster extends BaseBitrateAdjuster { + // Change the bitrate at most once every three seconds. + private static final double BITRATE_ADJUSTMENT_SEC = 3.0; + // Maximum bitrate adjustment scale - no more than 4 times. + private static final double BITRATE_ADJUSTMENT_MAX_SCALE = 4; + // Amount of adjustment steps to reach maximum scale. + private static final int BITRATE_ADJUSTMENT_STEPS = 20; + + private static final double BITS_PER_BYTE = 8.0; + + // How far the codec has deviated above (or below) the target bitrate (tracked in bytes). + private double deviationBytes; + private double timeSinceLastAdjustmentMs; + private int bitrateAdjustmentScaleExp; + + @Override + public void setTargets(int targetBitrateBps, double targetFramerateFps) { + if (this.targetBitrateBps > 0 && targetBitrateBps < this.targetBitrateBps) { + // Rescale the accumulator level if the accumulator max decreases + deviationBytes = deviationBytes * targetBitrateBps / this.targetBitrateBps; + } + super.setTargets(targetBitrateBps, targetFramerateFps); + } + + @Override + public void reportEncodedFrame(int size) { + if (targetFramerateFps == 0) { + return; + } + + // Accumulate the difference between actual and expected frame sizes. + double expectedBytesPerFrame = (targetBitrateBps / BITS_PER_BYTE) / targetFramerateFps; + deviationBytes += (size - expectedBytesPerFrame); + timeSinceLastAdjustmentMs += 1000.0 / targetFramerateFps; + + // Adjust the bitrate when the encoder accumulates one second's worth of data in excess or + // shortfall of the target. + double deviationThresholdBytes = targetBitrateBps / BITS_PER_BYTE; + + // Cap the deviation, i.e., don't let it grow beyond some level to avoid using too old data for + // bitrate adjustment. This also prevents taking more than 3 "steps" in a given 3-second cycle. + double deviationCap = BITRATE_ADJUSTMENT_SEC * deviationThresholdBytes; + deviationBytes = Math.min(deviationBytes, deviationCap); + deviationBytes = Math.max(deviationBytes, -deviationCap); + + // Do bitrate adjustment every 3 seconds if actual encoder bitrate deviates too much + // from the target value. + if (timeSinceLastAdjustmentMs <= 1000 * BITRATE_ADJUSTMENT_SEC) { + return; + } + + if (deviationBytes > deviationThresholdBytes) { + // Encoder generates too high bitrate - need to reduce the scale. + int bitrateAdjustmentInc = (int) (deviationBytes / deviationThresholdBytes + 0.5); + bitrateAdjustmentScaleExp -= bitrateAdjustmentInc; + // Don't let the adjustment scale drop below -BITRATE_ADJUSTMENT_STEPS. + // This sets a minimum exponent of -1 (bitrateAdjustmentScaleExp / BITRATE_ADJUSTMENT_STEPS). + bitrateAdjustmentScaleExp = Math.max(bitrateAdjustmentScaleExp, -BITRATE_ADJUSTMENT_STEPS); + deviationBytes = deviationThresholdBytes; + } else if (deviationBytes < -deviationThresholdBytes) { + // Encoder generates too low bitrate - need to increase the scale. + int bitrateAdjustmentInc = (int) (-deviationBytes / deviationThresholdBytes + 0.5); + bitrateAdjustmentScaleExp += bitrateAdjustmentInc; + // Don't let the adjustment scale exceed BITRATE_ADJUSTMENT_STEPS. + // This sets a maximum exponent of 1 (bitrateAdjustmentScaleExp / BITRATE_ADJUSTMENT_STEPS). + bitrateAdjustmentScaleExp = Math.min(bitrateAdjustmentScaleExp, BITRATE_ADJUSTMENT_STEPS); + deviationBytes = -deviationThresholdBytes; + } + timeSinceLastAdjustmentMs = 0; + } + + private double getBitrateAdjustmentScale() { + return Math.pow(BITRATE_ADJUSTMENT_MAX_SCALE, + (double) bitrateAdjustmentScaleExp / BITRATE_ADJUSTMENT_STEPS); + } + + @Override + public int getAdjustedBitrateBps() { + return (int) (targetBitrateBps * getBitrateAdjustmentScale()); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase10Impl.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase10Impl.java new file mode 100644 index 0000000000..caa10e7e51 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase10Impl.java @@ -0,0 +1,448 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.Canvas; +import android.graphics.Rect; +import android.graphics.SurfaceTexture; +import android.opengl.GLException; +import android.view.Surface; +import android.view.SurfaceHolder; +import androidx.annotation.Nullable; +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; +import javax.microedition.khronos.egl.EGLSurface; + +/** + * Holds EGL state and utility methods for handling an egl 1.0 EGLContext, an EGLDisplay, + * and an EGLSurface. + */ +class EglBase10Impl implements EglBase10 { + private static final String TAG = "EglBase10Impl"; + // This constant is taken from EGL14.EGL_CONTEXT_CLIENT_VERSION. + private static final int EGL_CONTEXT_CLIENT_VERSION = 0x3098; + + private static final EglConnection EGL_NO_CONNECTION = new EglConnection(); + + private EGLSurface eglSurface = EGL10.EGL_NO_SURFACE; + private EglConnection eglConnection; + + // EGL wrapper for an actual EGLContext. + private static class Context implements EglBase10.Context { + private final EGL10 egl; + private final EGLContext eglContext; + private final EGLConfig eglContextConfig; + + @Override + public EGLContext getRawContext() { + return eglContext; + } + + @Override + public long getNativeEglContext() { + EGLContext previousContext = egl.eglGetCurrentContext(); + EGLDisplay currentDisplay = egl.eglGetCurrentDisplay(); + EGLSurface previousDrawSurface = egl.eglGetCurrentSurface(EGL10.EGL_DRAW); + EGLSurface previousReadSurface = egl.eglGetCurrentSurface(EGL10.EGL_READ); + EGLSurface tempEglSurface = null; + + if (currentDisplay == EGL10.EGL_NO_DISPLAY) { + currentDisplay = egl.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY); + } + + try { + if (previousContext != eglContext) { + int[] surfaceAttribs = {EGL10.EGL_WIDTH, 1, EGL10.EGL_HEIGHT, 1, EGL10.EGL_NONE}; + tempEglSurface = + egl.eglCreatePbufferSurface(currentDisplay, eglContextConfig, surfaceAttribs); + if (!egl.eglMakeCurrent(currentDisplay, tempEglSurface, tempEglSurface, eglContext)) { + throw new GLException(egl.eglGetError(), + "Failed to make temporary EGL surface active: " + egl.eglGetError()); + } + } + + return nativeGetCurrentNativeEGLContext(); + } finally { + if (tempEglSurface != null) { + egl.eglMakeCurrent( + currentDisplay, previousDrawSurface, previousReadSurface, previousContext); + egl.eglDestroySurface(currentDisplay, tempEglSurface); + } + } + } + + public Context(EGL10 egl, EGLContext eglContext, EGLConfig eglContextConfig) { + this.egl = egl; + this.eglContext = eglContext; + this.eglContextConfig = eglContextConfig; + } + } + + public static class EglConnection implements EglBase10.EglConnection { + private final EGL10 egl; + private final EGLContext eglContext; + private final EGLDisplay eglDisplay; + private final EGLConfig eglConfig; + private final RefCountDelegate refCountDelegate; + private EGLSurface currentSurface = EGL10.EGL_NO_SURFACE; + + public EglConnection(EGLContext sharedContext, int[] configAttributes) { + egl = (EGL10) EGLContext.getEGL(); + eglDisplay = getEglDisplay(egl); + eglConfig = getEglConfig(egl, eglDisplay, configAttributes); + final int openGlesVersion = EglBase.getOpenGlesVersionFromConfig(configAttributes); + Logging.d(TAG, "Using OpenGL ES version " + openGlesVersion); + eglContext = createEglContext(egl, sharedContext, eglDisplay, eglConfig, openGlesVersion); + + // Ref count delegate with release callback. + refCountDelegate = new RefCountDelegate(() -> { + synchronized (EglBase.lock) { + egl.eglMakeCurrent( + eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT); + } + egl.eglDestroyContext(eglDisplay, eglContext); + egl.eglTerminate(eglDisplay); + currentSurface = EGL10.EGL_NO_SURFACE; + }); + } + + // Returns a "null" EglConnection. Useful to represent a released instance with default values. + private EglConnection() { + egl = (EGL10) EGLContext.getEGL(); + eglContext = EGL10.EGL_NO_CONTEXT; + eglDisplay = EGL10.EGL_NO_DISPLAY; + eglConfig = null; + refCountDelegate = new RefCountDelegate(() -> {}); + } + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public EGL10 getEgl() { + return egl; + } + + @Override + public EGLContext getContext() { + return eglContext; + } + + @Override + public EGLDisplay getDisplay() { + return eglDisplay; + } + + @Override + public EGLConfig getConfig() { + return eglConfig; + } + + public void makeCurrent(EGLSurface eglSurface) { + if (egl.eglGetCurrentContext() == eglContext && currentSurface == eglSurface) { + return; + } + + synchronized (EglBase.lock) { + if (!egl.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { + throw new GLException(egl.eglGetError(), + "eglMakeCurrent failed: 0x" + Integer.toHexString(egl.eglGetError())); + } + } + currentSurface = eglSurface; + } + + public void detachCurrent() { + synchronized (EglBase.lock) { + if (!egl.eglMakeCurrent( + eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT)) { + throw new GLException(egl.eglGetError(), + "eglDetachCurrent failed: 0x" + Integer.toHexString(egl.eglGetError())); + } + } + currentSurface = EGL10.EGL_NO_SURFACE; + } + } + + // Create a new context with the specified config type, sharing data with sharedContext. + public EglBase10Impl(EGLContext sharedContext, int[] configAttributes) { + this.eglConnection = new EglConnection(sharedContext, configAttributes); + } + + public EglBase10Impl(EglConnection eglConnection) { + this.eglConnection = eglConnection; + this.eglConnection.retain(); + } + + @Override + public void createSurface(Surface surface) { + /** + * We have to wrap Surface in a SurfaceHolder because for some reason eglCreateWindowSurface + * couldn't actually take a Surface object until API 17. Older versions fortunately just call + * SurfaceHolder.getSurface(), so we'll do that. No other methods are relevant. + */ + class FakeSurfaceHolder implements SurfaceHolder { + private final Surface surface; + + FakeSurfaceHolder(Surface surface) { + this.surface = surface; + } + + @Override + public void addCallback(Callback callback) {} + + @Override + public void removeCallback(Callback callback) {} + + @Override + public boolean isCreating() { + return false; + } + + @Deprecated + @Override + public void setType(int i) {} + + @Override + public void setFixedSize(int i, int i2) {} + + @Override + public void setSizeFromLayout() {} + + @Override + public void setFormat(int i) {} + + @Override + public void setKeepScreenOn(boolean b) {} + + @Nullable + @Override + public Canvas lockCanvas() { + return null; + } + + @Nullable + @Override + public Canvas lockCanvas(Rect rect) { + return null; + } + + @Override + public void unlockCanvasAndPost(Canvas canvas) {} + + @Nullable + @Override + public Rect getSurfaceFrame() { + return null; + } + + @Override + public Surface getSurface() { + return surface; + } + } + + createSurfaceInternal(new FakeSurfaceHolder(surface)); + } + + // Create EGLSurface from the Android SurfaceTexture. + @Override + public void createSurface(SurfaceTexture surfaceTexture) { + createSurfaceInternal(surfaceTexture); + } + + // Create EGLSurface from either a SurfaceHolder or a SurfaceTexture. + private void createSurfaceInternal(Object nativeWindow) { + if (!(nativeWindow instanceof SurfaceHolder) && !(nativeWindow instanceof SurfaceTexture)) { + throw new IllegalStateException("Input must be either a SurfaceHolder or SurfaceTexture"); + } + checkIsNotReleased(); + if (eglSurface != EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + + EGL10 egl = eglConnection.getEgl(); + int[] surfaceAttribs = {EGL10.EGL_NONE}; + eglSurface = egl.eglCreateWindowSurface( + eglConnection.getDisplay(), eglConnection.getConfig(), nativeWindow, surfaceAttribs); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new GLException(egl.eglGetError(), + "Failed to create window surface: 0x" + Integer.toHexString(egl.eglGetError())); + } + } + + // Create dummy 1x1 pixel buffer surface so the context can be made current. + @Override + public void createDummyPbufferSurface() { + createPbufferSurface(1, 1); + } + + @Override + public void createPbufferSurface(int width, int height) { + checkIsNotReleased(); + if (eglSurface != EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + EGL10 egl = eglConnection.getEgl(); + int[] surfaceAttribs = {EGL10.EGL_WIDTH, width, EGL10.EGL_HEIGHT, height, EGL10.EGL_NONE}; + eglSurface = egl.eglCreatePbufferSurface( + eglConnection.getDisplay(), eglConnection.getConfig(), surfaceAttribs); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new GLException(egl.eglGetError(), + "Failed to create pixel buffer surface with size " + width + "x" + height + ": 0x" + + Integer.toHexString(egl.eglGetError())); + } + } + + @Override + public org.webrtc.EglBase.Context getEglBaseContext() { + return new Context( + eglConnection.getEgl(), eglConnection.getContext(), eglConnection.getConfig()); + } + + @Override + public boolean hasSurface() { + return eglSurface != EGL10.EGL_NO_SURFACE; + } + + @Override + public int surfaceWidth() { + final int widthArray[] = new int[1]; + eglConnection.getEgl().eglQuerySurface( + eglConnection.getDisplay(), eglSurface, EGL10.EGL_WIDTH, widthArray); + return widthArray[0]; + } + + @Override + public int surfaceHeight() { + final int heightArray[] = new int[1]; + eglConnection.getEgl().eglQuerySurface( + eglConnection.getDisplay(), eglSurface, EGL10.EGL_HEIGHT, heightArray); + return heightArray[0]; + } + + @Override + public void releaseSurface() { + if (eglSurface != EGL10.EGL_NO_SURFACE) { + eglConnection.getEgl().eglDestroySurface(eglConnection.getDisplay(), eglSurface); + eglSurface = EGL10.EGL_NO_SURFACE; + } + } + + private void checkIsNotReleased() { + if (eglConnection == EGL_NO_CONNECTION) { + throw new RuntimeException("This object has been released"); + } + } + + @Override + public void release() { + checkIsNotReleased(); + releaseSurface(); + eglConnection.release(); + eglConnection = EGL_NO_CONNECTION; + } + + @Override + public void makeCurrent() { + checkIsNotReleased(); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't make current"); + } + eglConnection.makeCurrent(eglSurface); + } + + // Detach the current EGL context, so that it can be made current on another thread. + @Override + public void detachCurrent() { + eglConnection.detachCurrent(); + } + + @Override + public void swapBuffers() { + checkIsNotReleased(); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + synchronized (EglBase.lock) { + eglConnection.getEgl().eglSwapBuffers(eglConnection.getDisplay(), eglSurface); + } + } + + @Override + public void swapBuffers(long timeStampNs) { + // Setting presentation time is not supported for EGL 1.0. + swapBuffers(); + } + + // Return an EGLDisplay, or die trying. + private static EGLDisplay getEglDisplay(EGL10 egl) { + EGLDisplay eglDisplay = egl.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL10.EGL_NO_DISPLAY) { + throw new GLException(egl.eglGetError(), + "Unable to get EGL10 display: 0x" + Integer.toHexString(egl.eglGetError())); + } + int[] version = new int[2]; + if (!egl.eglInitialize(eglDisplay, version)) { + throw new GLException(egl.eglGetError(), + "Unable to initialize EGL10: 0x" + Integer.toHexString(egl.eglGetError())); + } + return eglDisplay; + } + + // Return an EGLConfig, or die trying. + private static EGLConfig getEglConfig(EGL10 egl, EGLDisplay eglDisplay, int[] configAttributes) { + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + if (!egl.eglChooseConfig(eglDisplay, configAttributes, configs, configs.length, numConfigs)) { + throw new GLException( + egl.eglGetError(), "eglChooseConfig failed: 0x" + Integer.toHexString(egl.eglGetError())); + } + if (numConfigs[0] <= 0) { + throw new RuntimeException("Unable to find any matching EGL config"); + } + final EGLConfig eglConfig = configs[0]; + if (eglConfig == null) { + throw new RuntimeException("eglChooseConfig returned null"); + } + return eglConfig; + } + + // Return an EGLConfig, or die trying. + private static EGLContext createEglContext(EGL10 egl, @Nullable EGLContext sharedContext, + EGLDisplay eglDisplay, EGLConfig eglConfig, int openGlesVersion) { + if (sharedContext != null && sharedContext == EGL10.EGL_NO_CONTEXT) { + throw new RuntimeException("Invalid sharedContext"); + } + int[] contextAttributes = {EGL_CONTEXT_CLIENT_VERSION, openGlesVersion, EGL10.EGL_NONE}; + EGLContext rootContext = sharedContext == null ? EGL10.EGL_NO_CONTEXT : sharedContext; + final EGLContext eglContext; + synchronized (EglBase.lock) { + eglContext = egl.eglCreateContext(eglDisplay, eglConfig, rootContext, contextAttributes); + } + if (eglContext == EGL10.EGL_NO_CONTEXT) { + throw new GLException(egl.eglGetError(), + "Failed to create EGL context: 0x" + Integer.toHexString(egl.eglGetError())); + } + return eglContext; + } + + private static native long nativeGetCurrentNativeEGLContext(); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase14Impl.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase14Impl.java new file mode 100644 index 0000000000..22cee8668b --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/EglBase14Impl.java @@ -0,0 +1,340 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.opengl.EGLConfig; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLExt; +import android.opengl.EGLSurface; +import android.opengl.GLException; +import android.view.Surface; +import androidx.annotation.Nullable; + +/** + * Holds EGL state and utility methods for handling an EGL14 EGLContext, an EGLDisplay, + * and an EGLSurface. + */ +@SuppressWarnings("ReferenceEquality") // We want to compare to EGL14 constants. +class EglBase14Impl implements EglBase14 { + private static final String TAG = "EglBase14Impl"; + private static final EglConnection EGL_NO_CONNECTION = new EglConnection(); + + private EGLSurface eglSurface = EGL14.EGL_NO_SURFACE; + private EglConnection eglConnection; + + public static class Context implements EglBase14.Context { + private final EGLContext egl14Context; + + @Override + public EGLContext getRawContext() { + return egl14Context; + } + + @Override + public long getNativeEglContext() { + return egl14Context.getNativeHandle(); + } + + public Context(android.opengl.EGLContext eglContext) { + this.egl14Context = eglContext; + } + } + + public static class EglConnection implements EglBase14.EglConnection { + private final EGLContext eglContext; + private final EGLDisplay eglDisplay; + private final EGLConfig eglConfig; + private final RefCountDelegate refCountDelegate; + private EGLSurface currentSurface = EGL14.EGL_NO_SURFACE; + + public EglConnection(EGLContext sharedContext, int[] configAttributes) { + eglDisplay = getEglDisplay(); + eglConfig = getEglConfig(eglDisplay, configAttributes); + final int openGlesVersion = EglBase.getOpenGlesVersionFromConfig(configAttributes); + Logging.d(TAG, "Using OpenGL ES version " + openGlesVersion); + eglContext = createEglContext(sharedContext, eglDisplay, eglConfig, openGlesVersion); + + // Ref count delegate with release callback. + refCountDelegate = new RefCountDelegate(() -> { + synchronized (EglBase.lock) { + EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT); + EGL14.eglDestroyContext(eglDisplay, eglContext); + } + EGL14.eglReleaseThread(); + EGL14.eglTerminate(eglDisplay); + currentSurface = EGL14.EGL_NO_SURFACE; + }); + } + + // Returns a "null" EglConnection. Useful to represent a released instance with default values. + private EglConnection() { + eglContext = EGL14.EGL_NO_CONTEXT; + eglDisplay = EGL14.EGL_NO_DISPLAY; + eglConfig = null; + refCountDelegate = new RefCountDelegate(() -> {}); + } + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public EGLContext getContext() { + return eglContext; + } + + @Override + public EGLDisplay getDisplay() { + return eglDisplay; + } + + @Override + public EGLConfig getConfig() { + return eglConfig; + } + + public void makeCurrent(EGLSurface eglSurface) { + if (EGL14.eglGetCurrentContext() == eglContext && currentSurface == eglSurface) { + return; + } + + synchronized (EglBase.lock) { + if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { + throw new GLException(EGL14.eglGetError(), + "eglMakeCurrent failed: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + } + currentSurface = eglSurface; + } + + public void detachCurrent() { + synchronized (EglBase.lock) { + if (!EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT)) { + throw new GLException(EGL14.eglGetError(), + "eglDetachCurrent failed: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + } + currentSurface = EGL14.EGL_NO_SURFACE; + } + } + // Create a new context with the specified config type, sharing data with sharedContext. + // `sharedContext` may be null. + public EglBase14Impl(EGLContext sharedContext, int[] configAttributes) { + this.eglConnection = new EglConnection(sharedContext, configAttributes); + } + + // Create a new EglBase using an existing, possibly externally managed, EglConnection. + public EglBase14Impl(EglConnection eglConnection) { + this.eglConnection = eglConnection; + this.eglConnection.retain(); + } + + // Create EGLSurface from the Android Surface. + @Override + public void createSurface(Surface surface) { + createSurfaceInternal(surface); + } + + // Create EGLSurface from the Android SurfaceTexture. + @Override + public void createSurface(SurfaceTexture surfaceTexture) { + createSurfaceInternal(surfaceTexture); + } + + // Create EGLSurface from either Surface or SurfaceTexture. + private void createSurfaceInternal(Object surface) { + if (!(surface instanceof Surface) && !(surface instanceof SurfaceTexture)) { + throw new IllegalStateException("Input must be either a Surface or SurfaceTexture"); + } + checkIsNotReleased(); + if (eglSurface != EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL14.EGL_NONE}; + eglSurface = EGL14.eglCreateWindowSurface( + eglConnection.getDisplay(), eglConnection.getConfig(), surface, surfaceAttribs, 0); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new GLException(EGL14.eglGetError(), + "Failed to create window surface: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + } + + @Override + public void createDummyPbufferSurface() { + createPbufferSurface(1, 1); + } + + @Override + public void createPbufferSurface(int width, int height) { + checkIsNotReleased(); + if (eglSurface != EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL14.EGL_WIDTH, width, EGL14.EGL_HEIGHT, height, EGL14.EGL_NONE}; + eglSurface = EGL14.eglCreatePbufferSurface( + eglConnection.getDisplay(), eglConnection.getConfig(), surfaceAttribs, 0); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new GLException(EGL14.eglGetError(), + "Failed to create pixel buffer surface with size " + width + "x" + height + ": 0x" + + Integer.toHexString(EGL14.eglGetError())); + } + } + + @Override + public Context getEglBaseContext() { + return new Context(eglConnection.getContext()); + } + + @Override + public boolean hasSurface() { + return eglSurface != EGL14.EGL_NO_SURFACE; + } + + @Override + public int surfaceWidth() { + final int[] widthArray = new int[1]; + EGL14.eglQuerySurface(eglConnection.getDisplay(), eglSurface, EGL14.EGL_WIDTH, widthArray, 0); + return widthArray[0]; + } + + @Override + public int surfaceHeight() { + final int[] heightArray = new int[1]; + EGL14.eglQuerySurface(eglConnection.getDisplay(), eglSurface, EGL14.EGL_HEIGHT, heightArray, 0); + return heightArray[0]; + } + + @Override + public void releaseSurface() { + if (eglSurface != EGL14.EGL_NO_SURFACE) { + EGL14.eglDestroySurface(eglConnection.getDisplay(), eglSurface); + eglSurface = EGL14.EGL_NO_SURFACE; + } + } + + private void checkIsNotReleased() { + if (eglConnection == EGL_NO_CONNECTION) { + throw new RuntimeException("This object has been released"); + } + } + + @Override + public void release() { + checkIsNotReleased(); + releaseSurface(); + eglConnection.release(); + eglConnection = EGL_NO_CONNECTION; + } + + @Override + public void makeCurrent() { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't make current"); + } + eglConnection.makeCurrent(eglSurface); + } + + // Detach the current EGL context, so that it can be made current on another thread. + @Override + public void detachCurrent() { + eglConnection.detachCurrent(); + } + + @Override + public void swapBuffers() { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + synchronized (EglBase.lock) { + EGL14.eglSwapBuffers(eglConnection.getDisplay(), eglSurface); + } + } + + @Override + public void swapBuffers(long timeStampNs) { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + synchronized (EglBase.lock) { + // See + // https://android.googlesource.com/platform/frameworks/native/+/tools_r22.2/opengl/specs/EGL_ANDROID_presentation_time.txt + EGLExt.eglPresentationTimeANDROID(eglConnection.getDisplay(), eglSurface, timeStampNs); + EGL14.eglSwapBuffers(eglConnection.getDisplay(), eglSurface); + } + } + + // Return an EGLDisplay, or die trying. + private static EGLDisplay getEglDisplay() { + EGLDisplay eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL14.EGL_NO_DISPLAY) { + throw new GLException(EGL14.eglGetError(), + "Unable to get EGL14 display: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + int[] version = new int[2]; + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw new GLException(EGL14.eglGetError(), + "Unable to initialize EGL14: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + return eglDisplay; + } + + // Return an EGLConfig, or die trying. + private static EGLConfig getEglConfig(EGLDisplay eglDisplay, int[] configAttributes) { + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + if (!EGL14.eglChooseConfig( + eglDisplay, configAttributes, 0, configs, 0, configs.length, numConfigs, 0)) { + throw new GLException(EGL14.eglGetError(), + "eglChooseConfig failed: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + if (numConfigs[0] <= 0) { + throw new RuntimeException("Unable to find any matching EGL config"); + } + final EGLConfig eglConfig = configs[0]; + if (eglConfig == null) { + throw new RuntimeException("eglChooseConfig returned null"); + } + return eglConfig; + } + + // Return an EGLConfig, or die trying. + private static EGLContext createEglContext(@Nullable EGLContext sharedContext, + EGLDisplay eglDisplay, EGLConfig eglConfig, int openGlesVersion) { + if (sharedContext != null && sharedContext == EGL14.EGL_NO_CONTEXT) { + throw new RuntimeException("Invalid sharedContext"); + } + int[] contextAttributes = {EGL14.EGL_CONTEXT_CLIENT_VERSION, openGlesVersion, EGL14.EGL_NONE}; + EGLContext rootContext = sharedContext == null ? EGL14.EGL_NO_CONTEXT : sharedContext; + final EGLContext eglContext; + synchronized (EglBase.lock) { + eglContext = EGL14.eglCreateContext(eglDisplay, eglConfig, rootContext, contextAttributes, 0); + } + if (eglContext == EGL14.EGL_NO_CONTEXT) { + throw new GLException(EGL14.eglGetError(), + "Failed to create EGL context: 0x" + Integer.toHexString(EGL14.eglGetError())); + } + return eglContext; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Empty.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Empty.java new file mode 100644 index 0000000000..fe9481e182 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Empty.java @@ -0,0 +1,17 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Empty class for use in libjingle_peerconnection_java because all targets require at least one + * Java file. + */ +class Empty {} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java new file mode 100644 index 0000000000..e28b7b5a26 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * BitrateAdjuster that adjusts the bitrate to compensate for changes in the framerate. Used with + * hardware codecs that assume the framerate never changes. + */ +class FramerateBitrateAdjuster extends BaseBitrateAdjuster { + private static final int DEFAULT_FRAMERATE_FPS = 30; + + @Override + public void setTargets(int targetBitrateBps, double targetFramerateFps) { + // Keep frame rate unchanged and adjust bit rate. + this.targetFramerateFps = DEFAULT_FRAMERATE_FPS; + this.targetBitrateBps = (int) (targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/GlGenericDrawer.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/GlGenericDrawer.java new file mode 100644 index 0000000000..b70a3728b9 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/GlGenericDrawer.java @@ -0,0 +1,284 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import androidx.annotation.Nullable; +import java.nio.FloatBuffer; +import org.webrtc.GlShader; +import org.webrtc.GlUtil; +import org.webrtc.RendererCommon; + +/** + * Helper class to implement an instance of RendererCommon.GlDrawer that can accept multiple input + * sources (OES, RGB, or YUV) using a generic fragment shader as input. The generic fragment shader + * should sample pixel values from the function "sample" that will be provided by this class and + * provides an abstraction for the input source type (OES, RGB, or YUV). The texture coordinate + * variable name will be "tc" and the texture matrix in the vertex shader will be "tex_mat". The + * simplest possible generic shader that just draws pixel from the frame unmodified looks like: + * void main() { + * gl_FragColor = sample(tc); + * } + * This class covers the cases for most simple shaders and generates the necessary boiler plate. + * Advanced shaders can always implement RendererCommon.GlDrawer directly. + */ +class GlGenericDrawer implements RendererCommon.GlDrawer { + /** + * The different shader types representing different input sources. YUV here represents three + * separate Y, U, V textures. + */ + public static enum ShaderType { OES, RGB, YUV } + + /** + * The shader callbacks is used to customize behavior for a GlDrawer. It provides a hook to set + * uniform variables in the shader before a frame is drawn. + */ + public static interface ShaderCallbacks { + /** + * This callback is called when a new shader has been compiled and created. It will be called + * for the first frame as well as when the shader type is changed. This callback can be used to + * do custom initialization of the shader that only needs to happen once. + */ + void onNewShader(GlShader shader); + + /** + * This callback is called before rendering a frame. It can be used to do custom preparation of + * the shader that needs to happen every frame. + */ + void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, + int viewportWidth, int viewportHeight); + } + + private static final String INPUT_VERTEX_COORDINATE_NAME = "in_pos"; + private static final String INPUT_TEXTURE_COORDINATE_NAME = "in_tc"; + private static final String TEXTURE_MATRIX_NAME = "tex_mat"; + private static final String DEFAULT_VERTEX_SHADER_STRING = "varying vec2 tc;\n" + + "attribute vec4 in_pos;\n" + + "attribute vec4 in_tc;\n" + + "uniform mat4 tex_mat;\n" + + "void main() {\n" + + " gl_Position = in_pos;\n" + + " tc = (tex_mat * in_tc).xy;\n" + + "}\n"; + + // Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1) + // is top-right. + private static final FloatBuffer FULL_RECTANGLE_BUFFER = GlUtil.createFloatBuffer(new float[] { + -1.0f, -1.0f, // Bottom left. + 1.0f, -1.0f, // Bottom right. + -1.0f, 1.0f, // Top left. + 1.0f, 1.0f, // Top right. + }); + + // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. + private static final FloatBuffer FULL_RECTANGLE_TEXTURE_BUFFER = + GlUtil.createFloatBuffer(new float[] { + 0.0f, 0.0f, // Bottom left. + 1.0f, 0.0f, // Bottom right. + 0.0f, 1.0f, // Top left. + 1.0f, 1.0f, // Top right. + }); + + static String createFragmentShaderString(String genericFragmentSource, ShaderType shaderType) { + final StringBuilder stringBuilder = new StringBuilder(); + if (shaderType == ShaderType.OES) { + stringBuilder.append("#extension GL_OES_EGL_image_external : require\n"); + } + stringBuilder.append("precision mediump float;\n"); + stringBuilder.append("varying vec2 tc;\n"); + + if (shaderType == ShaderType.YUV) { + stringBuilder.append("uniform sampler2D y_tex;\n"); + stringBuilder.append("uniform sampler2D u_tex;\n"); + stringBuilder.append("uniform sampler2D v_tex;\n"); + + // Add separate function for sampling texture. + // yuv_to_rgb_mat is inverse of the matrix defined in YuvConverter. + stringBuilder.append("vec4 sample(vec2 p) {\n"); + stringBuilder.append(" float y = texture2D(y_tex, p).r * 1.16438;\n"); + stringBuilder.append(" float u = texture2D(u_tex, p).r;\n"); + stringBuilder.append(" float v = texture2D(v_tex, p).r;\n"); + stringBuilder.append(" return vec4(y + 1.59603 * v - 0.874202,\n"); + stringBuilder.append(" y - 0.391762 * u - 0.812968 * v + 0.531668,\n"); + stringBuilder.append(" y + 2.01723 * u - 1.08563, 1);\n"); + stringBuilder.append("}\n"); + stringBuilder.append(genericFragmentSource); + } else { + final String samplerName = shaderType == ShaderType.OES ? "samplerExternalOES" : "sampler2D"; + stringBuilder.append("uniform ").append(samplerName).append(" tex;\n"); + + // Update the sampling function in-place. + stringBuilder.append(genericFragmentSource.replace("sample(", "texture2D(tex, ")); + } + + return stringBuilder.toString(); + } + + private final String genericFragmentSource; + private final String vertexShader; + private final ShaderCallbacks shaderCallbacks; + @Nullable private ShaderType currentShaderType; + @Nullable private GlShader currentShader; + private int inPosLocation; + private int inTcLocation; + private int texMatrixLocation; + + public GlGenericDrawer(String genericFragmentSource, ShaderCallbacks shaderCallbacks) { + this(DEFAULT_VERTEX_SHADER_STRING, genericFragmentSource, shaderCallbacks); + } + + public GlGenericDrawer( + String vertexShader, String genericFragmentSource, ShaderCallbacks shaderCallbacks) { + this.vertexShader = vertexShader; + this.genericFragmentSource = genericFragmentSource; + this.shaderCallbacks = shaderCallbacks; + } + + // Visible for testing. + GlShader createShader(ShaderType shaderType) { + return new GlShader( + vertexShader, createFragmentShaderString(genericFragmentSource, shaderType)); + } + + /** + * Draw an OES texture frame with specified texture transformation matrix. Required resources are + * allocated at the first call to this function. + */ + @Override + public void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight, + int viewportX, int viewportY, int viewportWidth, int viewportHeight) { + prepareShader( + ShaderType.OES, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); + // Bind the texture. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId); + // Draw the texture. + GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + // Unbind the texture as a precaution. + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); + } + + /** + * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources + * are allocated at the first call to this function. + */ + @Override + public void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight, + int viewportX, int viewportY, int viewportWidth, int viewportHeight) { + prepareShader( + ShaderType.RGB, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); + // Bind the texture. + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); + // Draw the texture. + GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + // Unbind the texture as a precaution. + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + } + + /** + * Draw a YUV frame with specified texture transformation matrix. Required resources are allocated + * at the first call to this function. + */ + @Override + public void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight, + int viewportX, int viewportY, int viewportWidth, int viewportHeight) { + prepareShader( + ShaderType.YUV, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); + // Bind the textures. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); + } + // Draw the textures. + GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + // Unbind the textures as a precaution. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + } + } + + private void prepareShader(ShaderType shaderType, float[] texMatrix, int frameWidth, + int frameHeight, int viewportWidth, int viewportHeight) { + final GlShader shader; + if (shaderType.equals(currentShaderType)) { + // Same shader type as before, reuse exising shader. + shader = currentShader; + } else { + // Allocate new shader. + currentShaderType = null; + if (currentShader != null) { + currentShader.release(); + currentShader = null; + } + + shader = createShader(shaderType); + currentShaderType = shaderType; + currentShader = shader; + + shader.useProgram(); + // Set input texture units. + if (shaderType == ShaderType.YUV) { + GLES20.glUniform1i(shader.getUniformLocation("y_tex"), 0); + GLES20.glUniform1i(shader.getUniformLocation("u_tex"), 1); + GLES20.glUniform1i(shader.getUniformLocation("v_tex"), 2); + } else { + GLES20.glUniform1i(shader.getUniformLocation("tex"), 0); + } + + GlUtil.checkNoGLES2Error("Create shader"); + shaderCallbacks.onNewShader(shader); + texMatrixLocation = shader.getUniformLocation(TEXTURE_MATRIX_NAME); + inPosLocation = shader.getAttribLocation(INPUT_VERTEX_COORDINATE_NAME); + inTcLocation = shader.getAttribLocation(INPUT_TEXTURE_COORDINATE_NAME); + } + + shader.useProgram(); + + // Upload the vertex coordinates. + GLES20.glEnableVertexAttribArray(inPosLocation); + GLES20.glVertexAttribPointer(inPosLocation, /* size= */ 2, + /* type= */ GLES20.GL_FLOAT, /* normalized= */ false, /* stride= */ 0, + FULL_RECTANGLE_BUFFER); + + // Upload the texture coordinates. + GLES20.glEnableVertexAttribArray(inTcLocation); + GLES20.glVertexAttribPointer(inTcLocation, /* size= */ 2, + /* type= */ GLES20.GL_FLOAT, /* normalized= */ false, /* stride= */ 0, + FULL_RECTANGLE_TEXTURE_BUFFER); + + // Upload the texture transformation matrix. + GLES20.glUniformMatrix4fv( + texMatrixLocation, 1 /* count= */, false /* transpose= */, texMatrix, 0 /* offset= */); + + // Do custom per-frame shader preparation. + shaderCallbacks.onPrepareShader( + shader, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); + GlUtil.checkNoGLES2Error("Prepare shader"); + } + + /** + * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked. + */ + @Override + public void release() { + if (currentShader != null) { + currentShader.release(); + currentShader = null; + currentShaderType = null; + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/H264Utils.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/H264Utils.java new file mode 100644 index 0000000000..abb79c6582 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/H264Utils.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.util.Map; +import java.util.HashMap; + +/** Container for static helper functions related to dealing with H264 codecs. */ +class H264Utils { + public static final String H264_FMTP_PROFILE_LEVEL_ID = "profile-level-id"; + public static final String H264_FMTP_LEVEL_ASYMMETRY_ALLOWED = "level-asymmetry-allowed"; + public static final String H264_FMTP_PACKETIZATION_MODE = "packetization-mode"; + + public static final String H264_PROFILE_CONSTRAINED_BASELINE = "42e0"; + public static final String H264_PROFILE_CONSTRAINED_HIGH = "640c"; + public static final String H264_LEVEL_3_1 = "1f"; // 31 in hex. + public static final String H264_CONSTRAINED_HIGH_3_1 = + H264_PROFILE_CONSTRAINED_HIGH + H264_LEVEL_3_1; + public static final String H264_CONSTRAINED_BASELINE_3_1 = + H264_PROFILE_CONSTRAINED_BASELINE + H264_LEVEL_3_1; + + public static Map<String, String> getDefaultH264Params(boolean isHighProfile) { + final Map<String, String> params = new HashMap<>(); + params.put(VideoCodecInfo.H264_FMTP_LEVEL_ASYMMETRY_ALLOWED, "1"); + params.put(VideoCodecInfo.H264_FMTP_PACKETIZATION_MODE, "1"); + params.put(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, + isHighProfile ? VideoCodecInfo.H264_CONSTRAINED_HIGH_3_1 + : VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1); + return params; + } + + public static VideoCodecInfo DEFAULT_H264_BASELINE_PROFILE_CODEC = + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false)); + public static VideoCodecInfo DEFAULT_H264_HIGH_PROFILE_CODEC = + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true)); + + public static boolean isSameH264Profile( + Map<String, String> params1, Map<String, String> params2) { + return nativeIsSameH264Profile(params1, params2); + } + + private static native boolean nativeIsSameH264Profile( + Map<String, String> params1, Map<String, String> params2); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java new file mode 100644 index 0000000000..94dfdf0728 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -0,0 +1,793 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static android.media.MediaCodecInfo.CodecProfileLevel.AVCLevel3; +import static android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; +import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaFormat; +import android.opengl.GLES20; +import android.os.Build; +import android.os.Bundle; +import android.view.Surface; +import androidx.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import org.webrtc.ThreadUtils.ThreadChecker; + +/** + * Android hardware video encoder. + */ +class HardwareVideoEncoder implements VideoEncoder { + private static final String TAG = "HardwareVideoEncoder"; + + private static final int MAX_VIDEO_FRAMERATE = 30; + + // See MAX_ENCODER_Q_SIZE in androidmediaencoder.cc. + private static final int MAX_ENCODER_Q_SIZE = 2; + + private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; + private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; + + // Size of the input frames should be multiple of 16 for the H/W encoder. + private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16; + + /** + * Keeps track of the number of output buffers that have been passed down the pipeline and not yet + * released. We need to wait for this to go down to zero before operations invalidating the output + * buffers, i.e., stop() and getOutputBuffer(). + */ + private static class BusyCount { + private final Object countLock = new Object(); + private int count; + + public void increment() { + synchronized (countLock) { + count++; + } + } + + // This method may be called on an arbitrary thread. + public void decrement() { + synchronized (countLock) { + count--; + if (count == 0) { + countLock.notifyAll(); + } + } + } + + // The increment and waitForZero methods are called on the same thread (deliverEncodedImage, + // running on the output thread). Hence, after waitForZero returns, the count will stay zero + // until the same thread calls increment. + public void waitForZero() { + boolean wasInterrupted = false; + synchronized (countLock) { + while (count > 0) { + try { + countLock.wait(); + } catch (InterruptedException e) { + Logging.e(TAG, "Interrupted while waiting on busy count", e); + wasInterrupted = true; + } + } + } + + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } + } + // --- Initialized on construction. + private final MediaCodecWrapperFactory mediaCodecWrapperFactory; + private final String codecName; + private final VideoCodecMimeType codecType; + private final Integer surfaceColorFormat; + private final Integer yuvColorFormat; + private final Map<String, String> params; + private final int keyFrameIntervalSec; // Base interval for generating key frames. + // Interval at which to force a key frame. Used to reduce color distortions caused by some + // Qualcomm video encoders. + private final long forcedKeyFrameNs; + private final BitrateAdjuster bitrateAdjuster; + // EGL context shared with the application. Used to access texture inputs. + private final EglBase14.Context sharedContext; + + // Drawer used to draw input textures onto the codec's input surface. + private final GlRectDrawer textureDrawer = new GlRectDrawer(); + private final VideoFrameDrawer videoFrameDrawer = new VideoFrameDrawer(); + // A queue of EncodedImage.Builders that correspond to frames in the codec. These builders are + // pre-populated with all the information that can't be sent through MediaCodec. + private final BlockingDeque<EncodedImage.Builder> outputBuilders = new LinkedBlockingDeque<>(); + + private final ThreadChecker encodeThreadChecker = new ThreadChecker(); + private final ThreadChecker outputThreadChecker = new ThreadChecker(); + private final BusyCount outputBuffersBusyCount = new BusyCount(); + + // --- Set on initialize and immutable until release. + private Callback callback; + private boolean automaticResizeOn; + + // --- Valid and immutable while an encoding session is running. + @Nullable private MediaCodecWrapper codec; + // Thread that delivers encoded frames to the user callback. + @Nullable private Thread outputThread; + + // EGL base wrapping the shared texture context. Holds hooks to both the shared context and the + // input surface. Making this base current allows textures from the context to be drawn onto the + // surface. + @Nullable private EglBase14 textureEglBase; + // Input surface for the codec. The encoder will draw input textures onto this surface. + @Nullable private Surface textureInputSurface; + + private int width; + private int height; + // Y-plane strides in the encoder's input + private int stride; + // Y-plane slice-height in the encoder's input + private int sliceHeight; + // True if encoder input color format is semi-planar (NV12). + private boolean isSemiPlanar; + // Size of frame for current color format and stride, in bytes. + private int frameSizeBytes; + private boolean useSurfaceMode; + + // --- Only accessed from the encoding thread. + // Presentation timestamp of next frame to encode. + private long nextPresentationTimestampUs; + // Presentation timestamp of the last requested (or forced) key frame. + private long lastKeyFrameNs; + + // --- Only accessed on the output thread. + // Contents of the last observed config frame output by the MediaCodec. Used by H.264. + @Nullable private ByteBuffer configBuffer; + private int adjustedBitrate; + + // Whether the encoder is running. Volatile so that the output thread can watch this value and + // exit when the encoder stops. + private volatile boolean running; + // Any exception thrown during shutdown. The output thread releases the MediaCodec and uses this + // value to send exceptions thrown during release back to the encoder thread. + @Nullable private volatile Exception shutdownException; + + // True if collection of encoding statistics is enabled. + private boolean isEncodingStatisticsEnabled; + + /** + * Creates a new HardwareVideoEncoder with the given codecName, codecType, colorFormat, key frame + * intervals, and bitrateAdjuster. + * + * @param codecName the hardware codec implementation to use + * @param codecType the type of the given video codec (eg. VP8, VP9, H264, H265, AV1) + * @param surfaceColorFormat color format for surface mode or null if not available + * @param yuvColorFormat color format for bytebuffer mode + * @param keyFrameIntervalSec interval in seconds between key frames; used to initialize the codec + * @param forceKeyFrameIntervalMs interval at which to force a key frame if one is not requested; + * used to reduce distortion caused by some codec implementations + * @param bitrateAdjuster algorithm used to correct codec implementations that do not produce the + * desired bitrates + * @throws IllegalArgumentException if colorFormat is unsupported + */ + public HardwareVideoEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, + VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat, + Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs, + BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) { + this.mediaCodecWrapperFactory = mediaCodecWrapperFactory; + this.codecName = codecName; + this.codecType = codecType; + this.surfaceColorFormat = surfaceColorFormat; + this.yuvColorFormat = yuvColorFormat; + this.params = params; + this.keyFrameIntervalSec = keyFrameIntervalSec; + this.forcedKeyFrameNs = TimeUnit.MILLISECONDS.toNanos(forceKeyFrameIntervalMs); + this.bitrateAdjuster = bitrateAdjuster; + this.sharedContext = sharedContext; + + // Allow construction on a different thread. + encodeThreadChecker.detachThread(); + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + encodeThreadChecker.checkIsOnValidThread(); + + this.callback = callback; + automaticResizeOn = settings.automaticResizeOn; + + this.width = settings.width; + this.height = settings.height; + useSurfaceMode = canUseSurface(); + + if (settings.startBitrate != 0 && settings.maxFramerate != 0) { + bitrateAdjuster.setTargets(settings.startBitrate * 1000, settings.maxFramerate); + } + adjustedBitrate = bitrateAdjuster.getAdjustedBitrateBps(); + + Logging.d(TAG, + "initEncode name: " + codecName + " type: " + codecType + " width: " + width + + " height: " + height + " framerate_fps: " + settings.maxFramerate + + " bitrate_kbps: " + settings.startBitrate + " surface mode: " + useSurfaceMode); + return initEncodeInternal(); + } + + private VideoCodecStatus initEncodeInternal() { + encodeThreadChecker.checkIsOnValidThread(); + + nextPresentationTimestampUs = 0; + lastKeyFrameNs = -1; + + isEncodingStatisticsEnabled = false; + + try { + codec = mediaCodecWrapperFactory.createByCodecName(codecName); + } catch (IOException | IllegalArgumentException e) { + Logging.e(TAG, "Cannot create media encoder " + codecName); + return VideoCodecStatus.FALLBACK_SOFTWARE; + } + + final int colorFormat = useSurfaceMode ? surfaceColorFormat : yuvColorFormat; + try { + MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height); + format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate); + format.setInteger(MediaFormat.KEY_BITRATE_MODE, BITRATE_MODE_CBR); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); + format.setFloat( + MediaFormat.KEY_FRAME_RATE, (float) bitrateAdjuster.getAdjustedFramerateFps()); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec); + if (codecType == VideoCodecMimeType.H264) { + String profileLevelId = params.get(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID); + if (profileLevelId == null) { + profileLevelId = VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1; + } + switch (profileLevelId) { + case VideoCodecInfo.H264_CONSTRAINED_HIGH_3_1: + format.setInteger("profile", AVCProfileHigh); + format.setInteger("level", AVCLevel3); + break; + case VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1: + break; + default: + Logging.w(TAG, "Unknown profile level id: " + profileLevelId); + } + } + + if (codecName.equals("c2.google.av1.encoder")) { + // Enable RTC mode in AV1 HW encoder. + format.setInteger("vendor.google-av1enc.encoding-preset.int32.value", 1); + } + + if (isEncodingStatisticsSupported()) { + format.setInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL, + MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1); + isEncodingStatisticsEnabled = true; + } + + Logging.d(TAG, "Format: " + format); + codec.configure( + format, null /* surface */, null /* crypto */, MediaCodec.CONFIGURE_FLAG_ENCODE); + + if (useSurfaceMode) { + textureEglBase = EglBase.createEgl14(sharedContext, EglBase.CONFIG_RECORDABLE); + textureInputSurface = codec.createInputSurface(); + textureEglBase.createSurface(textureInputSurface); + textureEglBase.makeCurrent(); + } + + updateInputFormat(codec.getInputFormat()); + + codec.start(); + } catch (IllegalStateException e) { + Logging.e(TAG, "initEncodeInternal failed", e); + release(); + return VideoCodecStatus.FALLBACK_SOFTWARE; + } + + running = true; + outputThreadChecker.detachThread(); + outputThread = createOutputThread(); + outputThread.start(); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + encodeThreadChecker.checkIsOnValidThread(); + + final VideoCodecStatus returnValue; + if (outputThread == null) { + returnValue = VideoCodecStatus.OK; + } else { + // The outputThread actually stops and releases the codec once running is false. + running = false; + if (!ThreadUtils.joinUninterruptibly(outputThread, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { + Logging.e(TAG, "Media encoder release timeout"); + returnValue = VideoCodecStatus.TIMEOUT; + } else if (shutdownException != null) { + // Log the exception and turn it into an error. + Logging.e(TAG, "Media encoder release exception", shutdownException); + returnValue = VideoCodecStatus.ERROR; + } else { + returnValue = VideoCodecStatus.OK; + } + } + + textureDrawer.release(); + videoFrameDrawer.release(); + if (textureEglBase != null) { + textureEglBase.release(); + textureEglBase = null; + } + if (textureInputSurface != null) { + textureInputSurface.release(); + textureInputSurface = null; + } + outputBuilders.clear(); + + codec = null; + outputThread = null; + + // Allow changing thread after release. + encodeThreadChecker.detachThread(); + + return returnValue; + } + + @Override + public VideoCodecStatus encode(VideoFrame videoFrame, EncodeInfo encodeInfo) { + encodeThreadChecker.checkIsOnValidThread(); + if (codec == null) { + return VideoCodecStatus.UNINITIALIZED; + } + + final boolean isTextureBuffer = videoFrame.getBuffer() instanceof VideoFrame.TextureBuffer; + + // If input resolution changed, restart the codec with the new resolution. + final int frameWidth = videoFrame.getBuffer().getWidth(); + final int frameHeight = videoFrame.getBuffer().getHeight(); + final boolean shouldUseSurfaceMode = canUseSurface() && isTextureBuffer; + if (frameWidth != width || frameHeight != height || shouldUseSurfaceMode != useSurfaceMode) { + VideoCodecStatus status = resetCodec(frameWidth, frameHeight, shouldUseSurfaceMode); + if (status != VideoCodecStatus.OK) { + return status; + } + } + + if (outputBuilders.size() > MAX_ENCODER_Q_SIZE) { + // Too many frames in the encoder. Drop this frame. + Logging.e(TAG, "Dropped frame, encoder queue full"); + return VideoCodecStatus.NO_OUTPUT; // See webrtc bug 2887. + } + + boolean requestedKeyFrame = false; + for (EncodedImage.FrameType frameType : encodeInfo.frameTypes) { + if (frameType == EncodedImage.FrameType.VideoFrameKey) { + requestedKeyFrame = true; + } + } + + if (requestedKeyFrame || shouldForceKeyFrame(videoFrame.getTimestampNs())) { + requestKeyFrame(videoFrame.getTimestampNs()); + } + + EncodedImage.Builder builder = EncodedImage.builder() + .setCaptureTimeNs(videoFrame.getTimestampNs()) + .setEncodedWidth(videoFrame.getBuffer().getWidth()) + .setEncodedHeight(videoFrame.getBuffer().getHeight()) + .setRotation(videoFrame.getRotation()); + outputBuilders.offer(builder); + + long presentationTimestampUs = nextPresentationTimestampUs; + // Round frame duration down to avoid bitrate overshoot. + long frameDurationUs = + (long) (TimeUnit.SECONDS.toMicros(1) / bitrateAdjuster.getAdjustedFramerateFps()); + nextPresentationTimestampUs += frameDurationUs; + + final VideoCodecStatus returnValue; + if (useSurfaceMode) { + returnValue = encodeTextureBuffer(videoFrame, presentationTimestampUs); + } else { + returnValue = encodeByteBuffer(videoFrame, presentationTimestampUs); + } + + // Check if the queue was successful. + if (returnValue != VideoCodecStatus.OK) { + // Keep the output builders in sync with buffers in the codec. + outputBuilders.pollLast(); + } + + return returnValue; + } + + private VideoCodecStatus encodeTextureBuffer( + VideoFrame videoFrame, long presentationTimestampUs) { + encodeThreadChecker.checkIsOnValidThread(); + try { + // TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway, + // but it's a workaround for bug webrtc:5147. + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + // It is not necessary to release this frame because it doesn't own the buffer. + VideoFrame derotatedFrame = + new VideoFrame(videoFrame.getBuffer(), 0 /* rotation */, videoFrame.getTimestampNs()); + videoFrameDrawer.drawFrame(derotatedFrame, textureDrawer, null /* additionalRenderMatrix */); + textureEglBase.swapBuffers(TimeUnit.MICROSECONDS.toNanos(presentationTimestampUs)); + } catch (RuntimeException e) { + Logging.e(TAG, "encodeTexture failed", e); + return VideoCodecStatus.ERROR; + } + return VideoCodecStatus.OK; + } + + private VideoCodecStatus encodeByteBuffer(VideoFrame videoFrame, long presentationTimestampUs) { + encodeThreadChecker.checkIsOnValidThread(); + // No timeout. Don't block for an input buffer, drop frames if the encoder falls behind. + int index; + try { + index = codec.dequeueInputBuffer(0 /* timeout */); + } catch (IllegalStateException e) { + Logging.e(TAG, "dequeueInputBuffer failed", e); + return VideoCodecStatus.ERROR; + } + + if (index == -1) { + // Encoder is falling behind. No input buffers available. Drop the frame. + Logging.d(TAG, "Dropped frame, no input buffers available"); + return VideoCodecStatus.NO_OUTPUT; // See webrtc bug 2887. + } + + ByteBuffer buffer; + try { + buffer = codec.getInputBuffer(index); + } catch (IllegalStateException e) { + Logging.e(TAG, "getInputBuffer with index=" + index + " failed", e); + return VideoCodecStatus.ERROR; + } + + if (buffer.capacity() < frameSizeBytes) { + Logging.e(TAG, + "Input buffer size: " + buffer.capacity() + + " is smaller than frame size: " + frameSizeBytes); + return VideoCodecStatus.ERROR; + } + + fillInputBuffer(buffer, videoFrame.getBuffer()); + + try { + codec.queueInputBuffer( + index, 0 /* offset */, frameSizeBytes, presentationTimestampUs, 0 /* flags */); + } catch (IllegalStateException e) { + Logging.e(TAG, "queueInputBuffer failed", e); + // IllegalStateException thrown when the codec is in the wrong state. + return VideoCodecStatus.ERROR; + } + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRateAllocation(BitrateAllocation bitrateAllocation, int framerate) { + encodeThreadChecker.checkIsOnValidThread(); + if (framerate > MAX_VIDEO_FRAMERATE) { + framerate = MAX_VIDEO_FRAMERATE; + } + bitrateAdjuster.setTargets(bitrateAllocation.getSum(), framerate); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRates(RateControlParameters rcParameters) { + encodeThreadChecker.checkIsOnValidThread(); + bitrateAdjuster.setTargets(rcParameters.bitrate.getSum(), rcParameters.framerateFps); + return VideoCodecStatus.OK; + } + + @Override + public ScalingSettings getScalingSettings() { + if (automaticResizeOn) { + if (codecType == VideoCodecMimeType.VP8) { + final int kLowVp8QpThreshold = 29; + final int kHighVp8QpThreshold = 95; + return new ScalingSettings(kLowVp8QpThreshold, kHighVp8QpThreshold); + } else if (codecType == VideoCodecMimeType.H264) { + final int kLowH264QpThreshold = 24; + final int kHighH264QpThreshold = 37; + return new ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); + } + } + return ScalingSettings.OFF; + } + + @Override + public String getImplementationName() { + return codecName; + } + + @Override + public EncoderInfo getEncoderInfo() { + // Since our MediaCodec is guaranteed to encode 16-pixel-aligned frames only, we set alignment + // value to be 16. Additionally, this encoder produces a single stream. So it should not require + // alignment for all layers. + return new EncoderInfo( + /* requestedResolutionAlignment= */ REQUIRED_RESOLUTION_ALIGNMENT, + /* applyAlignmentToAllSimulcastLayers= */ false); + } + + private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseSurfaceMode) { + encodeThreadChecker.checkIsOnValidThread(); + VideoCodecStatus status = release(); + if (status != VideoCodecStatus.OK) { + return status; + } + width = newWidth; + height = newHeight; + useSurfaceMode = newUseSurfaceMode; + return initEncodeInternal(); + } + + private boolean shouldForceKeyFrame(long presentationTimestampNs) { + encodeThreadChecker.checkIsOnValidThread(); + return forcedKeyFrameNs > 0 && presentationTimestampNs > lastKeyFrameNs + forcedKeyFrameNs; + } + + private void requestKeyFrame(long presentationTimestampNs) { + encodeThreadChecker.checkIsOnValidThread(); + // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could + // indicate this in queueInputBuffer() below and guarantee _this_ frame + // be encoded as a key frame, but sadly that flag is ignored. Instead, + // we request a key frame "soon". + try { + Bundle b = new Bundle(); + b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + codec.setParameters(b); + } catch (IllegalStateException e) { + Logging.e(TAG, "requestKeyFrame failed", e); + return; + } + lastKeyFrameNs = presentationTimestampNs; + } + + private Thread createOutputThread() { + return new Thread() { + @Override + public void run() { + while (running) { + deliverEncodedImage(); + } + releaseCodecOnOutputThread(); + } + }; + } + + // Visible for testing. + protected void deliverEncodedImage() { + outputThreadChecker.checkIsOnValidThread(); + try { + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + int index = codec.dequeueOutputBuffer(info, DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US); + if (index < 0) { + if (index == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + outputBuffersBusyCount.waitForZero(); + } + return; + } + + ByteBuffer outputBuffer = codec.getOutputBuffer(index); + outputBuffer.position(info.offset); + outputBuffer.limit(info.offset + info.size); + + if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + Logging.d(TAG, "Config frame generated. Offset: " + info.offset + ". Size: " + info.size); + if (info.size > 0 + && (codecType == VideoCodecMimeType.H264 || codecType == VideoCodecMimeType.H265)) { + // In case of H264 and H265 config buffer contains SPS and PPS headers. Presence of these + // headers makes IDR frame a truly keyframe. Some encoders issue IDR frames without SPS + // and PPS. We save config buffer here to prepend it to all IDR frames encoder delivers. + configBuffer = ByteBuffer.allocateDirect(info.size); + configBuffer.put(outputBuffer); + } + return; + } + + bitrateAdjuster.reportEncodedFrame(info.size); + if (adjustedBitrate != bitrateAdjuster.getAdjustedBitrateBps()) { + updateBitrate(); + } + + final boolean isKeyFrame = (info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0; + if (isKeyFrame) { + Logging.d(TAG, "Sync frame generated"); + } + + // Extract QP before releasing output buffer. + Integer qp = null; + if (isEncodingStatisticsEnabled) { + MediaFormat format = codec.getOutputFormat(index); + if (format != null && format.containsKey(MediaFormat.KEY_VIDEO_QP_AVERAGE)) { + qp = format.getInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE); + } + } + + final ByteBuffer frameBuffer; + final Runnable releaseCallback; + if (isKeyFrame && configBuffer != null) { + Logging.d(TAG, + "Prepending config buffer of size " + configBuffer.capacity() + + " to output buffer with offset " + info.offset + ", size " + info.size); + frameBuffer = ByteBuffer.allocateDirect(info.size + configBuffer.capacity()); + configBuffer.rewind(); + frameBuffer.put(configBuffer); + frameBuffer.put(outputBuffer); + frameBuffer.rewind(); + codec.releaseOutputBuffer(index, /* render= */ false); + releaseCallback = null; + } else { + frameBuffer = outputBuffer.slice(); + outputBuffersBusyCount.increment(); + releaseCallback = () -> { + // This callback should not throw any exceptions since + // it may be called on an arbitrary thread. + // Check bug webrtc:11230 for more details. + try { + codec.releaseOutputBuffer(index, /* render= */ false); + } catch (Exception e) { + Logging.e(TAG, "releaseOutputBuffer failed", e); + } + outputBuffersBusyCount.decrement(); + }; + } + + final EncodedImage.FrameType frameType = isKeyFrame ? EncodedImage.FrameType.VideoFrameKey + : EncodedImage.FrameType.VideoFrameDelta; + + EncodedImage.Builder builder = outputBuilders.poll(); + builder.setBuffer(frameBuffer, releaseCallback); + builder.setFrameType(frameType); + builder.setQp(qp); + + EncodedImage encodedImage = builder.createEncodedImage(); + // TODO(mellem): Set codec-specific info. + callback.onEncodedFrame(encodedImage, new CodecSpecificInfo()); + // Note that the callback may have retained the image. + encodedImage.release(); + } catch (IllegalStateException e) { + Logging.e(TAG, "deliverOutput failed", e); + } + } + + private void releaseCodecOnOutputThread() { + outputThreadChecker.checkIsOnValidThread(); + Logging.d(TAG, "Releasing MediaCodec on output thread"); + outputBuffersBusyCount.waitForZero(); + try { + codec.stop(); + } catch (Exception e) { + Logging.e(TAG, "Media encoder stop failed", e); + } + try { + codec.release(); + } catch (Exception e) { + Logging.e(TAG, "Media encoder release failed", e); + // Propagate exceptions caught during release back to the main thread. + shutdownException = e; + } + configBuffer = null; + Logging.d(TAG, "Release on output thread done"); + } + + private VideoCodecStatus updateBitrate() { + outputThreadChecker.checkIsOnValidThread(); + adjustedBitrate = bitrateAdjuster.getAdjustedBitrateBps(); + try { + Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, adjustedBitrate); + codec.setParameters(params); + return VideoCodecStatus.OK; + } catch (IllegalStateException e) { + Logging.e(TAG, "updateBitrate failed", e); + return VideoCodecStatus.ERROR; + } + } + + private boolean canUseSurface() { + return sharedContext != null && surfaceColorFormat != null; + } + + /** Fetches stride and slice height from input media format */ + private void updateInputFormat(MediaFormat format) { + stride = width; + sliceHeight = height; + + if (format != null) { + if (format.containsKey(MediaFormat.KEY_STRIDE)) { + stride = format.getInteger(MediaFormat.KEY_STRIDE); + stride = Math.max(stride, width); + } + + if (format.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { + sliceHeight = format.getInteger(MediaFormat.KEY_SLICE_HEIGHT); + sliceHeight = Math.max(sliceHeight, height); + } + } + + isSemiPlanar = isSemiPlanar(yuvColorFormat); + if (isSemiPlanar) { + int chromaHeight = (height + 1) / 2; + frameSizeBytes = sliceHeight * stride + chromaHeight * stride; + } else { + int chromaStride = (stride + 1) / 2; + int chromaSliceHeight = (sliceHeight + 1) / 2; + frameSizeBytes = sliceHeight * stride + chromaSliceHeight * chromaStride * 2; + } + + Logging.d(TAG, + "updateInputFormat format: " + format + " stride: " + stride + + " sliceHeight: " + sliceHeight + " isSemiPlanar: " + isSemiPlanar + + " frameSizeBytes: " + frameSizeBytes); + } + + protected boolean isEncodingStatisticsSupported() { + // WebRTC quality scaler, which adjusts resolution and/or frame rate based on encoded QP, + // expects QP to be in native bitstream range for given codec. Native QP range for VP8 is + // [0, 127] and for VP9 is [0, 255]. MediaCodec VP8 and VP9 encoders (perhaps not all) + // return QP in range [0, 64], which is libvpx API specific range. Due to this mismatch we + // can't use QP feedback from these codecs. + if (codecType == VideoCodecMimeType.VP8 || codecType == VideoCodecMimeType.VP9) { + return false; + } + + MediaCodecInfo codecInfo = codec.getCodecInfo(); + if (codecInfo == null) { + return false; + } + + CodecCapabilities codecCaps = codecInfo.getCapabilitiesForType(codecType.mimeType()); + if (codecCaps == null) { + return false; + } + + return codecCaps.isFeatureSupported(CodecCapabilities.FEATURE_EncodingStatistics); + } + + // Visible for testing. + protected void fillInputBuffer(ByteBuffer buffer, VideoFrame.Buffer frame) { + VideoFrame.I420Buffer i420 = frame.toI420(); + if (isSemiPlanar) { + YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), + i420.getDataV(), i420.getStrideV(), buffer, i420.getWidth(), i420.getHeight(), stride, + sliceHeight); + } else { + YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), + i420.getDataV(), i420.getStrideV(), buffer, i420.getWidth(), i420.getHeight(), stride, + sliceHeight); + } + i420.release(); + } + + protected boolean isSemiPlanar(int colorFormat) { + switch (colorFormat) { + case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar: + return false; + case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar: + case MediaCodecInfo.CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar: + case MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: + return true; + default: + throw new IllegalArgumentException("Unsupported colorFormat: " + colorFormat); + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Histogram.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Histogram.java new file mode 100644 index 0000000000..c1d2d61a71 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/Histogram.java @@ -0,0 +1,39 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Class for holding the native pointer of a histogram. Since there is no way to destroy a + * histogram, please don't create unnecessary instances of this object. This class is thread safe. + * + * Usage example: + * private static final Histogram someMetricHistogram = + * Histogram.createCounts("WebRTC.Video.SomeMetric", 1, 10000, 50); + * someMetricHistogram.addSample(someVariable); + */ +class Histogram { + private final long handle; + + private Histogram(long handle) { + this.handle = handle; + } + + static public Histogram createCounts(String name, int min, int max, int bucketCount) { + return new Histogram(0); + } + + static public Histogram createEnumeration(String name, int max) { + return new Histogram(0); + } + + public void addSample(int sample) { + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JNILogging.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JNILogging.java new file mode 100644 index 0000000000..f391db61a1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JNILogging.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import org.webrtc.CalledByNative; +import org.webrtc.Loggable; +import org.webrtc.Logging.Severity; + +class JNILogging { + private final Loggable loggable; + + public JNILogging(Loggable loggable) { + this.loggable = loggable; + } + + @CalledByNative + public void logToInjectable(String message, Integer severity, String tag) { + loggable.onLogMessage(message, Severity.values()[severity], tag); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JniCommon.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JniCommon.java new file mode 100644 index 0000000000..e1b2e513d7 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/JniCommon.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** Class with static JNI helper functions that are used in many places. */ +public class JniCommon { + /** Functions to increment/decrement an rtc::RefCountInterface pointer. */ + public static native void nativeAddRef(long refCountedPointer); + public static native void nativeReleaseRef(long refCountedPointer); + + public static native ByteBuffer nativeAllocateByteBuffer(int size); + public static native void nativeFreeByteBuffer(ByteBuffer buffer); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecUtils.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecUtils.java new file mode 100644 index 0000000000..5417fec4d4 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecUtils.java @@ -0,0 +1,130 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.annotation.TargetApi; +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.os.Build; +import androidx.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; + +/** Container class for static constants and helpers used with MediaCodec. */ +// We are forced to use the old API because we want to support API level < 21. +@SuppressWarnings("deprecation") +class MediaCodecUtils { + private static final String TAG = "MediaCodecUtils"; + + // Prefixes for supported hardware encoder/decoder component names. + static final String EXYNOS_PREFIX = "OMX.Exynos."; + static final String INTEL_PREFIX = "OMX.Intel."; + static final String NVIDIA_PREFIX = "OMX.Nvidia."; + static final String QCOM_PREFIX = "OMX.qcom."; + static final String[] SOFTWARE_IMPLEMENTATION_PREFIXES = { + "OMX.google.", "OMX.SEC.", "c2.android"}; + + // NV12 color format supported by QCOM codec, but not declared in MediaCodec - + // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h + static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01; + static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02; + static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03; + static final int COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; + + // Color formats supported by hardware decoder - in order of preference. + static final int[] DECODER_COLOR_FORMATS = new int[] {CodecCapabilities.COLOR_FormatYUV420Planar, + CodecCapabilities.COLOR_FormatYUV420SemiPlanar, + CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, + MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka, + MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka, + MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka, + MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m}; + + // Color formats supported by hardware encoder - in order of preference. + static final int[] ENCODER_COLOR_FORMATS = { + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar, + MediaCodecInfo.CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, + MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m}; + + // Color formats supported by texture mode encoding - in order of preference. + static final int[] TEXTURE_COLOR_FORMATS = + new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface}; + + static @Nullable Integer selectColorFormat( + int[] supportedColorFormats, CodecCapabilities capabilities) { + for (int supportedColorFormat : supportedColorFormats) { + for (int codecColorFormat : capabilities.colorFormats) { + if (codecColorFormat == supportedColorFormat) { + return codecColorFormat; + } + } + } + return null; + } + + static boolean codecSupportsType(MediaCodecInfo info, VideoCodecMimeType type) { + for (String mimeType : info.getSupportedTypes()) { + if (type.mimeType().equals(mimeType)) { + return true; + } + } + return false; + } + + static Map<String, String> getCodecProperties(VideoCodecMimeType type, boolean highProfile) { + switch (type) { + case VP8: + case VP9: + case AV1: + case H265: + return new HashMap<String, String>(); + case H264: + return H264Utils.getDefaultH264Params(highProfile); + default: + throw new IllegalArgumentException("Unsupported codec: " + type); + } + } + + static boolean isHardwareAccelerated(MediaCodecInfo info) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return isHardwareAcceleratedQOrHigher(info); + } + return !isSoftwareOnly(info); + } + + @TargetApi(29) + private static boolean isHardwareAcceleratedQOrHigher(android.media.MediaCodecInfo codecInfo) { + return codecInfo.isHardwareAccelerated(); + } + + static boolean isSoftwareOnly(android.media.MediaCodecInfo codecInfo) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return isSoftwareOnlyQOrHigher(codecInfo); + } + String name = codecInfo.getName(); + for (String prefix : SOFTWARE_IMPLEMENTATION_PREFIXES) { + if (name.startsWith(prefix)) { + return true; + } + } + return false; + } + + @TargetApi(29) + private static boolean isSoftwareOnlyQOrHigher(android.media.MediaCodecInfo codecInfo) { + return codecInfo.isSoftwareOnly(); + } + + private MediaCodecUtils() { + // This class should not be instantiated. + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java new file mode 100644 index 0000000000..9a73bc49ff --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java @@ -0,0 +1,140 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import static org.webrtc.MediaCodecUtils.EXYNOS_PREFIX; +import static org.webrtc.MediaCodecUtils.QCOM_PREFIX; + +import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCodecList; +import android.os.Build; +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +/** Factory for decoders backed by Android MediaCodec API. */ +@SuppressWarnings("deprecation") // API level 16 requires use of deprecated methods. +class MediaCodecVideoDecoderFactory implements VideoDecoderFactory { + private static final String TAG = "MediaCodecVideoDecoderFactory"; + + private final @Nullable EglBase.Context sharedContext; + private final @Nullable Predicate<MediaCodecInfo> codecAllowedPredicate; + + /** + * MediaCodecVideoDecoderFactory with support of codecs filtering. + * + * @param sharedContext The textures generated will be accessible from this context. May be null, + * this disables texture support. + * @param codecAllowedPredicate optional predicate to test if codec allowed. All codecs are + * allowed when predicate is not provided. + */ + public MediaCodecVideoDecoderFactory(@Nullable EglBase.Context sharedContext, + @Nullable Predicate<MediaCodecInfo> codecAllowedPredicate) { + this.sharedContext = sharedContext; + this.codecAllowedPredicate = codecAllowedPredicate; + } + + @Nullable + @Override + public VideoDecoder createDecoder(VideoCodecInfo codecType) { + VideoCodecMimeType type = VideoCodecMimeType.valueOf(codecType.getName()); + MediaCodecInfo info = findCodecForType(type); + + if (info == null) { + return null; + } + + CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType()); + return new AndroidVideoDecoder(new MediaCodecWrapperFactoryImpl(), info.getName(), type, + MediaCodecUtils.selectColorFormat(MediaCodecUtils.DECODER_COLOR_FORMATS, capabilities), + sharedContext); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List<VideoCodecInfo> supportedCodecInfos = new ArrayList<VideoCodecInfo>(); + // Generate a list of supported codecs in order of preference: + // VP8, VP9, H264 (high profile), H264 (baseline profile), AV1 and H265. + for (VideoCodecMimeType type : + new VideoCodecMimeType[] {VideoCodecMimeType.VP8, VideoCodecMimeType.VP9, + VideoCodecMimeType.H264, VideoCodecMimeType.AV1, VideoCodecMimeType.H265}) { + MediaCodecInfo codec = findCodecForType(type); + if (codec != null) { + String name = type.name(); + if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) { + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + } + + supportedCodecInfos.add(new VideoCodecInfo( + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + } + } + + return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } + + private @Nullable MediaCodecInfo findCodecForType(VideoCodecMimeType type) { + for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { + MediaCodecInfo info = null; + try { + info = MediaCodecList.getCodecInfoAt(i); + } catch (IllegalArgumentException e) { + Logging.e(TAG, "Cannot retrieve decoder codec info", e); + } + + if (info == null || info.isEncoder()) { + continue; + } + + if (isSupportedCodec(info, type)) { + return info; + } + } + + return null; // No support for this type. + } + + // Returns true if the given MediaCodecInfo indicates a supported encoder for the given type. + private boolean isSupportedCodec(MediaCodecInfo info, VideoCodecMimeType type) { + if (!MediaCodecUtils.codecSupportsType(info, type)) { + return false; + } + // Check for a supported color format. + if (MediaCodecUtils.selectColorFormat( + MediaCodecUtils.DECODER_COLOR_FORMATS, info.getCapabilitiesForType(type.mimeType())) + == null) { + return false; + } + return isCodecAllowed(info); + } + + private boolean isCodecAllowed(MediaCodecInfo info) { + if (codecAllowedPredicate == null) { + return true; + } + return codecAllowedPredicate.test(info); + } + + private boolean isH264HighProfileSupported(MediaCodecInfo info) { + String name = info.getName(); + // Support H.264 HP decoding on QCOM chips. + if (name.startsWith(QCOM_PREFIX)) { + return true; + } + // Support H.264 HP decoding on Exynos chips for Android M and above. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && name.startsWith(EXYNOS_PREFIX)) { + return true; + } + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java new file mode 100644 index 0000000000..11e0f58dfe --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.Bundle; +import android.view.Surface; +import java.nio.ByteBuffer; + +/** + * Subset of methods defined in {@link android.media.MediaCodec} needed by + * {@link HardwareVideoEncoder} and {@link AndroidVideoDecoder}. This interface + * exists to allow mocking and using a fake implementation in tests. + */ +interface MediaCodecWrapper { + void configure(MediaFormat format, Surface surface, MediaCrypto crypto, int flags); + + void start(); + + void flush(); + + void stop(); + + void release(); + + int dequeueInputBuffer(long timeoutUs); + + void queueInputBuffer(int index, int offset, int size, long presentationTimeUs, int flags); + + int dequeueOutputBuffer(MediaCodec.BufferInfo info, long timeoutUs); + + void releaseOutputBuffer(int index, boolean render); + + MediaFormat getInputFormat(); + + MediaFormat getOutputFormat(); + + MediaFormat getOutputFormat(int index); + + ByteBuffer getInputBuffer(int index); + + ByteBuffer getOutputBuffer(int index); + + Surface createInputSurface(); + + void setParameters(Bundle params); + + MediaCodecInfo getCodecInfo(); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java new file mode 100644 index 0000000000..2962cb62a7 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.io.IOException; + +interface MediaCodecWrapperFactory { + /** + * Creates a new {@link MediaCodecWrapper} by codec name. + * + * <p>For additional information see {@link android.media.MediaCodec#createByCodecName}. + */ + MediaCodecWrapper createByCodecName(String name) throws IOException; +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java new file mode 100644 index 0000000000..207492f31c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java @@ -0,0 +1,126 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodecInfo; +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.Bundle; +import android.view.Surface; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Implementation of MediaCodecWrapperFactory that returns MediaCodecInterfaces wrapping + * {@link android.media.MediaCodec} objects. + */ +class MediaCodecWrapperFactoryImpl implements MediaCodecWrapperFactory { + private static class MediaCodecWrapperImpl implements MediaCodecWrapper { + private final MediaCodec mediaCodec; + + public MediaCodecWrapperImpl(MediaCodec mediaCodec) { + this.mediaCodec = mediaCodec; + } + + @Override + public void configure(MediaFormat format, Surface surface, MediaCrypto crypto, int flags) { + mediaCodec.configure(format, surface, crypto, flags); + } + + @Override + public void start() { + mediaCodec.start(); + } + + @Override + public void flush() { + mediaCodec.flush(); + } + + @Override + public void stop() { + mediaCodec.stop(); + } + + @Override + public void release() { + mediaCodec.release(); + } + + @Override + public int dequeueInputBuffer(long timeoutUs) { + return mediaCodec.dequeueInputBuffer(timeoutUs); + } + + @Override + public void queueInputBuffer( + int index, int offset, int size, long presentationTimeUs, int flags) { + mediaCodec.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + + @Override + public int dequeueOutputBuffer(BufferInfo info, long timeoutUs) { + return mediaCodec.dequeueOutputBuffer(info, timeoutUs); + } + + @Override + public void releaseOutputBuffer(int index, boolean render) { + mediaCodec.releaseOutputBuffer(index, render); + } + + @Override + public MediaFormat getInputFormat() { + return mediaCodec.getInputFormat(); + } + + @Override + public MediaFormat getOutputFormat() { + return mediaCodec.getOutputFormat(); + } + + @Override + public MediaFormat getOutputFormat(int index) { + return mediaCodec.getOutputFormat(index); + } + + @Override + public ByteBuffer getInputBuffer(int index) { + return mediaCodec.getInputBuffer(index); + } + + @Override + public ByteBuffer getOutputBuffer(int index) { + return mediaCodec.getOutputBuffer(index); + } + + @Override + public Surface createInputSurface() { + return mediaCodec.createInputSurface(); + } + + @Override + public void setParameters(Bundle params) { + mediaCodec.setParameters(params); + } + + @Override + public MediaCodecInfo getCodecInfo() { + return mediaCodec.getCodecInfo(); + } + } + + @Override + public MediaCodecWrapper createByCodecName(String name) throws IOException { + return new MediaCodecWrapperImpl(MediaCodec.createByCodecName(name)); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV12Buffer.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV12Buffer.java new file mode 100644 index 0000000000..fe0221d826 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV12Buffer.java @@ -0,0 +1,73 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; + +public class NV12Buffer implements VideoFrame.Buffer { + private final int width; + private final int height; + private final int stride; + private final int sliceHeight; + private final ByteBuffer buffer; + private final RefCountDelegate refCountDelegate; + + public NV12Buffer(int width, int height, int stride, int sliceHeight, ByteBuffer buffer, + @Nullable Runnable releaseCallback) { + this.width = width; + this.height = height; + this.stride = stride; + this.sliceHeight = sliceHeight; + this.buffer = buffer; + this.refCountDelegate = new RefCountDelegate(releaseCallback); + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public VideoFrame.I420Buffer toI420() { + return (VideoFrame.I420Buffer) cropAndScale(0, 0, width, height, width, height); + } + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + JavaI420Buffer newBuffer = JavaI420Buffer.allocate(scaleWidth, scaleHeight); + nativeCropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, buffer, width, + height, stride, sliceHeight, newBuffer.getDataY(), newBuffer.getStrideY(), + newBuffer.getDataU(), newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV()); + return newBuffer; + } + + private static native void nativeCropAndScale(int cropX, int cropY, int cropWidth, int cropHeight, + int scaleWidth, int scaleHeight, ByteBuffer src, int srcWidth, int srcHeight, int srcStride, + int srcSliceHeight, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU, int dstStrideU, + ByteBuffer dstV, int dstStrideV); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV21Buffer.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV21Buffer.java new file mode 100644 index 0000000000..0fb1afe74b --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NV21Buffer.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; + +public class NV21Buffer implements VideoFrame.Buffer { + private final byte[] data; + private final int width; + private final int height; + private final RefCountDelegate refCountDelegate; + + public NV21Buffer(byte[] data, int width, int height, @Nullable Runnable releaseCallback) { + this.data = data; + this.width = width; + this.height = height; + this.refCountDelegate = new RefCountDelegate(releaseCallback); + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public VideoFrame.I420Buffer toI420() { + // Cropping converts the frame to I420. Just crop and scale to the whole image. + return (VideoFrame.I420Buffer) cropAndScale(0 /* cropX */, 0 /* cropY */, width /* cropWidth */, + height /* cropHeight */, width /* scaleWidth */, height /* scaleHeight */); + } + + @Override + public void retain() { + refCountDelegate.retain(); + } + + @Override + public void release() { + refCountDelegate.release(); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + JavaI420Buffer newBuffer = JavaI420Buffer.allocate(scaleWidth, scaleHeight); + nativeCropAndScale(cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, data, width, + height, newBuffer.getDataY(), newBuffer.getStrideY(), newBuffer.getDataU(), + newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV()); + return newBuffer; + } + + private static native void nativeCropAndScale(int cropX, int cropY, int cropWidth, int cropHeight, + int scaleWidth, int scaleHeight, byte[] src, int srcWidth, int srcHeight, ByteBuffer dstY, + int dstStrideY, ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java new file mode 100644 index 0000000000..d4fba481e8 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; + +/** + * This class is meant to be a simple layer that only handles the JNI wrapping of a C++ + * AndroidVideoTrackSource, that can easily be mocked out in Java unit tests. Refrain from adding + * any unnecessary logic to this class. + * This class is thred safe and methods can be called from any thread, but if frames A, B, ..., are + * sent to adaptFrame(), the adapted frames adaptedA, adaptedB, ..., needs to be passed in the same + * order to onFrameCaptured(). + */ +class NativeAndroidVideoTrackSource { + // Pointer to webrtc::jni::AndroidVideoTrackSource. + private final long nativeAndroidVideoTrackSource; + + public NativeAndroidVideoTrackSource(long nativeAndroidVideoTrackSource) { + this.nativeAndroidVideoTrackSource = nativeAndroidVideoTrackSource; + } + + /** + * Set the state for the native MediaSourceInterface. Maps boolean to either + * SourceState::kLive or SourceState::kEnded. + */ + public void setState(boolean isLive) { + nativeSetState(nativeAndroidVideoTrackSource, isLive); + } + + /** + * This function should be called before delivering any frame to determine if the frame should be + * dropped or what the cropping and scaling parameters should be. If the return value is null, the + * frame should be dropped, otherwise the frame should be adapted in accordance to the frame + * adaptation parameters before calling onFrameCaptured(). + */ + @Nullable + public VideoProcessor.FrameAdaptationParameters adaptFrame(VideoFrame frame) { + return nativeAdaptFrame(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs()); + } + + /** + * Pass an adapted frame to the native AndroidVideoTrackSource. Note that adaptFrame() is + * expected to be called first and that the passed frame conforms to those parameters. + */ + public void onFrameCaptured(VideoFrame frame) { + nativeOnFrameCaptured(nativeAndroidVideoTrackSource, frame.getRotation(), + frame.getTimestampNs(), frame.getBuffer()); + } + + /** + * Calling this function will cause frames to be scaled down to the requested resolution. Also, + * frames will be cropped to match the requested aspect ratio, and frames will be dropped to match + * the requested fps. + */ + public void adaptOutputFormat(VideoSource.AspectRatio targetLandscapeAspectRatio, + @Nullable Integer maxLandscapePixelCount, VideoSource.AspectRatio targetPortraitAspectRatio, + @Nullable Integer maxPortraitPixelCount, @Nullable Integer maxFps) { + nativeAdaptOutputFormat(nativeAndroidVideoTrackSource, targetLandscapeAspectRatio.width, + targetLandscapeAspectRatio.height, maxLandscapePixelCount, targetPortraitAspectRatio.width, + targetPortraitAspectRatio.height, maxPortraitPixelCount, maxFps); + } + + public void setIsScreencast(boolean isScreencast) { + nativeSetIsScreencast(nativeAndroidVideoTrackSource, isScreencast); + } + + @CalledByNative + static VideoProcessor.FrameAdaptationParameters createFrameAdaptationParameters(int cropX, + int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight, long timestampNs, + boolean drop) { + return new VideoProcessor.FrameAdaptationParameters( + cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, timestampNs, drop); + } + + private static native void nativeSetIsScreencast( + long nativeAndroidVideoTrackSource, boolean isScreencast); + private static native void nativeSetState(long nativeAndroidVideoTrackSource, boolean isLive); + private static native void nativeAdaptOutputFormat(long nativeAndroidVideoTrackSource, + int landscapeWidth, int landscapeHeight, @Nullable Integer maxLandscapePixelCount, + int portraitWidth, int portraitHeight, @Nullable Integer maxPortraitPixelCount, + @Nullable Integer maxFps); + @Nullable + private static native VideoProcessor.FrameAdaptationParameters nativeAdaptFrame( + long nativeAndroidVideoTrackSource, int width, int height, int rotation, long timestampNs); + private static native void nativeOnFrameCaptured( + long nativeAndroidVideoTrackSource, int rotation, long timestampNs, VideoFrame.Buffer buffer); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java new file mode 100644 index 0000000000..c195fb3a4c --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeCapturerObserver.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import org.webrtc.VideoFrame; + +/** + * Used from native api and implements a simple VideoCapturer.CapturerObserver that feeds frames to + * a webrtc::jni::AndroidVideoTrackSource. + */ +class NativeCapturerObserver implements CapturerObserver { + private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource; + + @CalledByNative + public NativeCapturerObserver(long nativeSource) { + this.nativeAndroidVideoTrackSource = new NativeAndroidVideoTrackSource(nativeSource); + } + + @Override + public void onCapturerStarted(boolean success) { + nativeAndroidVideoTrackSource.setState(success); + } + + @Override + public void onCapturerStopped() { + nativeAndroidVideoTrackSource.setState(/* isLive= */ false); + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + final VideoProcessor.FrameAdaptationParameters parameters = + nativeAndroidVideoTrackSource.adaptFrame(frame); + if (parameters == null) { + // Drop frame. + return; + } + + final VideoFrame.Buffer adaptedBuffer = + frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth, + parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight); + nativeAndroidVideoTrackSource.onFrameCaptured( + new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs)); + adaptedBuffer.release(); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeLibrary.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeLibrary.java new file mode 100644 index 0000000000..531c216302 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/NativeLibrary.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +class NativeLibrary { + private static String TAG = "NativeLibrary"; + + static class DefaultLoader implements NativeLibraryLoader { + @Override + public boolean load(String name) { + Logging.d(TAG, "Loading library: " + name); + System.loadLibrary(name); + + // Not relevant, but kept for API compatibility. + return true; + } + } + + private static Object lock = new Object(); + private static boolean libraryLoaded; + + /** + * Loads the native library. Clients should call PeerConnectionFactory.initialize. It will call + * this method for them. + */ + static void initialize(NativeLibraryLoader loader, String libraryName) { + synchronized (lock) { + if (libraryLoaded) { + Logging.d(TAG, "Native library has already been loaded."); + return; + } + Logging.d(TAG, "Loading native library: " + libraryName); + libraryLoaded = loader.load(libraryName); + } + } + + /** Returns true if the library has been loaded successfully. */ + static boolean isLoaded() { + synchronized (lock) { + return libraryLoaded; + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/RefCountDelegate.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/RefCountDelegate.java new file mode 100644 index 0000000000..b9210d26a4 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/RefCountDelegate.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Implementation of RefCounted that executes a Runnable once the ref count reaches zero. + */ +class RefCountDelegate implements RefCounted { + private final AtomicInteger refCount = new AtomicInteger(1); + private final @Nullable Runnable releaseCallback; + + /** + * @param releaseCallback Callback that will be executed once the ref count reaches zero. + */ + public RefCountDelegate(@Nullable Runnable releaseCallback) { + this.releaseCallback = releaseCallback; + } + + @Override + public void retain() { + int updated_count = refCount.incrementAndGet(); + if (updated_count < 2) { + throw new IllegalStateException("retain() called on an object with refcount < 1"); + } + } + + @Override + public void release() { + int updated_count = refCount.decrementAndGet(); + if (updated_count < 0) { + throw new IllegalStateException("release() called on an object with refcount < 1"); + } + if (updated_count == 0 && releaseCallback != null) { + releaseCallback.run(); + } + } + + /** + * Tries to retain the object. Can be used in scenarios where it is unknown if the object has + * already been released. Returns true if successful or false if the object was already released. + */ + boolean safeRetain() { + int currentRefCount = refCount.get(); + while (currentRefCount != 0) { + if (refCount.weakCompareAndSet(currentRefCount, currentRefCount + 1)) { + return true; + } + currentRefCount = refCount.get(); + } + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoCodecMimeType.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoCodecMimeType.java new file mode 100644 index 0000000000..5538f3201a --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoCodecMimeType.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** Enumeration of supported video codec types. */ +enum VideoCodecMimeType { + VP8("video/x-vnd.on2.vp8"), + VP9("video/x-vnd.on2.vp9"), + H264("video/avc"), + AV1("video/av01"), + H265("video/hevc"); + + private final String mimeType; + + private VideoCodecMimeType(String mimeType) { + this.mimeType = mimeType; + } + + String mimeType() { + return mimeType; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java new file mode 100644 index 0000000000..2aae041640 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import org.webrtc.VideoDecoder; + +/** + * This class contains the Java glue code for JNI generation of VideoDecoder. + */ +class VideoDecoderWrapper { + @CalledByNative + static VideoDecoder.Callback createDecoderCallback(final long nativeDecoder) { + return (VideoFrame frame, Integer decodeTimeMs, + Integer qp) -> nativeOnDecodedFrame(nativeDecoder, frame, decodeTimeMs, qp); + } + + private static native void nativeOnDecodedFrame( + long nativeVideoDecoderWrapper, VideoFrame frame, Integer decodeTimeMs, Integer qp); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java new file mode 100644 index 0000000000..b5485d4edb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java @@ -0,0 +1,46 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +// Explicit imports necessary for JNI generation. +import androidx.annotation.Nullable; +import org.webrtc.VideoEncoder; + +/** + * This class contains the Java glue code for JNI generation of VideoEncoder. + */ +class VideoEncoderWrapper { + @CalledByNative + static boolean getScalingSettingsOn(VideoEncoder.ScalingSettings scalingSettings) { + return scalingSettings.on; + } + + @Nullable + @CalledByNative + static Integer getScalingSettingsLow(VideoEncoder.ScalingSettings scalingSettings) { + return scalingSettings.low; + } + + @Nullable + @CalledByNative + static Integer getScalingSettingsHigh(VideoEncoder.ScalingSettings scalingSettings) { + return scalingSettings.high; + } + + @CalledByNative + static VideoEncoder.Callback createEncoderCallback(final long nativeEncoder) { + return (EncodedImage frame, + VideoEncoder.CodecSpecificInfo info) -> nativeOnEncodedFrame(nativeEncoder, frame); + } + + private static native void nativeOnEncodedFrame( + long nativeVideoEncoderWrapper, EncodedImage frame); +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WebRtcClassLoader.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WebRtcClassLoader.java new file mode 100644 index 0000000000..023e92cfb1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WebRtcClassLoader.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * This class provides a ClassLoader that is capable of loading WebRTC Java classes regardless of + * what thread it's called from. Such a ClassLoader is needed for the few cases where the JNI + * mechanism is unable to automatically determine the appropriate ClassLoader instance. + */ +class WebRtcClassLoader { + @CalledByNative + static Object getClassLoader() { + Object loader = WebRtcClassLoader.class.getClassLoader(); + if (loader == null) { + throw new RuntimeException("Failed to get WebRTC class loader."); + } + return loader; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java new file mode 100644 index 0000000000..0461660fcf --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java @@ -0,0 +1,110 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** + * This class wraps a webrtc::I420BufferInterface into a VideoFrame.I420Buffer. + */ +class WrappedNativeI420Buffer implements VideoFrame.I420Buffer { + private final int width; + private final int height; + private final ByteBuffer dataY; + private final int strideY; + private final ByteBuffer dataU; + private final int strideU; + private final ByteBuffer dataV; + private final int strideV; + private final long nativeBuffer; + + @CalledByNative + WrappedNativeI420Buffer(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU, + int strideU, ByteBuffer dataV, int strideV, long nativeBuffer) { + this.width = width; + this.height = height; + this.dataY = dataY; + this.strideY = strideY; + this.dataU = dataU; + this.strideU = strideU; + this.dataV = dataV; + this.strideV = strideV; + this.nativeBuffer = nativeBuffer; + + retain(); + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public ByteBuffer getDataY() { + // Return a slice to prevent relative reads from changing the position. + return dataY.slice(); + } + + @Override + public ByteBuffer getDataU() { + // Return a slice to prevent relative reads from changing the position. + return dataU.slice(); + } + + @Override + public ByteBuffer getDataV() { + // Return a slice to prevent relative reads from changing the position. + return dataV.slice(); + } + + @Override + public int getStrideY() { + return strideY; + } + + @Override + public int getStrideU() { + return strideU; + } + + @Override + public int getStrideV() { + return strideV; + } + + @Override + public VideoFrame.I420Buffer toI420() { + retain(); + return this; + } + + @Override + public void retain() { + JniCommon.nativeAddRef(nativeBuffer); + } + + @Override + public void release() { + JniCommon.nativeReleaseRef(nativeBuffer); + } + + @Override + public VideoFrame.Buffer cropAndScale( + int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) { + return JavaI420Buffer.cropAndScaleI420( + this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java new file mode 100644 index 0000000000..70c625ab4f --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.media.AudioTrack; +import android.os.Build; +import org.webrtc.Logging; + +// Lowers the buffer size if no underruns are detected for 100 ms. Once an +// underrun is detected, the buffer size is increased by 10 ms and it will not +// be lowered further. The buffer size will never be increased more than +// 5 times, to avoid the possibility of the buffer size increasing without +// bounds. +class LowLatencyAudioBufferManager { + private static final String TAG = "LowLatencyAudioBufferManager"; + // The underrun count that was valid during the previous call to maybeAdjustBufferSize(). Used to + // detect increases in the value. + private int prevUnderrunCount; + // The number of ticks to wait without an underrun before decreasing the buffer size. + private int ticksUntilNextDecrease; + // Indicate if we should continue to decrease the buffer size. + private boolean keepLoweringBufferSize; + // How often the buffer size was increased. + private int bufferIncreaseCounter; + + public LowLatencyAudioBufferManager() { + this.prevUnderrunCount = 0; + this.ticksUntilNextDecrease = 10; + this.keepLoweringBufferSize = true; + this.bufferIncreaseCounter = 0; + } + + public void maybeAdjustBufferSize(AudioTrack audioTrack) { + if (Build.VERSION.SDK_INT >= 26) { + final int underrunCount = audioTrack.getUnderrunCount(); + if (underrunCount > prevUnderrunCount) { + // Don't increase buffer more than 5 times. Continuing to increase the buffer size + // could be harmful on low-power devices that regularly experience underruns under + // normal conditions. + if (bufferIncreaseCounter < 5) { + // Underrun detected, increase buffer size by 10ms. + final int currentBufferSize = audioTrack.getBufferSizeInFrames(); + final int newBufferSize = currentBufferSize + audioTrack.getPlaybackRate() / 100; + Logging.d(TAG, + "Underrun detected! Increasing AudioTrack buffer size from " + currentBufferSize + + " to " + newBufferSize); + audioTrack.setBufferSizeInFrames(newBufferSize); + bufferIncreaseCounter++; + } + // Stop trying to lower the buffer size. + keepLoweringBufferSize = false; + prevUnderrunCount = underrunCount; + ticksUntilNextDecrease = 10; + } else if (keepLoweringBufferSize) { + ticksUntilNextDecrease--; + if (ticksUntilNextDecrease <= 0) { + // No underrun seen for 100 ms, try to lower the buffer size by 10ms. + final int bufferSize10ms = audioTrack.getPlaybackRate() / 100; + // Never go below a buffer size of 10ms. + final int currentBufferSize = audioTrack.getBufferSizeInFrames(); + final int newBufferSize = Math.max(bufferSize10ms, currentBufferSize - bufferSize10ms); + if (newBufferSize != currentBufferSize) { + Logging.d(TAG, + "Lowering AudioTrack buffer size from " + currentBufferSize + " to " + + newBufferSize); + audioTrack.setBufferSizeInFrames(newBufferSize); + } + ticksUntilNextDecrease = 10; + } + } + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/VolumeLogger.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/VolumeLogger.java new file mode 100644 index 0000000000..06d5cd3a8e --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/VolumeLogger.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.media.AudioManager; +import androidx.annotation.Nullable; +import java.util.Timer; +import java.util.TimerTask; +import org.webrtc.Logging; + +// TODO(magjed): Do we really need to spawn a new thread just to log volume? Can we re-use the +// AudioTrackThread instead? +/** + * Private utility class that periodically checks and logs the volume level of the audio stream that + * is currently controlled by the volume control. A timer triggers logs once every 30 seconds and + * the timer's associated thread is named "WebRtcVolumeLevelLoggerThread". + */ +class VolumeLogger { + private static final String TAG = "VolumeLogger"; + private static final String THREAD_NAME = "WebRtcVolumeLevelLoggerThread"; + private static final int TIMER_PERIOD_IN_SECONDS = 30; + + private final AudioManager audioManager; + private @Nullable Timer timer; + + public VolumeLogger(AudioManager audioManager) { + this.audioManager = audioManager; + } + + public void start() { + Logging.d(TAG, "start" + WebRtcAudioUtils.getThreadInfo()); + if (timer != null) { + return; + } + Logging.d(TAG, "audio mode is: " + WebRtcAudioUtils.modeToString(audioManager.getMode())); + + timer = new Timer(THREAD_NAME); + timer.schedule(new LogVolumeTask(audioManager.getStreamMaxVolume(AudioManager.STREAM_RING), + audioManager.getStreamMaxVolume(AudioManager.STREAM_VOICE_CALL)), + 0, TIMER_PERIOD_IN_SECONDS * 1000); + } + + private class LogVolumeTask extends TimerTask { + private final int maxRingVolume; + private final int maxVoiceCallVolume; + + LogVolumeTask(int maxRingVolume, int maxVoiceCallVolume) { + this.maxRingVolume = maxRingVolume; + this.maxVoiceCallVolume = maxVoiceCallVolume; + } + + @Override + public void run() { + final int mode = audioManager.getMode(); + if (mode == AudioManager.MODE_RINGTONE) { + Logging.d(TAG, + "STREAM_RING stream volume: " + audioManager.getStreamVolume(AudioManager.STREAM_RING) + + " (max=" + maxRingVolume + ")"); + } else if (mode == AudioManager.MODE_IN_COMMUNICATION) { + Logging.d(TAG, + "VOICE_CALL stream volume: " + + audioManager.getStreamVolume(AudioManager.STREAM_VOICE_CALL) + + " (max=" + maxVoiceCallVolume + ")"); + } + } + } + + public void stop() { + Logging.d(TAG, "stop" + WebRtcAudioUtils.getThreadInfo()); + if (timer != null) { + timer.cancel(); + timer = null; + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java new file mode 100644 index 0000000000..1e80e485fb --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.media.audiofx.AcousticEchoCanceler; +import android.media.audiofx.AudioEffect; +import android.media.audiofx.AudioEffect.Descriptor; +import android.media.audiofx.NoiseSuppressor; +import android.os.Build; +import androidx.annotation.Nullable; +import java.util.UUID; +import org.webrtc.Logging; + +// This class wraps control of three different platform effects. Supported +// effects are: AcousticEchoCanceler (AEC) and NoiseSuppressor (NS). +// Calling enable() will active all effects that are +// supported by the device if the corresponding `shouldEnableXXX` member is set. +class WebRtcAudioEffects { + private static final boolean DEBUG = false; + + private static final String TAG = "WebRtcAudioEffectsExternal"; + + // UUIDs for Software Audio Effects that we want to avoid using. + // The implementor field will be set to "The Android Open Source Project". + private static final UUID AOSP_ACOUSTIC_ECHO_CANCELER = + UUID.fromString("bb392ec0-8d4d-11e0-a896-0002a5d5c51b"); + private static final UUID AOSP_NOISE_SUPPRESSOR = + UUID.fromString("c06c8400-8e06-11e0-9cb6-0002a5d5c51b"); + + // Contains the available effect descriptors returned from the + // AudioEffect.getEffects() call. This result is cached to avoid doing the + // slow OS call multiple times. + private static @Nullable Descriptor[] cachedEffects; + + // Contains the audio effect objects. Created in enable() and destroyed + // in release(). + private @Nullable AcousticEchoCanceler aec; + private @Nullable NoiseSuppressor ns; + + // Affects the final state given to the setEnabled() method on each effect. + // The default state is set to "disabled" but each effect can also be enabled + // by calling setAEC() and setNS(). + private boolean shouldEnableAec; + private boolean shouldEnableNs; + + // Returns true if all conditions for supporting HW Acoustic Echo Cancellation (AEC) are + // fulfilled. + public static boolean isAcousticEchoCancelerSupported() { + return isEffectTypeAvailable(AudioEffect.EFFECT_TYPE_AEC, AOSP_ACOUSTIC_ECHO_CANCELER); + } + + // Returns true if all conditions for supporting HW Noise Suppression (NS) are fulfilled. + public static boolean isNoiseSuppressorSupported() { + return isEffectTypeAvailable(AudioEffect.EFFECT_TYPE_NS, AOSP_NOISE_SUPPRESSOR); + } + + public WebRtcAudioEffects() { + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + } + + // Call this method to enable or disable the platform AEC. It modifies + // `shouldEnableAec` which is used in enable() where the actual state + // of the AEC effect is modified. Returns true if HW AEC is supported and + // false otherwise. + public boolean setAEC(boolean enable) { + Logging.d(TAG, "setAEC(" + enable + ")"); + if (!isAcousticEchoCancelerSupported()) { + Logging.w(TAG, "Platform AEC is not supported"); + shouldEnableAec = false; + return false; + } + if (aec != null && (enable != shouldEnableAec)) { + Logging.e(TAG, "Platform AEC state can't be modified while recording"); + return false; + } + shouldEnableAec = enable; + return true; + } + + // Call this method to enable or disable the platform NS. It modifies + // `shouldEnableNs` which is used in enable() where the actual state + // of the NS effect is modified. Returns true if HW NS is supported and + // false otherwise. + public boolean setNS(boolean enable) { + Logging.d(TAG, "setNS(" + enable + ")"); + if (!isNoiseSuppressorSupported()) { + Logging.w(TAG, "Platform NS is not supported"); + shouldEnableNs = false; + return false; + } + if (ns != null && (enable != shouldEnableNs)) { + Logging.e(TAG, "Platform NS state can't be modified while recording"); + return false; + } + shouldEnableNs = enable; + return true; + } + + // Toggles an existing NoiseSuppressor to be enabled or disabled. + // Returns true if the toggling was successful, otherwise false is returned (this is also the case + // if no NoiseSuppressor was present). + public boolean toggleNS(boolean enable) { + if (ns == null) { + Logging.e(TAG, "Attempting to enable or disable nonexistent NoiseSuppressor."); + return false; + } + Logging.d(TAG, "toggleNS(" + enable + ")"); + boolean toggling_succeeded = ns.setEnabled(enable) == AudioEffect.SUCCESS; + return toggling_succeeded; + } + + public void enable(int audioSession) { + Logging.d(TAG, "enable(audioSession=" + audioSession + ")"); + assertTrue(aec == null); + assertTrue(ns == null); + + if (DEBUG) { + // Add logging of supported effects but filter out "VoIP effects", i.e., + // AEC, AEC and NS. Avoid calling AudioEffect.queryEffects() unless the + // DEBUG flag is set since we have seen crashes in this API. + for (Descriptor d : AudioEffect.queryEffects()) { + if (effectTypeIsVoIP(d.type)) { + Logging.d(TAG, + "name: " + d.name + ", " + + "mode: " + d.connectMode + ", " + + "implementor: " + d.implementor + ", " + + "UUID: " + d.uuid); + } + } + } + + if (isAcousticEchoCancelerSupported()) { + // Create an AcousticEchoCanceler and attach it to the AudioRecord on + // the specified audio session. + aec = AcousticEchoCanceler.create(audioSession); + if (aec != null) { + boolean enabled = aec.getEnabled(); + boolean enable = shouldEnableAec && isAcousticEchoCancelerSupported(); + if (aec.setEnabled(enable) != AudioEffect.SUCCESS) { + Logging.e(TAG, "Failed to set the AcousticEchoCanceler state"); + } + Logging.d(TAG, + "AcousticEchoCanceler: was " + (enabled ? "enabled" : "disabled") + ", enable: " + + enable + ", is now: " + (aec.getEnabled() ? "enabled" : "disabled")); + } else { + Logging.e(TAG, "Failed to create the AcousticEchoCanceler instance"); + } + } + + if (isNoiseSuppressorSupported()) { + // Create an NoiseSuppressor and attach it to the AudioRecord on the + // specified audio session. + ns = NoiseSuppressor.create(audioSession); + if (ns != null) { + boolean enabled = ns.getEnabled(); + boolean enable = shouldEnableNs && isNoiseSuppressorSupported(); + if (ns.setEnabled(enable) != AudioEffect.SUCCESS) { + Logging.e(TAG, "Failed to set the NoiseSuppressor state"); + } + Logging.d(TAG, + "NoiseSuppressor: was " + (enabled ? "enabled" : "disabled") + ", enable: " + enable + + ", is now: " + (ns.getEnabled() ? "enabled" : "disabled")); + } else { + Logging.e(TAG, "Failed to create the NoiseSuppressor instance"); + } + } + } + + // Releases all native audio effect resources. It is a good practice to + // release the effect engine when not in use as control can be returned + // to other applications or the native resources released. + public void release() { + Logging.d(TAG, "release"); + if (aec != null) { + aec.release(); + aec = null; + } + if (ns != null) { + ns.release(); + ns = null; + } + } + + // Returns true for effect types in `type` that are of "VoIP" types: + // Acoustic Echo Canceler (AEC) or Automatic Gain Control (AGC) or + // Noise Suppressor (NS). Note that, an extra check for support is needed + // in each comparison since some devices includes effects in the + // AudioEffect.Descriptor array that are actually not available on the device. + // As an example: Samsung Galaxy S6 includes an AGC in the descriptor but + // AutomaticGainControl.isAvailable() returns false. + private boolean effectTypeIsVoIP(UUID type) { + return (AudioEffect.EFFECT_TYPE_AEC.equals(type) && isAcousticEchoCancelerSupported()) + || (AudioEffect.EFFECT_TYPE_NS.equals(type) && isNoiseSuppressorSupported()); + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + // Returns the cached copy of the audio effects array, if available, or + // queries the operating system for the list of effects. + private static @Nullable Descriptor[] getAvailableEffects() { + if (cachedEffects != null) { + return cachedEffects; + } + // The caching is best effort only - if this method is called from several + // threads in parallel, they may end up doing the underlying OS call + // multiple times. It's normally only called on one thread so there's no + // real need to optimize for the multiple threads case. + cachedEffects = AudioEffect.queryEffects(); + return cachedEffects; + } + + // Returns true if an effect of the specified type is available. Functionally + // equivalent to (NoiseSuppressor`AutomaticGainControl`...).isAvailable(), but + // faster as it avoids the expensive OS call to enumerate effects. + private static boolean isEffectTypeAvailable(UUID effectType, UUID blockListedUuid) { + Descriptor[] effects = getAvailableEffects(); + if (effects == null) { + return false; + } + for (Descriptor d : effects) { + if (d.type.equals(effectType)) { + return !d.uuid.equals(blockListedUuid); + } + } + return false; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java new file mode 100644 index 0000000000..506e33ffe4 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.content.Context; +import android.content.pm.PackageManager; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioRecord; +import android.media.AudioTrack; +import android.os.Build; +import org.webrtc.Logging; +import org.webrtc.CalledByNative; + +/** + * This class contains static functions to query sample rate and input/output audio buffer sizes. + */ +class WebRtcAudioManager { + private static final String TAG = "WebRtcAudioManagerExternal"; + + private static final int DEFAULT_SAMPLE_RATE_HZ = 16000; + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + private static final int BITS_PER_SAMPLE = 16; + + private static final int DEFAULT_FRAME_PER_BUFFER = 256; + + @CalledByNative + static AudioManager getAudioManager(Context context) { + return (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + } + + @CalledByNative + static int getOutputBufferSize( + Context context, AudioManager audioManager, int sampleRate, int numberOfOutputChannels) { + return isLowLatencyOutputSupported(context) + ? getLowLatencyFramesPerBuffer(audioManager) + : getMinOutputFrameSize(sampleRate, numberOfOutputChannels); + } + + @CalledByNative + static int getInputBufferSize( + Context context, AudioManager audioManager, int sampleRate, int numberOfInputChannels) { + return isLowLatencyInputSupported(context) + ? getLowLatencyFramesPerBuffer(audioManager) + : getMinInputFrameSize(sampleRate, numberOfInputChannels); + } + + @CalledByNative + static boolean isLowLatencyOutputSupported(Context context) { + return context.getPackageManager().hasSystemFeature(PackageManager.FEATURE_AUDIO_LOW_LATENCY); + } + + @CalledByNative + static boolean isLowLatencyInputSupported(Context context) { + // TODO(henrika): investigate if some sort of device list is needed here + // as well. The NDK doc states that: "As of API level 21, lower latency + // audio input is supported on select devices. To take advantage of this + // feature, first confirm that lower latency output is available". + return isLowLatencyOutputSupported(context); + } + + /** + * Returns the native input/output sample rate for this device's output stream. + */ + @CalledByNative + static int getSampleRate(AudioManager audioManager) { + // Override this if we're running on an old emulator image which only + // supports 8 kHz and doesn't support PROPERTY_OUTPUT_SAMPLE_RATE. + if (WebRtcAudioUtils.runningOnEmulator()) { + Logging.d(TAG, "Running emulator, overriding sample rate to 8 kHz."); + return 8000; + } + // Deliver best possible estimate based on default Android AudioManager APIs. + final int sampleRateHz = getSampleRateForApiLevel(audioManager); + Logging.d(TAG, "Sample rate is set to " + sampleRateHz + " Hz"); + return sampleRateHz; + } + + private static int getSampleRateForApiLevel(AudioManager audioManager) { + String sampleRateString = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE); + return (sampleRateString == null) ? DEFAULT_SAMPLE_RATE_HZ : Integer.parseInt(sampleRateString); + } + + // Returns the native output buffer size for low-latency output streams. + private static int getLowLatencyFramesPerBuffer(AudioManager audioManager) { + String framesPerBuffer = + audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER); + return framesPerBuffer == null ? DEFAULT_FRAME_PER_BUFFER : Integer.parseInt(framesPerBuffer); + } + + // Returns the minimum output buffer size for Java based audio (AudioTrack). + // This size can also be used for OpenSL ES implementations on devices that + // lacks support of low-latency output. + private static int getMinOutputFrameSize(int sampleRateInHz, int numChannels) { + final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); + return AudioTrack.getMinBufferSize( + sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) + / bytesPerFrame; + } + + // Returns the minimum input buffer size for Java based audio (AudioRecord). + // This size can calso be used for OpenSL ES implementations on devices that + // lacks support of low-latency input. + private static int getMinInputFrameSize(int sampleRateInHz, int numChannels) { + final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); + return AudioRecord.getMinBufferSize( + sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) + / bytesPerFrame; + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java new file mode 100644 index 0000000000..cfb651f6cd --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -0,0 +1,755 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.annotation.TargetApi; +import android.content.Context; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioRecord; +import android.media.AudioRecordingConfiguration; +import android.media.AudioTimestamp; +import android.media.MediaRecorder.AudioSource; +import android.os.Build; +import android.os.Process; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import java.lang.System; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.webrtc.CalledByNative; +import org.webrtc.Logging; +import org.webrtc.ThreadUtils; +import org.webrtc.audio.JavaAudioDeviceModule.AudioRecordErrorCallback; +import org.webrtc.audio.JavaAudioDeviceModule.AudioRecordStartErrorCode; +import org.webrtc.audio.JavaAudioDeviceModule.AudioRecordStateCallback; +import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback; + +class WebRtcAudioRecord { + private static final String TAG = "WebRtcAudioRecordExternal"; + + // Requested size of each recorded buffer provided to the client. + private static final int CALLBACK_BUFFER_SIZE_MS = 10; + + // Average number of callbacks per second. + private static final int BUFFERS_PER_SECOND = 1000 / CALLBACK_BUFFER_SIZE_MS; + + // We ask for a native buffer size of BUFFER_SIZE_FACTOR * (minimum required + // buffer size). The extra space is allocated to guard against glitches under + // high load. + private static final int BUFFER_SIZE_FACTOR = 2; + + // The AudioRecordJavaThread is allowed to wait for successful call to join() + // but the wait times out afther this amount of time. + private static final long AUDIO_RECORD_THREAD_JOIN_TIMEOUT_MS = 2000; + + public static final int DEFAULT_AUDIO_SOURCE = AudioSource.VOICE_COMMUNICATION; + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + public static final int DEFAULT_AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT; + + // Indicates AudioRecord has started recording audio. + private static final int AUDIO_RECORD_START = 0; + + // Indicates AudioRecord has stopped recording audio. + private static final int AUDIO_RECORD_STOP = 1; + + // Time to wait before checking recording status after start has been called. Tests have + // shown that the result can sometimes be invalid (our own status might be missing) if we check + // directly after start. + private static final int CHECK_REC_STATUS_DELAY_MS = 100; + + private final Context context; + private final AudioManager audioManager; + private final int audioSource; + private final int audioFormat; + + private long nativeAudioRecord; + + private final WebRtcAudioEffects effects = new WebRtcAudioEffects(); + + private @Nullable ByteBuffer byteBuffer; + + private @Nullable AudioRecord audioRecord; + private @Nullable AudioRecordThread audioThread; + private @Nullable AudioDeviceInfo preferredDevice; + + private final ScheduledExecutorService executor; + private @Nullable ScheduledFuture<String> future; + + private volatile boolean microphoneMute; + private final AtomicReference<Boolean> audioSourceMatchesRecordingSessionRef = + new AtomicReference<>(); + private byte[] emptyBytes; + + private final @Nullable AudioRecordErrorCallback errorCallback; + private final @Nullable AudioRecordStateCallback stateCallback; + private final @Nullable SamplesReadyCallback audioSamplesReadyCallback; + private final boolean isAcousticEchoCancelerSupported; + private final boolean isNoiseSuppressorSupported; + + /** + * Audio thread which keeps calling ByteBuffer.read() waiting for audio + * to be recorded. Feeds recorded data to the native counterpart as a + * periodic sequence of callbacks using DataIsRecorded(). + * This thread uses a Process.THREAD_PRIORITY_URGENT_AUDIO priority. + */ + private class AudioRecordThread extends Thread { + private volatile boolean keepAlive = true; + + public AudioRecordThread(String name) { + super(name); + } + + @Override + public void run() { + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO); + Logging.d(TAG, "AudioRecordThread" + WebRtcAudioUtils.getThreadInfo()); + assertTrue(audioRecord.getRecordingState() == AudioRecord.RECORDSTATE_RECORDING); + + // Audio recording has started and the client is informed about it. + doAudioRecordStateCallback(AUDIO_RECORD_START); + + long lastTime = System.nanoTime(); + AudioTimestamp audioTimestamp = null; + if (Build.VERSION.SDK_INT >= 24) { + audioTimestamp = new AudioTimestamp(); + } + while (keepAlive) { + int bytesRead = audioRecord.read(byteBuffer, byteBuffer.capacity()); + if (bytesRead == byteBuffer.capacity()) { + if (microphoneMute) { + byteBuffer.clear(); + byteBuffer.put(emptyBytes); + } + // It's possible we've been shut down during the read, and stopRecording() tried and + // failed to join this thread. To be a bit safer, try to avoid calling any native methods + // in case they've been unregistered after stopRecording() returned. + if (keepAlive) { + long captureTimeNs = 0; + if (Build.VERSION.SDK_INT >= 24) { + if (audioRecord.getTimestamp(audioTimestamp, AudioTimestamp.TIMEBASE_MONOTONIC) + == AudioRecord.SUCCESS) { + captureTimeNs = audioTimestamp.nanoTime; + } + } + nativeDataIsRecorded(nativeAudioRecord, bytesRead, captureTimeNs); + } + if (audioSamplesReadyCallback != null) { + // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily + // at index 0. + byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), + byteBuffer.capacity() + byteBuffer.arrayOffset()); + audioSamplesReadyCallback.onWebRtcAudioRecordSamplesReady( + new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), + audioRecord.getChannelCount(), audioRecord.getSampleRate(), data)); + } + } else { + String errorMessage = "AudioRecord.read failed: " + bytesRead; + Logging.e(TAG, errorMessage); + if (bytesRead == AudioRecord.ERROR_INVALID_OPERATION) { + keepAlive = false; + reportWebRtcAudioRecordError(errorMessage); + } + } + } + + try { + if (audioRecord != null) { + audioRecord.stop(); + doAudioRecordStateCallback(AUDIO_RECORD_STOP); + } + } catch (IllegalStateException e) { + Logging.e(TAG, "AudioRecord.stop failed: " + e.getMessage()); + } + } + + // Stops the inner thread loop and also calls AudioRecord.stop(). + // Does not block the calling thread. + public void stopThread() { + Logging.d(TAG, "stopThread"); + keepAlive = false; + } + } + + @CalledByNative + WebRtcAudioRecord(Context context, AudioManager audioManager) { + this(context, newDefaultScheduler() /* scheduler */, audioManager, DEFAULT_AUDIO_SOURCE, + DEFAULT_AUDIO_FORMAT, null /* errorCallback */, null /* stateCallback */, + null /* audioSamplesReadyCallback */, WebRtcAudioEffects.isAcousticEchoCancelerSupported(), + WebRtcAudioEffects.isNoiseSuppressorSupported()); + } + + public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler, + AudioManager audioManager, int audioSource, int audioFormat, + @Nullable AudioRecordErrorCallback errorCallback, + @Nullable AudioRecordStateCallback stateCallback, + @Nullable SamplesReadyCallback audioSamplesReadyCallback, + boolean isAcousticEchoCancelerSupported, boolean isNoiseSuppressorSupported) { + if (isAcousticEchoCancelerSupported && !WebRtcAudioEffects.isAcousticEchoCancelerSupported()) { + throw new IllegalArgumentException("HW AEC not supported"); + } + if (isNoiseSuppressorSupported && !WebRtcAudioEffects.isNoiseSuppressorSupported()) { + throw new IllegalArgumentException("HW NS not supported"); + } + this.context = context; + this.executor = scheduler; + this.audioManager = audioManager; + this.audioSource = audioSource; + this.audioFormat = audioFormat; + this.errorCallback = errorCallback; + this.stateCallback = stateCallback; + this.audioSamplesReadyCallback = audioSamplesReadyCallback; + this.isAcousticEchoCancelerSupported = isAcousticEchoCancelerSupported; + this.isNoiseSuppressorSupported = isNoiseSuppressorSupported; + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + } + + @CalledByNative + public void setNativeAudioRecord(long nativeAudioRecord) { + this.nativeAudioRecord = nativeAudioRecord; + } + + @CalledByNative + boolean isAcousticEchoCancelerSupported() { + return isAcousticEchoCancelerSupported; + } + + @CalledByNative + boolean isNoiseSuppressorSupported() { + return isNoiseSuppressorSupported; + } + + // Returns true if a valid call to verifyAudioConfig() has been done. Should always be + // checked before using the returned value of isAudioSourceMatchingRecordingSession(). + @CalledByNative + boolean isAudioConfigVerified() { + return audioSourceMatchesRecordingSessionRef.get() != null; + } + + // Returns true if verifyAudioConfig() succeeds. This value is set after a specific delay when + // startRecording() has been called. Hence, should preferably be called in combination with + // stopRecording() to ensure that it has been set properly. `isAudioConfigVerified` is + // enabled in WebRtcAudioRecord to ensure that the returned value is valid. + @CalledByNative + boolean isAudioSourceMatchingRecordingSession() { + Boolean audioSourceMatchesRecordingSession = audioSourceMatchesRecordingSessionRef.get(); + if (audioSourceMatchesRecordingSession == null) { + Logging.w(TAG, "Audio configuration has not yet been verified"); + return false; + } + return audioSourceMatchesRecordingSession; + } + + @CalledByNative + private boolean enableBuiltInAEC(boolean enable) { + Logging.d(TAG, "enableBuiltInAEC(" + enable + ")"); + return effects.setAEC(enable); + } + + @CalledByNative + private boolean enableBuiltInNS(boolean enable) { + Logging.d(TAG, "enableBuiltInNS(" + enable + ")"); + return effects.setNS(enable); + } + + @CalledByNative + private int initRecording(int sampleRate, int channels) { + Logging.d(TAG, "initRecording(sampleRate=" + sampleRate + ", channels=" + channels + ")"); + if (audioRecord != null) { + reportWebRtcAudioRecordInitError("InitRecording called twice without StopRecording."); + return -1; + } + final int bytesPerFrame = channels * getBytesPerSample(audioFormat); + final int framesPerBuffer = sampleRate / BUFFERS_PER_SECOND; + byteBuffer = ByteBuffer.allocateDirect(bytesPerFrame * framesPerBuffer); + if (!(byteBuffer.hasArray())) { + reportWebRtcAudioRecordInitError("ByteBuffer does not have backing array."); + return -1; + } + Logging.d(TAG, "byteBuffer.capacity: " + byteBuffer.capacity()); + emptyBytes = new byte[byteBuffer.capacity()]; + // Rather than passing the ByteBuffer with every callback (requiring + // the potentially expensive GetDirectBufferAddress) we simply have the + // the native class cache the address to the memory once. + nativeCacheDirectBufferAddress(nativeAudioRecord, byteBuffer); + + // Get the minimum buffer size required for the successful creation of + // an AudioRecord object, in byte units. + // Note that this size doesn't guarantee a smooth recording under load. + final int channelConfig = channelCountToConfiguration(channels); + int minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat); + if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { + reportWebRtcAudioRecordInitError("AudioRecord.getMinBufferSize failed: " + minBufferSize); + return -1; + } + Logging.d(TAG, "AudioRecord.getMinBufferSize: " + minBufferSize); + + // Use a larger buffer size than the minimum required when creating the + // AudioRecord instance to ensure smooth recording under load. It has been + // verified that it does not increase the actual recording latency. + int bufferSizeInBytes = Math.max(BUFFER_SIZE_FACTOR * minBufferSize, byteBuffer.capacity()); + Logging.d(TAG, "bufferSizeInBytes: " + bufferSizeInBytes); + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + // Use the AudioRecord.Builder class on Android M (23) and above. + // Throws IllegalArgumentException. + audioRecord = createAudioRecordOnMOrHigher( + audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); + audioSourceMatchesRecordingSessionRef.set(null); + if (preferredDevice != null) { + setPreferredDevice(preferredDevice); + } + } else { + // Use the old AudioRecord constructor for API levels below 23. + // Throws UnsupportedOperationException. + audioRecord = createAudioRecordOnLowerThanM( + audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); + audioSourceMatchesRecordingSessionRef.set(null); + } + } catch (IllegalArgumentException | UnsupportedOperationException e) { + // Report of exception message is sufficient. Example: "Cannot create AudioRecord". + reportWebRtcAudioRecordInitError(e.getMessage()); + releaseAudioResources(); + return -1; + } + if (audioRecord == null || audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { + reportWebRtcAudioRecordInitError("Creation or initialization of audio recorder failed."); + releaseAudioResources(); + return -1; + } + effects.enable(audioRecord.getAudioSessionId()); + logMainParameters(); + logMainParametersExtended(); + // Check number of active recording sessions. Should be zero but we have seen conflict cases + // and adding a log for it can help us figure out details about conflicting sessions. + final int numActiveRecordingSessions = + logRecordingConfigurations(audioRecord, false /* verifyAudioConfig */); + if (numActiveRecordingSessions != 0) { + // Log the conflict as a warning since initialization did in fact succeed. Most likely, the + // upcoming call to startRecording() will fail under these conditions. + Logging.w( + TAG, "Potential microphone conflict. Active sessions: " + numActiveRecordingSessions); + } + return framesPerBuffer; + } + + /** + * Prefer a specific {@link AudioDeviceInfo} device for recording. Calling after recording starts + * is valid but may cause a temporary interruption if the audio routing changes. + */ + @RequiresApi(Build.VERSION_CODES.M) + @TargetApi(Build.VERSION_CODES.M) + void setPreferredDevice(@Nullable AudioDeviceInfo preferredDevice) { + Logging.d( + TAG, "setPreferredDevice " + (preferredDevice != null ? preferredDevice.getId() : null)); + this.preferredDevice = preferredDevice; + if (audioRecord != null) { + if (!audioRecord.setPreferredDevice(preferredDevice)) { + Logging.e(TAG, "setPreferredDevice failed"); + } + } + } + + @CalledByNative + private boolean startRecording() { + Logging.d(TAG, "startRecording"); + assertTrue(audioRecord != null); + assertTrue(audioThread == null); + try { + audioRecord.startRecording(); + } catch (IllegalStateException e) { + reportWebRtcAudioRecordStartError(AudioRecordStartErrorCode.AUDIO_RECORD_START_EXCEPTION, + "AudioRecord.startRecording failed: " + e.getMessage()); + return false; + } + if (audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) { + reportWebRtcAudioRecordStartError(AudioRecordStartErrorCode.AUDIO_RECORD_START_STATE_MISMATCH, + "AudioRecord.startRecording failed - incorrect state: " + + audioRecord.getRecordingState()); + return false; + } + audioThread = new AudioRecordThread("AudioRecordJavaThread"); + audioThread.start(); + scheduleLogRecordingConfigurationsTask(audioRecord); + return true; + } + + @CalledByNative + private boolean stopRecording() { + Logging.d(TAG, "stopRecording"); + assertTrue(audioThread != null); + if (future != null) { + if (!future.isDone()) { + // Might be needed if the client calls startRecording(), stopRecording() back-to-back. + future.cancel(true /* mayInterruptIfRunning */); + } + future = null; + } + audioThread.stopThread(); + if (!ThreadUtils.joinUninterruptibly(audioThread, AUDIO_RECORD_THREAD_JOIN_TIMEOUT_MS)) { + Logging.e(TAG, "Join of AudioRecordJavaThread timed out"); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + } + audioThread = null; + effects.release(); + releaseAudioResources(); + return true; + } + + @TargetApi(Build.VERSION_CODES.M) + private static AudioRecord createAudioRecordOnMOrHigher( + int audioSource, int sampleRate, int channelConfig, int audioFormat, int bufferSizeInBytes) { + Logging.d(TAG, "createAudioRecordOnMOrHigher"); + return new AudioRecord.Builder() + .setAudioSource(audioSource) + .setAudioFormat(new AudioFormat.Builder() + .setEncoding(audioFormat) + .setSampleRate(sampleRate) + .setChannelMask(channelConfig) + .build()) + .setBufferSizeInBytes(bufferSizeInBytes) + .build(); + } + + private static AudioRecord createAudioRecordOnLowerThanM( + int audioSource, int sampleRate, int channelConfig, int audioFormat, int bufferSizeInBytes) { + Logging.d(TAG, "createAudioRecordOnLowerThanM"); + return new AudioRecord(audioSource, sampleRate, channelConfig, audioFormat, bufferSizeInBytes); + } + + private void logMainParameters() { + Logging.d(TAG, + "AudioRecord: " + + "session ID: " + audioRecord.getAudioSessionId() + ", " + + "channels: " + audioRecord.getChannelCount() + ", " + + "sample rate: " + audioRecord.getSampleRate()); + } + + @TargetApi(Build.VERSION_CODES.M) + private void logMainParametersExtended() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + Logging.d(TAG, + "AudioRecord: " + // The frame count of the native AudioRecord buffer. + + "buffer size in frames: " + audioRecord.getBufferSizeInFrames()); + } + } + + @TargetApi(Build.VERSION_CODES.N) + // Checks the number of active recording sessions and logs the states of all active sessions. + // Returns number of active sessions. Note that this could occur on arbituary thread. + private int logRecordingConfigurations(AudioRecord audioRecord, boolean verifyAudioConfig) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + Logging.w(TAG, "AudioManager#getActiveRecordingConfigurations() requires N or higher"); + return 0; + } + if (audioRecord == null) { + return 0; + } + + // Get a list of the currently active audio recording configurations of the device (can be more + // than one). An empty list indicates there is no recording active when queried. + List<AudioRecordingConfiguration> configs = audioManager.getActiveRecordingConfigurations(); + final int numActiveRecordingSessions = configs.size(); + Logging.d(TAG, "Number of active recording sessions: " + numActiveRecordingSessions); + if (numActiveRecordingSessions > 0) { + logActiveRecordingConfigs(audioRecord.getAudioSessionId(), configs); + if (verifyAudioConfig) { + // Run an extra check to verify that the existing audio source doing the recording (tied + // to the AudioRecord instance) is matching what the audio recording configuration lists + // as its client parameters. If these do not match, recording might work but under invalid + // conditions. + audioSourceMatchesRecordingSessionRef.set( + verifyAudioConfig(audioRecord.getAudioSource(), audioRecord.getAudioSessionId(), + audioRecord.getFormat(), audioRecord.getRoutedDevice(), configs)); + } + } + return numActiveRecordingSessions; + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); + } + + private native void nativeCacheDirectBufferAddress( + long nativeAudioRecordJni, ByteBuffer byteBuffer); + private native void nativeDataIsRecorded( + long nativeAudioRecordJni, int bytes, long captureTimestampNs); + + // Sets all recorded samples to zero if `mute` is true, i.e., ensures that + // the microphone is muted. + public void setMicrophoneMute(boolean mute) { + Logging.w(TAG, "setMicrophoneMute(" + mute + ")"); + microphoneMute = mute; + } + + // Sets whether NoiseSuppressor should be enabled or disabled. + // Returns true if the enabling was successful, otherwise false is returned (this is also the case + // if the NoiseSuppressor effect is not supported). + public boolean setNoiseSuppressorEnabled(boolean enabled) { + if (!WebRtcAudioEffects.isNoiseSuppressorSupported()) { + Logging.e(TAG, "Noise suppressor is not supported."); + return false; + } + Logging.w(TAG, "SetNoiseSuppressorEnabled(" + enabled + ")"); + return effects.toggleNS(enabled); + } + + // Releases the native AudioRecord resources. + private void releaseAudioResources() { + Logging.d(TAG, "releaseAudioResources"); + if (audioRecord != null) { + audioRecord.release(); + audioRecord = null; + } + audioSourceMatchesRecordingSessionRef.set(null); + } + + private void reportWebRtcAudioRecordInitError(String errorMessage) { + Logging.e(TAG, "Init recording error: " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + logRecordingConfigurations(audioRecord, false /* verifyAudioConfig */); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordInitError(errorMessage); + } + } + + private void reportWebRtcAudioRecordStartError( + AudioRecordStartErrorCode errorCode, String errorMessage) { + Logging.e(TAG, "Start recording error: " + errorCode + ". " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + logRecordingConfigurations(audioRecord, false /* verifyAudioConfig */); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordStartError(errorCode, errorMessage); + } + } + + private void reportWebRtcAudioRecordError(String errorMessage) { + Logging.e(TAG, "Run-time recording error: " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + if (errorCallback != null) { + errorCallback.onWebRtcAudioRecordError(errorMessage); + } + } + + private void doAudioRecordStateCallback(int audioState) { + Logging.d(TAG, "doAudioRecordStateCallback: " + audioStateToString(audioState)); + if (stateCallback != null) { + if (audioState == WebRtcAudioRecord.AUDIO_RECORD_START) { + stateCallback.onWebRtcAudioRecordStart(); + } else if (audioState == WebRtcAudioRecord.AUDIO_RECORD_STOP) { + stateCallback.onWebRtcAudioRecordStop(); + } else { + Logging.e(TAG, "Invalid audio state"); + } + } + } + + // Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8 + // Default audio data format is PCM 16 bits per sample. + // Guaranteed to be supported by all devices + private static int getBytesPerSample(int audioFormat) { + switch (audioFormat) { + case AudioFormat.ENCODING_PCM_8BIT: + return 1; + case AudioFormat.ENCODING_PCM_16BIT: + case AudioFormat.ENCODING_IEC61937: + case AudioFormat.ENCODING_DEFAULT: + return 2; + case AudioFormat.ENCODING_PCM_FLOAT: + return 4; + case AudioFormat.ENCODING_INVALID: + default: + throw new IllegalArgumentException("Bad audio format " + audioFormat); + } + } + + // Use an ExecutorService to schedule a task after a given delay where the task consists of + // checking (by logging) the current status of active recording sessions. + private void scheduleLogRecordingConfigurationsTask(AudioRecord audioRecord) { + Logging.d(TAG, "scheduleLogRecordingConfigurationsTask"); + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + return; + } + + Callable<String> callable = () -> { + if (this.audioRecord == audioRecord) { + logRecordingConfigurations(audioRecord, true /* verifyAudioConfig */); + } else { + Logging.d(TAG, "audio record has changed"); + } + return "Scheduled task is done"; + }; + + if (future != null && !future.isDone()) { + future.cancel(true /* mayInterruptIfRunning */); + } + // Schedule call to logRecordingConfigurations() from executor thread after fixed delay. + future = executor.schedule(callable, CHECK_REC_STATUS_DELAY_MS, TimeUnit.MILLISECONDS); + }; + + @TargetApi(Build.VERSION_CODES.N) + private static boolean logActiveRecordingConfigs( + int session, List<AudioRecordingConfiguration> configs) { + assertTrue(!configs.isEmpty()); + final Iterator<AudioRecordingConfiguration> it = configs.iterator(); + Logging.d(TAG, "AudioRecordingConfigurations: "); + while (it.hasNext()) { + final AudioRecordingConfiguration config = it.next(); + StringBuilder conf = new StringBuilder(); + // The audio source selected by the client. + final int audioSource = config.getClientAudioSource(); + conf.append(" client audio source=") + .append(WebRtcAudioUtils.audioSourceToString(audioSource)) + .append(", client session id=") + .append(config.getClientAudioSessionId()) + // Compare with our own id (based on AudioRecord#getAudioSessionId()). + .append(" (") + .append(session) + .append(")") + .append("\n"); + // Audio format at which audio is recorded on this Android device. Note that it may differ + // from the client application recording format (see getClientFormat()). + AudioFormat format = config.getFormat(); + conf.append(" Device AudioFormat: ") + .append("channel count=") + .append(format.getChannelCount()) + .append(", channel index mask=") + .append(format.getChannelIndexMask()) + // Only AudioFormat#CHANNEL_IN_MONO is guaranteed to work on all devices. + .append(", channel mask=") + .append(WebRtcAudioUtils.channelMaskToString(format.getChannelMask())) + .append(", encoding=") + .append(WebRtcAudioUtils.audioEncodingToString(format.getEncoding())) + .append(", sample rate=") + .append(format.getSampleRate()) + .append("\n"); + // Audio format at which the client application is recording audio. + format = config.getClientFormat(); + conf.append(" Client AudioFormat: ") + .append("channel count=") + .append(format.getChannelCount()) + .append(", channel index mask=") + .append(format.getChannelIndexMask()) + // Only AudioFormat#CHANNEL_IN_MONO is guaranteed to work on all devices. + .append(", channel mask=") + .append(WebRtcAudioUtils.channelMaskToString(format.getChannelMask())) + .append(", encoding=") + .append(WebRtcAudioUtils.audioEncodingToString(format.getEncoding())) + .append(", sample rate=") + .append(format.getSampleRate()) + .append("\n"); + // Audio input device used for this recording session. + final AudioDeviceInfo device = config.getAudioDevice(); + if (device != null) { + assertTrue(device.isSource()); + conf.append(" AudioDevice: ") + .append("type=") + .append(WebRtcAudioUtils.deviceTypeToString(device.getType())) + .append(", id=") + .append(device.getId()); + } + Logging.d(TAG, conf.toString()); + } + return true; + } + + // Verify that the client audio configuration (device and format) matches the requested + // configuration (same as AudioRecord's). + @TargetApi(Build.VERSION_CODES.N) + private static boolean verifyAudioConfig(int source, int session, AudioFormat format, + AudioDeviceInfo device, List<AudioRecordingConfiguration> configs) { + assertTrue(!configs.isEmpty()); + final Iterator<AudioRecordingConfiguration> it = configs.iterator(); + while (it.hasNext()) { + final AudioRecordingConfiguration config = it.next(); + final AudioDeviceInfo configDevice = config.getAudioDevice(); + if (configDevice == null) { + continue; + } + if ((config.getClientAudioSource() == source) + && (config.getClientAudioSessionId() == session) + // Check the client format (should match the format of the AudioRecord instance). + && (config.getClientFormat().getEncoding() == format.getEncoding()) + && (config.getClientFormat().getSampleRate() == format.getSampleRate()) + && (config.getClientFormat().getChannelMask() == format.getChannelMask()) + && (config.getClientFormat().getChannelIndexMask() == format.getChannelIndexMask()) + // Ensure that the device format is properly configured. + && (config.getFormat().getEncoding() != AudioFormat.ENCODING_INVALID) + && (config.getFormat().getSampleRate() > 0) + // For the channel mask, either the position or index-based value must be valid. + && ((config.getFormat().getChannelMask() != AudioFormat.CHANNEL_INVALID) + || (config.getFormat().getChannelIndexMask() != AudioFormat.CHANNEL_INVALID)) + && checkDeviceMatch(configDevice, device)) { + Logging.d(TAG, "verifyAudioConfig: PASS"); + return true; + } + } + Logging.e(TAG, "verifyAudioConfig: FAILED"); + return false; + } + + @TargetApi(Build.VERSION_CODES.N) + // Returns true if device A parameters matches those of device B. + // TODO(henrika): can be improved by adding AudioDeviceInfo#getAddress() but it requires API 29. + private static boolean checkDeviceMatch(AudioDeviceInfo devA, AudioDeviceInfo devB) { + return ((devA.getId() == devB.getId() && (devA.getType() == devB.getType()))); + } + + private static String audioStateToString(int state) { + switch (state) { + case WebRtcAudioRecord.AUDIO_RECORD_START: + return "START"; + case WebRtcAudioRecord.AUDIO_RECORD_STOP: + return "STOP"; + default: + return "INVALID"; + } + } + + private static final AtomicInteger nextSchedulerId = new AtomicInteger(0); + + static ScheduledExecutorService newDefaultScheduler() { + AtomicInteger nextThreadId = new AtomicInteger(0); + return Executors.newScheduledThreadPool(0, new ThreadFactory() { + /** + * Constructs a new {@code Thread} + */ + @Override + public Thread newThread(Runnable r) { + Thread thread = Executors.defaultThreadFactory().newThread(r); + thread.setName(String.format("WebRtcAudioRecordScheduler-%s-%s", + nextSchedulerId.getAndIncrement(), nextThreadId.getAndIncrement())); + return thread; + } + }); + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java new file mode 100644 index 0000000000..2b34e34013 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -0,0 +1,585 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import android.annotation.TargetApi; +import android.content.Context; +import android.media.AudioAttributes; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioTrack; +import android.os.Build; +import android.os.Process; +import androidx.annotation.Nullable; +import java.nio.ByteBuffer; +import org.webrtc.CalledByNative; +import org.webrtc.Logging; +import org.webrtc.ThreadUtils; +import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback; +import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode; +import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback; +import org.webrtc.audio.LowLatencyAudioBufferManager; + +class WebRtcAudioTrack { + private static final String TAG = "WebRtcAudioTrackExternal"; + + // Default audio data format is PCM 16 bit per sample. + // Guaranteed to be supported by all devices. + private static final int BITS_PER_SAMPLE = 16; + + // Requested size of each recorded buffer provided to the client. + private static final int CALLBACK_BUFFER_SIZE_MS = 10; + + // Average number of callbacks per second. + private static final int BUFFERS_PER_SECOND = 1000 / CALLBACK_BUFFER_SIZE_MS; + + // The AudioTrackThread is allowed to wait for successful call to join() + // but the wait times out afther this amount of time. + private static final long AUDIO_TRACK_THREAD_JOIN_TIMEOUT_MS = 2000; + + // By default, WebRTC creates audio tracks with a usage attribute + // corresponding to voice communications, such as telephony or VoIP. + private static final int DEFAULT_USAGE = AudioAttributes.USAGE_VOICE_COMMUNICATION; + + // Indicates the AudioTrack has started playing audio. + private static final int AUDIO_TRACK_START = 0; + + // Indicates the AudioTrack has stopped playing audio. + private static final int AUDIO_TRACK_STOP = 1; + + private long nativeAudioTrack; + private final Context context; + private final AudioManager audioManager; + private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); + + private ByteBuffer byteBuffer; + + private @Nullable final AudioAttributes audioAttributes; + private @Nullable AudioTrack audioTrack; + private @Nullable AudioTrackThread audioThread; + private final VolumeLogger volumeLogger; + + // Samples to be played are replaced by zeros if `speakerMute` is set to true. + // Can be used to ensure that the speaker is fully muted. + private volatile boolean speakerMute; + private byte[] emptyBytes; + private boolean useLowLatency; + private int initialBufferSizeInFrames; + + private final @Nullable AudioTrackErrorCallback errorCallback; + private final @Nullable AudioTrackStateCallback stateCallback; + + /** + * Audio thread which keeps calling AudioTrack.write() to stream audio. + * Data is periodically acquired from the native WebRTC layer using the + * nativeGetPlayoutData callback function. + * This thread uses a Process.THREAD_PRIORITY_URGENT_AUDIO priority. + */ + private class AudioTrackThread extends Thread { + private volatile boolean keepAlive = true; + private LowLatencyAudioBufferManager bufferManager; + + public AudioTrackThread(String name) { + super(name); + bufferManager = new LowLatencyAudioBufferManager(); + } + + @Override + public void run() { + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO); + Logging.d(TAG, "AudioTrackThread" + WebRtcAudioUtils.getThreadInfo()); + assertTrue(audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PLAYING); + + // Audio playout has started and the client is informed about it. + doAudioTrackStateCallback(AUDIO_TRACK_START); + + // Fixed size in bytes of each 10ms block of audio data that we ask for + // using callbacks to the native WebRTC client. + final int sizeInBytes = byteBuffer.capacity(); + + while (keepAlive) { + // Get 10ms of PCM data from the native WebRTC client. Audio data is + // written into the common ByteBuffer using the address that was + // cached at construction. + nativeGetPlayoutData(nativeAudioTrack, sizeInBytes); + // Write data until all data has been written to the audio sink. + // Upon return, the buffer position will have been advanced to reflect + // the amount of data that was successfully written to the AudioTrack. + assertTrue(sizeInBytes <= byteBuffer.remaining()); + if (speakerMute) { + byteBuffer.clear(); + byteBuffer.put(emptyBytes); + byteBuffer.position(0); + } + int bytesWritten = audioTrack.write(byteBuffer, sizeInBytes, AudioTrack.WRITE_BLOCKING); + if (bytesWritten != sizeInBytes) { + Logging.e(TAG, "AudioTrack.write played invalid number of bytes: " + bytesWritten); + // If a write() returns a negative value, an error has occurred. + // Stop playing and report an error in this case. + if (bytesWritten < 0) { + keepAlive = false; + reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten); + } + } + if (useLowLatency) { + bufferManager.maybeAdjustBufferSize(audioTrack); + } + // The byte buffer must be rewinded since byteBuffer.position() is + // increased at each call to AudioTrack.write(). If we don't do this, + // next call to AudioTrack.write() will fail. + byteBuffer.rewind(); + + // TODO(henrika): it is possible to create a delay estimate here by + // counting number of written frames and subtracting the result from + // audioTrack.getPlaybackHeadPosition(). + } + } + + // Stops the inner thread loop which results in calling AudioTrack.stop(). + // Does not block the calling thread. + public void stopThread() { + Logging.d(TAG, "stopThread"); + keepAlive = false; + } + } + + @CalledByNative + WebRtcAudioTrack(Context context, AudioManager audioManager) { + this(context, audioManager, null /* audioAttributes */, null /* errorCallback */, + null /* stateCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); + } + + WebRtcAudioTrack(Context context, AudioManager audioManager, + @Nullable AudioAttributes audioAttributes, @Nullable AudioTrackErrorCallback errorCallback, + @Nullable AudioTrackStateCallback stateCallback, boolean useLowLatency, + boolean enableVolumeLogger) { + threadChecker.detachThread(); + this.context = context; + this.audioManager = audioManager; + this.audioAttributes = audioAttributes; + this.errorCallback = errorCallback; + this.stateCallback = stateCallback; + this.volumeLogger = enableVolumeLogger ? new VolumeLogger(audioManager) : null; + this.useLowLatency = useLowLatency; + Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); + } + + @CalledByNative + public void setNativeAudioTrack(long nativeAudioTrack) { + this.nativeAudioTrack = nativeAudioTrack; + } + + @CalledByNative + private int initPlayout(int sampleRate, int channels, double bufferSizeFactor) { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, + "initPlayout(sampleRate=" + sampleRate + ", channels=" + channels + + ", bufferSizeFactor=" + bufferSizeFactor + ")"); + final int bytesPerFrame = channels * (BITS_PER_SAMPLE / 8); + byteBuffer = ByteBuffer.allocateDirect(bytesPerFrame * (sampleRate / BUFFERS_PER_SECOND)); + Logging.d(TAG, "byteBuffer.capacity: " + byteBuffer.capacity()); + emptyBytes = new byte[byteBuffer.capacity()]; + // Rather than passing the ByteBuffer with every callback (requiring + // the potentially expensive GetDirectBufferAddress) we simply have the + // the native class cache the address to the memory once. + nativeCacheDirectBufferAddress(nativeAudioTrack, byteBuffer); + + // Get the minimum buffer size required for the successful creation of an + // AudioTrack object to be created in the MODE_STREAM mode. + // Note that this size doesn't guarantee a smooth playback under load. + final int channelConfig = channelCountToConfiguration(channels); + final int minBufferSizeInBytes = (int) (AudioTrack.getMinBufferSize(sampleRate, channelConfig, + AudioFormat.ENCODING_PCM_16BIT) + * bufferSizeFactor); + Logging.d(TAG, "minBufferSizeInBytes: " + minBufferSizeInBytes); + // For the streaming mode, data must be written to the audio sink in + // chunks of size (given by byteBuffer.capacity()) less than or equal + // to the total buffer size `minBufferSizeInBytes`. But, we have seen + // reports of "getMinBufferSize(): error querying hardware". Hence, it + // can happen that `minBufferSizeInBytes` contains an invalid value. + if (minBufferSizeInBytes < byteBuffer.capacity()) { + reportWebRtcAudioTrackInitError("AudioTrack.getMinBufferSize returns an invalid value."); + return -1; + } + + // Don't use low-latency mode when a bufferSizeFactor > 1 is used. When bufferSizeFactor > 1 + // we want to use a larger buffer to prevent underruns. However, low-latency mode would + // decrease the buffer size, which makes the bufferSizeFactor have no effect. + if (bufferSizeFactor > 1.0) { + useLowLatency = false; + } + + // Ensure that prevision audio session was stopped correctly before trying + // to create a new AudioTrack. + if (audioTrack != null) { + reportWebRtcAudioTrackInitError("Conflict with existing AudioTrack."); + return -1; + } + try { + // Create an AudioTrack object and initialize its associated audio buffer. + // The size of this buffer determines how long an AudioTrack can play + // before running out of data. + if (useLowLatency && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // On API level 26 or higher, we can use a low latency mode. + audioTrack = createAudioTrackOnOreoOrHigher( + sampleRate, channelConfig, minBufferSizeInBytes, audioAttributes); + } else { + // As we are on API level 21 or higher, it is possible to use a special AudioTrack + // constructor that uses AudioAttributes and AudioFormat as input. It allows us to + // supersede the notion of stream types for defining the behavior of audio playback, + // and to allow certain platforms or routing policies to use this information for more + // refined volume or routing decisions. + audioTrack = createAudioTrackBeforeOreo( + sampleRate, channelConfig, minBufferSizeInBytes, audioAttributes); + } + } catch (IllegalArgumentException e) { + reportWebRtcAudioTrackInitError(e.getMessage()); + releaseAudioResources(); + return -1; + } + + // It can happen that an AudioTrack is created but it was not successfully + // initialized upon creation. Seems to be the case e.g. when the maximum + // number of globally available audio tracks is exceeded. + if (audioTrack == null || audioTrack.getState() != AudioTrack.STATE_INITIALIZED) { + reportWebRtcAudioTrackInitError("Initialization of audio track failed."); + releaseAudioResources(); + return -1; + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + initialBufferSizeInFrames = audioTrack.getBufferSizeInFrames(); + } else { + initialBufferSizeInFrames = -1; + } + logMainParameters(); + logMainParametersExtended(); + return minBufferSizeInBytes; + } + + @CalledByNative + private boolean startPlayout() { + threadChecker.checkIsOnValidThread(); + if (volumeLogger != null) { + volumeLogger.start(); + } + Logging.d(TAG, "startPlayout"); + assertTrue(audioTrack != null); + assertTrue(audioThread == null); + + // Starts playing an audio track. + try { + audioTrack.play(); + } catch (IllegalStateException e) { + reportWebRtcAudioTrackStartError(AudioTrackStartErrorCode.AUDIO_TRACK_START_EXCEPTION, + "AudioTrack.play failed: " + e.getMessage()); + releaseAudioResources(); + return false; + } + if (audioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { + reportWebRtcAudioTrackStartError(AudioTrackStartErrorCode.AUDIO_TRACK_START_STATE_MISMATCH, + "AudioTrack.play failed - incorrect state :" + audioTrack.getPlayState()); + releaseAudioResources(); + return false; + } + + // Create and start new high-priority thread which calls AudioTrack.write() + // and where we also call the native nativeGetPlayoutData() callback to + // request decoded audio from WebRTC. + audioThread = new AudioTrackThread("AudioTrackJavaThread"); + audioThread.start(); + return true; + } + + @CalledByNative + private boolean stopPlayout() { + threadChecker.checkIsOnValidThread(); + if (volumeLogger != null) { + volumeLogger.stop(); + } + Logging.d(TAG, "stopPlayout"); + assertTrue(audioThread != null); + logUnderrunCount(); + audioThread.stopThread(); + + Logging.d(TAG, "Stopping the AudioTrackThread..."); + audioThread.interrupt(); + if (!ThreadUtils.joinUninterruptibly(audioThread, AUDIO_TRACK_THREAD_JOIN_TIMEOUT_MS)) { + Logging.e(TAG, "Join of AudioTrackThread timed out."); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + } + Logging.d(TAG, "AudioTrackThread has now been stopped."); + audioThread = null; + if (audioTrack != null) { + Logging.d(TAG, "Calling AudioTrack.stop..."); + try { + audioTrack.stop(); + Logging.d(TAG, "AudioTrack.stop is done."); + doAudioTrackStateCallback(AUDIO_TRACK_STOP); + } catch (IllegalStateException e) { + Logging.e(TAG, "AudioTrack.stop failed: " + e.getMessage()); + } + } + releaseAudioResources(); + return true; + } + + // Get max possible volume index for a phone call audio stream. + @CalledByNative + private int getStreamMaxVolume() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "getStreamMaxVolume"); + return audioManager.getStreamMaxVolume(AudioManager.STREAM_VOICE_CALL); + } + + // Set current volume level for a phone call audio stream. + @CalledByNative + private boolean setStreamVolume(int volume) { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "setStreamVolume(" + volume + ")"); + if (audioManager.isVolumeFixed()) { + Logging.e(TAG, "The device implements a fixed volume policy."); + return false; + } + audioManager.setStreamVolume(AudioManager.STREAM_VOICE_CALL, volume, 0); + return true; + } + + /** Get current volume level for a phone call audio stream. */ + @CalledByNative + private int getStreamVolume() { + threadChecker.checkIsOnValidThread(); + Logging.d(TAG, "getStreamVolume"); + return audioManager.getStreamVolume(AudioManager.STREAM_VOICE_CALL); + } + + @CalledByNative + private int GetPlayoutUnderrunCount() { + if (Build.VERSION.SDK_INT >= 24) { + if (audioTrack != null) { + return audioTrack.getUnderrunCount(); + } else { + return -1; + } + } else { + return -2; + } + } + + private void logMainParameters() { + Logging.d(TAG, + "AudioTrack: " + + "session ID: " + audioTrack.getAudioSessionId() + ", " + + "channels: " + audioTrack.getChannelCount() + ", " + + "sample rate: " + audioTrack.getSampleRate() + + ", " + // Gain (>=1.0) expressed as linear multiplier on sample values. + + "max gain: " + AudioTrack.getMaxVolume()); + } + + private static void logNativeOutputSampleRate(int requestedSampleRateInHz) { + final int nativeOutputSampleRate = + AudioTrack.getNativeOutputSampleRate(AudioManager.STREAM_VOICE_CALL); + Logging.d(TAG, "nativeOutputSampleRate: " + nativeOutputSampleRate); + if (requestedSampleRateInHz != nativeOutputSampleRate) { + Logging.w(TAG, "Unable to use fast mode since requested sample rate is not native"); + } + } + + private static AudioAttributes getAudioAttributes(@Nullable AudioAttributes overrideAttributes) { + AudioAttributes.Builder attributesBuilder = + new AudioAttributes.Builder() + .setUsage(DEFAULT_USAGE) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH); + + if (overrideAttributes != null) { + if (overrideAttributes.getUsage() != AudioAttributes.USAGE_UNKNOWN) { + attributesBuilder.setUsage(overrideAttributes.getUsage()); + } + if (overrideAttributes.getContentType() != AudioAttributes.CONTENT_TYPE_UNKNOWN) { + attributesBuilder.setContentType(overrideAttributes.getContentType()); + } + + attributesBuilder.setFlags(overrideAttributes.getFlags()); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + attributesBuilder = applyAttributesOnQOrHigher(attributesBuilder, overrideAttributes); + } + } + return attributesBuilder.build(); + } + + // Creates and AudioTrack instance using AudioAttributes and AudioFormat as input. + // It allows certain platforms or routing policies to use this information for more + // refined volume or routing decisions. + private static AudioTrack createAudioTrackBeforeOreo(int sampleRateInHz, int channelConfig, + int bufferSizeInBytes, @Nullable AudioAttributes overrideAttributes) { + Logging.d(TAG, "createAudioTrackBeforeOreo"); + logNativeOutputSampleRate(sampleRateInHz); + + // Create an audio track where the audio usage is for VoIP and the content type is speech. + return new AudioTrack(getAudioAttributes(overrideAttributes), + new AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRateInHz) + .setChannelMask(channelConfig) + .build(), + bufferSizeInBytes, AudioTrack.MODE_STREAM, AudioManager.AUDIO_SESSION_ID_GENERATE); + } + + // Creates and AudioTrack instance using AudioAttributes and AudioFormat as input. + // Use the low-latency mode to improve audio latency. Note that the low-latency mode may + // prevent effects (such as AEC) from working. Assuming AEC is working, the delay changes + // that happen in low-latency mode during the call will cause the AEC to perform worse. + // The behavior of the low-latency mode may be device dependent, use at your own risk. + @TargetApi(Build.VERSION_CODES.O) + private static AudioTrack createAudioTrackOnOreoOrHigher(int sampleRateInHz, int channelConfig, + int bufferSizeInBytes, @Nullable AudioAttributes overrideAttributes) { + Logging.d(TAG, "createAudioTrackOnOreoOrHigher"); + logNativeOutputSampleRate(sampleRateInHz); + + // Create an audio track where the audio usage is for VoIP and the content type is speech. + return new AudioTrack.Builder() + .setAudioAttributes(getAudioAttributes(overrideAttributes)) + .setAudioFormat(new AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRateInHz) + .setChannelMask(channelConfig) + .build()) + .setBufferSizeInBytes(bufferSizeInBytes) + .setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) + .setTransferMode(AudioTrack.MODE_STREAM) + .setSessionId(AudioManager.AUDIO_SESSION_ID_GENERATE) + .build(); + } + + @TargetApi(Build.VERSION_CODES.Q) + private static AudioAttributes.Builder applyAttributesOnQOrHigher( + AudioAttributes.Builder builder, AudioAttributes overrideAttributes) { + return builder.setAllowedCapturePolicy(overrideAttributes.getAllowedCapturePolicy()); + } + + private void logBufferSizeInFrames() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + Logging.d(TAG, + "AudioTrack: " + // The effective size of the AudioTrack buffer that the app writes to. + + "buffer size in frames: " + audioTrack.getBufferSizeInFrames()); + } + } + + @CalledByNative + private int getBufferSizeInFrames() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + return audioTrack.getBufferSizeInFrames(); + } + return -1; + } + + @CalledByNative + private int getInitialBufferSizeInFrames() { + return initialBufferSizeInFrames; + } + + private void logBufferCapacityInFrames() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + Logging.d(TAG, + "AudioTrack: " + // Maximum size of the AudioTrack buffer in frames. + + "buffer capacity in frames: " + audioTrack.getBufferCapacityInFrames()); + } + } + + private void logMainParametersExtended() { + logBufferSizeInFrames(); + logBufferCapacityInFrames(); + } + + // Prints the number of underrun occurrences in the application-level write + // buffer since the AudioTrack was created. An underrun occurs if the app does + // not write audio data quickly enough, causing the buffer to underflow and a + // potential audio glitch. + // TODO(henrika): keep track of this value in the field and possibly add new + // UMA stat if needed. + private void logUnderrunCount() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + Logging.d(TAG, "underrun count: " + audioTrack.getUnderrunCount()); + } + } + + // Helper method which throws an exception when an assertion has failed. + private static void assertTrue(boolean condition) { + if (!condition) { + throw new AssertionError("Expected condition to be true"); + } + } + + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); + } + + private static native void nativeCacheDirectBufferAddress( + long nativeAudioTrackJni, ByteBuffer byteBuffer); + private static native void nativeGetPlayoutData(long nativeAudioTrackJni, int bytes); + + // Sets all samples to be played out to zero if `mute` is true, i.e., + // ensures that the speaker is muted. + public void setSpeakerMute(boolean mute) { + Logging.w(TAG, "setSpeakerMute(" + mute + ")"); + speakerMute = mute; + } + + // Releases the native AudioTrack resources. + private void releaseAudioResources() { + Logging.d(TAG, "releaseAudioResources"); + if (audioTrack != null) { + audioTrack.release(); + audioTrack = null; + } + } + + private void reportWebRtcAudioTrackInitError(String errorMessage) { + Logging.e(TAG, "Init playout error: " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackInitError(errorMessage); + } + } + + private void reportWebRtcAudioTrackStartError( + AudioTrackStartErrorCode errorCode, String errorMessage) { + Logging.e(TAG, "Start playout error: " + errorCode + ". " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackStartError(errorCode, errorMessage); + } + } + + private void reportWebRtcAudioTrackError(String errorMessage) { + Logging.e(TAG, "Run-time playback error: " + errorMessage); + WebRtcAudioUtils.logAudioState(TAG, context, audioManager); + if (errorCallback != null) { + errorCallback.onWebRtcAudioTrackError(errorMessage); + } + } + + private void doAudioTrackStateCallback(int audioState) { + Logging.d(TAG, "doAudioTrackStateCallback: " + audioState); + if (stateCallback != null) { + if (audioState == WebRtcAudioTrack.AUDIO_TRACK_START) { + stateCallback.onWebRtcAudioTrackStart(); + } else if (audioState == WebRtcAudioTrack.AUDIO_TRACK_STOP) { + stateCallback.onWebRtcAudioTrackStop(); + } else { + Logging.e(TAG, "Invalid audio state"); + } + } + } +} diff --git a/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java new file mode 100644 index 0000000000..7b4b809ab1 --- /dev/null +++ b/third_party/libwebrtc/sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.audio; + +import static android.media.AudioManager.MODE_IN_CALL; +import static android.media.AudioManager.MODE_IN_COMMUNICATION; +import static android.media.AudioManager.MODE_NORMAL; +import static android.media.AudioManager.MODE_RINGTONE; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.Context; +import android.content.pm.PackageManager; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.MediaRecorder.AudioSource; +import android.os.Build; +import java.lang.Thread; +import java.util.Arrays; +import org.webrtc.Logging; + +final class WebRtcAudioUtils { + private static final String TAG = "WebRtcAudioUtilsExternal"; + + // Helper method for building a string of thread information. + public static String getThreadInfo() { + return "@[name=" + Thread.currentThread().getName() + ", id=" + Thread.currentThread().getId() + + "]"; + } + + // Returns true if we're running on emulator. + public static boolean runningOnEmulator() { + return Build.HARDWARE.equals("goldfish") && Build.BRAND.startsWith("generic_"); + } + + // Information about the current build, taken from system properties. + static void logDeviceInfo(String tag) { + Logging.d(tag, + "Android SDK: " + Build.VERSION.SDK_INT + ", " + + "Release: " + Build.VERSION.RELEASE + ", " + + "Brand: " + Build.BRAND + ", " + + "Device: " + Build.DEVICE + ", " + + "Id: " + Build.ID + ", " + + "Hardware: " + Build.HARDWARE + ", " + + "Manufacturer: " + Build.MANUFACTURER + ", " + + "Model: " + Build.MODEL + ", " + + "Product: " + Build.PRODUCT); + } + + // Logs information about the current audio state. The idea is to call this + // method when errors are detected to log under what conditions the error + // occurred. Hopefully it will provide clues to what might be the root cause. + static void logAudioState(String tag, Context context, AudioManager audioManager) { + logDeviceInfo(tag); + logAudioStateBasic(tag, context, audioManager); + logAudioStateVolume(tag, audioManager); + logAudioDeviceInfo(tag, audioManager); + } + + // Converts AudioDeviceInfo types to local string representation. + static String deviceTypeToString(int type) { + switch (type) { + case AudioDeviceInfo.TYPE_UNKNOWN: + return "TYPE_UNKNOWN"; + case AudioDeviceInfo.TYPE_BUILTIN_EARPIECE: + return "TYPE_BUILTIN_EARPIECE"; + case AudioDeviceInfo.TYPE_BUILTIN_SPEAKER: + return "TYPE_BUILTIN_SPEAKER"; + case AudioDeviceInfo.TYPE_WIRED_HEADSET: + return "TYPE_WIRED_HEADSET"; + case AudioDeviceInfo.TYPE_WIRED_HEADPHONES: + return "TYPE_WIRED_HEADPHONES"; + case AudioDeviceInfo.TYPE_LINE_ANALOG: + return "TYPE_LINE_ANALOG"; + case AudioDeviceInfo.TYPE_LINE_DIGITAL: + return "TYPE_LINE_DIGITAL"; + case AudioDeviceInfo.TYPE_BLUETOOTH_SCO: + return "TYPE_BLUETOOTH_SCO"; + case AudioDeviceInfo.TYPE_BLUETOOTH_A2DP: + return "TYPE_BLUETOOTH_A2DP"; + case AudioDeviceInfo.TYPE_HDMI: + return "TYPE_HDMI"; + case AudioDeviceInfo.TYPE_HDMI_ARC: + return "TYPE_HDMI_ARC"; + case AudioDeviceInfo.TYPE_USB_DEVICE: + return "TYPE_USB_DEVICE"; + case AudioDeviceInfo.TYPE_USB_ACCESSORY: + return "TYPE_USB_ACCESSORY"; + case AudioDeviceInfo.TYPE_DOCK: + return "TYPE_DOCK"; + case AudioDeviceInfo.TYPE_FM: + return "TYPE_FM"; + case AudioDeviceInfo.TYPE_BUILTIN_MIC: + return "TYPE_BUILTIN_MIC"; + case AudioDeviceInfo.TYPE_FM_TUNER: + return "TYPE_FM_TUNER"; + case AudioDeviceInfo.TYPE_TV_TUNER: + return "TYPE_TV_TUNER"; + case AudioDeviceInfo.TYPE_TELEPHONY: + return "TYPE_TELEPHONY"; + case AudioDeviceInfo.TYPE_AUX_LINE: + return "TYPE_AUX_LINE"; + case AudioDeviceInfo.TYPE_IP: + return "TYPE_IP"; + case AudioDeviceInfo.TYPE_BUS: + return "TYPE_BUS"; + case AudioDeviceInfo.TYPE_USB_HEADSET: + return "TYPE_USB_HEADSET"; + default: + return "TYPE_UNKNOWN"; + } + } + + @TargetApi(Build.VERSION_CODES.N) + public static String audioSourceToString(int source) { + // AudioSource.UNPROCESSED requires API level 29. Use local define instead. + final int VOICE_PERFORMANCE = 10; + switch (source) { + case AudioSource.DEFAULT: + return "DEFAULT"; + case AudioSource.MIC: + return "MIC"; + case AudioSource.VOICE_UPLINK: + return "VOICE_UPLINK"; + case AudioSource.VOICE_DOWNLINK: + return "VOICE_DOWNLINK"; + case AudioSource.VOICE_CALL: + return "VOICE_CALL"; + case AudioSource.CAMCORDER: + return "CAMCORDER"; + case AudioSource.VOICE_RECOGNITION: + return "VOICE_RECOGNITION"; + case AudioSource.VOICE_COMMUNICATION: + return "VOICE_COMMUNICATION"; + case AudioSource.UNPROCESSED: + return "UNPROCESSED"; + case VOICE_PERFORMANCE: + return "VOICE_PERFORMANCE"; + default: + return "INVALID"; + } + } + + public static String channelMaskToString(int mask) { + // For input or AudioRecord, the mask should be AudioFormat#CHANNEL_IN_MONO or + // AudioFormat#CHANNEL_IN_STEREO. AudioFormat#CHANNEL_IN_MONO is guaranteed to work on all + // devices. + switch (mask) { + case AudioFormat.CHANNEL_IN_STEREO: + return "IN_STEREO"; + case AudioFormat.CHANNEL_IN_MONO: + return "IN_MONO"; + default: + return "INVALID"; + } + } + + @TargetApi(Build.VERSION_CODES.N) + public static String audioEncodingToString(int enc) { + switch (enc) { + case AudioFormat.ENCODING_INVALID: + return "INVALID"; + case AudioFormat.ENCODING_PCM_16BIT: + return "PCM_16BIT"; + case AudioFormat.ENCODING_PCM_8BIT: + return "PCM_8BIT"; + case AudioFormat.ENCODING_PCM_FLOAT: + return "PCM_FLOAT"; + case AudioFormat.ENCODING_AC3: + return "AC3"; + case AudioFormat.ENCODING_E_AC3: + return "AC3"; + case AudioFormat.ENCODING_DTS: + return "DTS"; + case AudioFormat.ENCODING_DTS_HD: + return "DTS_HD"; + case AudioFormat.ENCODING_MP3: + return "MP3"; + default: + return "Invalid encoding: " + enc; + } + } + + // Reports basic audio statistics. + private static void logAudioStateBasic(String tag, Context context, AudioManager audioManager) { + Logging.d(tag, + "Audio State: " + + "audio mode: " + modeToString(audioManager.getMode()) + ", " + + "has mic: " + hasMicrophone(context) + ", " + + "mic muted: " + audioManager.isMicrophoneMute() + ", " + + "music active: " + audioManager.isMusicActive() + ", " + + "speakerphone: " + audioManager.isSpeakerphoneOn() + ", " + + "BT SCO: " + audioManager.isBluetoothScoOn()); + } + + // Adds volume information for all possible stream types. + private static void logAudioStateVolume(String tag, AudioManager audioManager) { + final int[] streams = {AudioManager.STREAM_VOICE_CALL, AudioManager.STREAM_MUSIC, + AudioManager.STREAM_RING, AudioManager.STREAM_ALARM, AudioManager.STREAM_NOTIFICATION, + AudioManager.STREAM_SYSTEM}; + Logging.d(tag, "Audio State: "); + // Some devices may not have volume controls and might use a fixed volume. + boolean fixedVolume = audioManager.isVolumeFixed(); + Logging.d(tag, " fixed volume=" + fixedVolume); + if (!fixedVolume) { + for (int stream : streams) { + StringBuilder info = new StringBuilder(); + info.append(" " + streamTypeToString(stream) + ": "); + info.append("volume=").append(audioManager.getStreamVolume(stream)); + info.append(", max=").append(audioManager.getStreamMaxVolume(stream)); + logIsStreamMute(tag, audioManager, stream, info); + Logging.d(tag, info.toString()); + } + } + } + + private static void logIsStreamMute( + String tag, AudioManager audioManager, int stream, StringBuilder info) { + if (Build.VERSION.SDK_INT >= 23) { + info.append(", muted=").append(audioManager.isStreamMute(stream)); + } + } + + // Moz linting complains even though AudioManager.GET_DEVICES_ALL is + // listed in the docs here: + // https://developer.android.com/reference/android/media/AudioManager#GET_DEVICES_ALL + @SuppressLint("WrongConstant") + private static void logAudioDeviceInfo(String tag, AudioManager audioManager) { + if (Build.VERSION.SDK_INT < 23) { + return; + } + final AudioDeviceInfo[] devices = audioManager.getDevices(AudioManager.GET_DEVICES_ALL); + if (devices.length == 0) { + return; + } + Logging.d(tag, "Audio Devices: "); + for (AudioDeviceInfo device : devices) { + StringBuilder info = new StringBuilder(); + info.append(" ").append(deviceTypeToString(device.getType())); + info.append(device.isSource() ? "(in): " : "(out): "); + // An empty array indicates that the device supports arbitrary channel counts. + if (device.getChannelCounts().length > 0) { + info.append("channels=").append(Arrays.toString(device.getChannelCounts())); + info.append(", "); + } + if (device.getEncodings().length > 0) { + // Examples: ENCODING_PCM_16BIT = 2, ENCODING_PCM_FLOAT = 4. + info.append("encodings=").append(Arrays.toString(device.getEncodings())); + info.append(", "); + } + if (device.getSampleRates().length > 0) { + info.append("sample rates=").append(Arrays.toString(device.getSampleRates())); + info.append(", "); + } + info.append("id=").append(device.getId()); + Logging.d(tag, info.toString()); + } + } + + // Converts media.AudioManager modes into local string representation. + static String modeToString(int mode) { + switch (mode) { + case MODE_IN_CALL: + return "MODE_IN_CALL"; + case MODE_IN_COMMUNICATION: + return "MODE_IN_COMMUNICATION"; + case MODE_NORMAL: + return "MODE_NORMAL"; + case MODE_RINGTONE: + return "MODE_RINGTONE"; + default: + return "MODE_INVALID"; + } + } + + private static String streamTypeToString(int stream) { + switch (stream) { + case AudioManager.STREAM_VOICE_CALL: + return "STREAM_VOICE_CALL"; + case AudioManager.STREAM_MUSIC: + return "STREAM_MUSIC"; + case AudioManager.STREAM_RING: + return "STREAM_RING"; + case AudioManager.STREAM_ALARM: + return "STREAM_ALARM"; + case AudioManager.STREAM_NOTIFICATION: + return "STREAM_NOTIFICATION"; + case AudioManager.STREAM_SYSTEM: + return "STREAM_SYSTEM"; + default: + return "STREAM_INVALID"; + } + } + + // Returns true if the device can record audio via a microphone. + private static boolean hasMicrophone(Context context) { + return context.getPackageManager().hasSystemFeature(PackageManager.FEATURE_MICROPHONE); + } +} diff --git a/third_party/libwebrtc/sdk/base_objc_gn/moz.build b/third_party/libwebrtc/sdk/base_objc_gn/moz.build new file mode 100644 index 0000000000..7b25d1f059 --- /dev/null +++ b/third_party/libwebrtc/sdk/base_objc_gn/moz.build @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +CMFLAGS += [ + "-fobjc-arc" +] + +CMMFLAGS += [ + "-fobjc-arc" +] + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MAC"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" +DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True +DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" +DEFINES["__STDC_CONSTANT_MACROS"] = True +DEFINES["__STDC_FORMAT_MACROS"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/sdk/objc/", + "/third_party/libwebrtc/sdk/objc/base/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.m", + "/third_party/libwebrtc/sdk/objc/base/RTCLogging.mm", + "/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.m", + "/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.m", + "/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.m", + "/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.m", + "/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.mm" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["_DEBUG"] = True + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +Library("base_objc_gn") diff --git a/third_party/libwebrtc/sdk/helpers_objc_gn/moz.build b/third_party/libwebrtc/sdk/helpers_objc_gn/moz.build new file mode 100644 index 0000000000..a910dc1cb5 --- /dev/null +++ b/third_party/libwebrtc/sdk/helpers_objc_gn/moz.build @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +CMFLAGS += [ + "-fobjc-arc" +] + +CMMFLAGS += [ + "-fobjc-arc" +] + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MAC"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" +DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True +DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" +DEFINES["__STDC_CONSTANT_MACROS"] = True +DEFINES["__STDC_FORMAT_MACROS"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/sdk/objc/", + "/third_party/libwebrtc/sdk/objc/base/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.mm", + "/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.mm", + "/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.m" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["_DEBUG"] = True + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +Library("helpers_objc_gn") diff --git a/third_party/libwebrtc/sdk/media_constraints.cc b/third_party/libwebrtc/sdk/media_constraints.cc new file mode 100644 index 0000000000..88261e7530 --- /dev/null +++ b/third_party/libwebrtc/sdk/media_constraints.cc @@ -0,0 +1,248 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/media_constraints.h" + +#include "absl/types/optional.h" +#include "api/peer_connection_interface.h" + +namespace webrtc { +namespace { + +// Find the highest-priority instance of the T-valued constraint named by +// `key` and return its value as `value`. `constraints` can be null. +// If `mandatory_constraints` is non-null, it is incremented if the key appears +// among the mandatory constraints. +// Returns true if the key was found and has a valid value for type T. +// If the key appears multiple times as an optional constraint, appearances +// after the first are ignored. +// Note: Because this uses FindFirst, repeated optional constraints whose +// first instance has an unrecognized value are not handled precisely in +// accordance with the specification. +template <typename T> +bool FindConstraint(const MediaConstraints* constraints, + const std::string& key, + T* value, + size_t* mandatory_constraints) { + std::string string_value; + if (!FindConstraint(constraints, key, &string_value, mandatory_constraints)) { + return false; + } + return rtc::FromString(string_value, value); +} + +// Specialization for std::string, since a string doesn't need conversion. +template <> +bool FindConstraint(const MediaConstraints* constraints, + const std::string& key, + std::string* value, + size_t* mandatory_constraints) { + if (!constraints) { + return false; + } + if (constraints->GetMandatory().FindFirst(key, value)) { + if (mandatory_constraints) { + ++*mandatory_constraints; + } + return true; + } + if (constraints->GetOptional().FindFirst(key, value)) { + return true; + } + return false; +} + +bool FindConstraint(const MediaConstraints* constraints, + const std::string& key, + bool* value, + size_t* mandatory_constraints) { + return FindConstraint<bool>(constraints, key, value, mandatory_constraints); +} + +bool FindConstraint(const MediaConstraints* constraints, + const std::string& key, + int* value, + size_t* mandatory_constraints) { + return FindConstraint<int>(constraints, key, value, mandatory_constraints); +} + +// Converts a constraint (mandatory takes precedence over optional) to an +// absl::optional. +template <typename T> +void ConstraintToOptional(const MediaConstraints* constraints, + const std::string& key, + absl::optional<T>* value_out) { + T value; + bool present = FindConstraint<T>(constraints, key, &value, nullptr); + if (present) { + *value_out = value; + } +} +} // namespace + +const char MediaConstraints::kValueTrue[] = "true"; +const char MediaConstraints::kValueFalse[] = "false"; + +// Constraints declared as static members in mediastreaminterface.h + +// Audio constraints. +const char MediaConstraints::kGoogEchoCancellation[] = "googEchoCancellation"; +const char MediaConstraints::kAutoGainControl[] = "googAutoGainControl"; +const char MediaConstraints::kNoiseSuppression[] = "googNoiseSuppression"; +const char MediaConstraints::kHighpassFilter[] = "googHighpassFilter"; +const char MediaConstraints::kAudioMirroring[] = "googAudioMirroring"; +const char MediaConstraints::kAudioNetworkAdaptorConfig[] = + "googAudioNetworkAdaptorConfig"; +const char MediaConstraints::kInitAudioRecordingOnSend[] = + "InitAudioRecordingOnSend"; + +// Constraint keys for CreateOffer / CreateAnswer defined in W3C specification. +const char MediaConstraints::kOfferToReceiveAudio[] = "OfferToReceiveAudio"; +const char MediaConstraints::kOfferToReceiveVideo[] = "OfferToReceiveVideo"; +const char MediaConstraints::kVoiceActivityDetection[] = + "VoiceActivityDetection"; +const char MediaConstraints::kIceRestart[] = "IceRestart"; +// Google specific constraint for BUNDLE enable/disable. +const char MediaConstraints::kUseRtpMux[] = "googUseRtpMUX"; + +// Below constraints should be used during PeerConnection construction. +// Google-specific constraint keys. +const char MediaConstraints::kEnableDscp[] = "googDscp"; +const char MediaConstraints::kEnableVideoSuspendBelowMinBitrate[] = + "googSuspendBelowMinBitrate"; +const char MediaConstraints::kScreencastMinBitrate[] = + "googScreencastMinBitrate"; +// TODO(ronghuawu): Remove once cpu overuse detection is stable. +const char MediaConstraints::kCpuOveruseDetection[] = "googCpuOveruseDetection"; + +const char MediaConstraints::kRawPacketizationForVideoEnabled[] = + "googRawPacketizationForVideoEnabled"; + +const char MediaConstraints::kNumSimulcastLayers[] = "googNumSimulcastLayers"; + +// Set `value` to the value associated with the first appearance of `key`, or +// return false if `key` is not found. +bool MediaConstraints::Constraints::FindFirst(const std::string& key, + std::string* value) const { + for (Constraints::const_iterator iter = begin(); iter != end(); ++iter) { + if (iter->key == key) { + *value = iter->value; + return true; + } + } + return false; +} + +void CopyConstraintsIntoRtcConfiguration( + const MediaConstraints* constraints, + PeerConnectionInterface::RTCConfiguration* configuration) { + // Copy info from constraints into configuration, if present. + if (!constraints) { + return; + } + + FindConstraint(constraints, MediaConstraints::kEnableDscp, + &configuration->media_config.enable_dscp, nullptr); + FindConstraint(constraints, MediaConstraints::kCpuOveruseDetection, + &configuration->media_config.video.enable_cpu_adaptation, + nullptr); + // Find Suspend Below Min Bitrate constraint. + FindConstraint( + constraints, MediaConstraints::kEnableVideoSuspendBelowMinBitrate, + &configuration->media_config.video.suspend_below_min_bitrate, nullptr); + ConstraintToOptional<int>(constraints, + MediaConstraints::kScreencastMinBitrate, + &configuration->screencast_min_bitrate); +} + +void CopyConstraintsIntoAudioOptions(const MediaConstraints* constraints, + cricket::AudioOptions* options) { + if (!constraints) { + return; + } + + ConstraintToOptional<bool>(constraints, + MediaConstraints::kGoogEchoCancellation, + &options->echo_cancellation); + ConstraintToOptional<bool>(constraints, MediaConstraints::kAutoGainControl, + &options->auto_gain_control); + ConstraintToOptional<bool>(constraints, MediaConstraints::kNoiseSuppression, + &options->noise_suppression); + ConstraintToOptional<bool>(constraints, MediaConstraints::kHighpassFilter, + &options->highpass_filter); + ConstraintToOptional<bool>(constraints, MediaConstraints::kAudioMirroring, + &options->stereo_swapping); + ConstraintToOptional<std::string>( + constraints, MediaConstraints::kAudioNetworkAdaptorConfig, + &options->audio_network_adaptor_config); + // When `kAudioNetworkAdaptorConfig` is defined, it both means that audio + // network adaptor is desired, and provides the config string. + if (options->audio_network_adaptor_config) { + options->audio_network_adaptor = true; + } + ConstraintToOptional<bool>(constraints, + MediaConstraints::kInitAudioRecordingOnSend, + &options->init_recording_on_send); +} + +bool CopyConstraintsIntoOfferAnswerOptions( + const MediaConstraints* constraints, + PeerConnectionInterface::RTCOfferAnswerOptions* offer_answer_options) { + if (!constraints) { + return true; + } + + bool value = false; + size_t mandatory_constraints_satisfied = 0; + + if (FindConstraint(constraints, MediaConstraints::kOfferToReceiveAudio, + &value, &mandatory_constraints_satisfied)) { + offer_answer_options->offer_to_receive_audio = + value ? PeerConnectionInterface::RTCOfferAnswerOptions:: + kOfferToReceiveMediaTrue + : 0; + } + + if (FindConstraint(constraints, MediaConstraints::kOfferToReceiveVideo, + &value, &mandatory_constraints_satisfied)) { + offer_answer_options->offer_to_receive_video = + value ? PeerConnectionInterface::RTCOfferAnswerOptions:: + kOfferToReceiveMediaTrue + : 0; + } + if (FindConstraint(constraints, MediaConstraints::kVoiceActivityDetection, + &value, &mandatory_constraints_satisfied)) { + offer_answer_options->voice_activity_detection = value; + } + if (FindConstraint(constraints, MediaConstraints::kUseRtpMux, &value, + &mandatory_constraints_satisfied)) { + offer_answer_options->use_rtp_mux = value; + } + if (FindConstraint(constraints, MediaConstraints::kIceRestart, &value, + &mandatory_constraints_satisfied)) { + offer_answer_options->ice_restart = value; + } + + if (FindConstraint(constraints, + MediaConstraints::kRawPacketizationForVideoEnabled, &value, + &mandatory_constraints_satisfied)) { + offer_answer_options->raw_packetization_for_video = value; + } + + int layers; + if (FindConstraint(constraints, MediaConstraints::kNumSimulcastLayers, + &layers, &mandatory_constraints_satisfied)) { + offer_answer_options->num_simulcast_layers = layers; + } + + return mandatory_constraints_satisfied == constraints->GetMandatory().size(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/media_constraints.h b/third_party/libwebrtc/sdk/media_constraints.h new file mode 100644 index 0000000000..5bd38c2d1a --- /dev/null +++ b/third_party/libwebrtc/sdk/media_constraints.h @@ -0,0 +1,129 @@ +/* + * Copyright 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Implementation of the w3c constraints spec is the responsibility of the +// browser. Chrome no longer uses the constraints api declared here, and it will +// be removed from WebRTC. +// https://bugs.chromium.org/p/webrtc/issues/detail?id=9239 + +#ifndef SDK_MEDIA_CONSTRAINTS_H_ +#define SDK_MEDIA_CONSTRAINTS_H_ + +#include <stddef.h> + +#include <string> +#include <utility> +#include <vector> + +#include "api/audio_options.h" +#include "api/peer_connection_interface.h" + +namespace webrtc { + +// Class representing constraints, as used by the android and objc apis. +// +// Constraints may be either "mandatory", which means that unless satisfied, +// the method taking the constraints should fail, or "optional", which means +// they may not be satisfied.. +class MediaConstraints { + public: + struct Constraint { + Constraint() {} + Constraint(const std::string& key, const std::string value) + : key(key), value(value) {} + std::string key; + std::string value; + }; + + class Constraints : public std::vector<Constraint> { + public: + Constraints() = default; + Constraints(std::initializer_list<Constraint> l) + : std::vector<Constraint>(l) {} + + bool FindFirst(const std::string& key, std::string* value) const; + }; + + MediaConstraints() = default; + MediaConstraints(Constraints mandatory, Constraints optional) + : mandatory_(std::move(mandatory)), optional_(std::move(optional)) {} + + // Constraint keys used by a local audio source. + + // These keys are google specific. + static const char kGoogEchoCancellation[]; // googEchoCancellation + + static const char kAutoGainControl[]; // googAutoGainControl + static const char kNoiseSuppression[]; // googNoiseSuppression + static const char kHighpassFilter[]; // googHighpassFilter + static const char kAudioMirroring[]; // googAudioMirroring + static const char + kAudioNetworkAdaptorConfig[]; // googAudioNetworkAdaptorConfig + static const char kInitAudioRecordingOnSend[]; // InitAudioRecordingOnSend; + + // Constraint keys for CreateOffer / CreateAnswer + // Specified by the W3C PeerConnection spec + static const char kOfferToReceiveVideo[]; // OfferToReceiveVideo + static const char kOfferToReceiveAudio[]; // OfferToReceiveAudio + static const char kVoiceActivityDetection[]; // VoiceActivityDetection + static const char kIceRestart[]; // IceRestart + // These keys are google specific. + static const char kUseRtpMux[]; // googUseRtpMUX + + // Constraints values. + static const char kValueTrue[]; // true + static const char kValueFalse[]; // false + + // PeerConnection constraint keys. + // Google-specific constraint keys. + // Temporary pseudo-constraint for enabling DSCP through JS. + static const char kEnableDscp[]; // googDscp + // Constraint to enable IPv6 through JS. + static const char kEnableIPv6[]; // googIPv6 + // Temporary constraint to enable suspend below min bitrate feature. + static const char kEnableVideoSuspendBelowMinBitrate[]; + static const char kScreencastMinBitrate[]; // googScreencastMinBitrate + static const char kCpuOveruseDetection[]; // googCpuOveruseDetection + + // Constraint to enable negotiating raw RTP packetization using attribute + // "a=packetization:<payload_type> raw" in the SDP for all video payload. + static const char kRawPacketizationForVideoEnabled[]; + + // Specifies number of simulcast layers for all video tracks + // with a Plan B offer/answer + // (see RTCOfferAnswerOptions::num_simulcast_layers). + static const char kNumSimulcastLayers[]; + + ~MediaConstraints() = default; + + const Constraints& GetMandatory() const { return mandatory_; } + const Constraints& GetOptional() const { return optional_; } + + private: + const Constraints mandatory_ = {}; + const Constraints optional_ = {}; +}; + +// Copy all relevant constraints into an RTCConfiguration object. +void CopyConstraintsIntoRtcConfiguration( + const MediaConstraints* constraints, + PeerConnectionInterface::RTCConfiguration* configuration); + +// Copy all relevant constraints into an AudioOptions object. +void CopyConstraintsIntoAudioOptions(const MediaConstraints* constraints, + cricket::AudioOptions* options); + +bool CopyConstraintsIntoOfferAnswerOptions( + const MediaConstraints* constraints, + PeerConnectionInterface::RTCOfferAnswerOptions* offer_answer_options); + +} // namespace webrtc + +#endif // SDK_MEDIA_CONSTRAINTS_H_ diff --git a/third_party/libwebrtc/sdk/media_constraints_unittest.cc b/third_party/libwebrtc/sdk/media_constraints_unittest.cc new file mode 100644 index 0000000000..5e6b157e3a --- /dev/null +++ b/third_party/libwebrtc/sdk/media_constraints_unittest.cc @@ -0,0 +1,55 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/media_constraints.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Checks all settings touched by CopyConstraintsIntoRtcConfiguration, +// plus audio_jitter_buffer_max_packets. +bool Matches(const PeerConnectionInterface::RTCConfiguration& a, + const PeerConnectionInterface::RTCConfiguration& b) { + return a.audio_jitter_buffer_max_packets == + b.audio_jitter_buffer_max_packets && + a.screencast_min_bitrate == b.screencast_min_bitrate && + a.media_config == b.media_config; +} + +TEST(MediaConstraints, CopyConstraintsIntoRtcConfiguration) { + const MediaConstraints constraints_empty; + PeerConnectionInterface::RTCConfiguration old_configuration; + PeerConnectionInterface::RTCConfiguration configuration; + + CopyConstraintsIntoRtcConfiguration(&constraints_empty, &configuration); + EXPECT_TRUE(Matches(old_configuration, configuration)); + + const MediaConstraints constraints_screencast( + {MediaConstraints::Constraint(MediaConstraints::kScreencastMinBitrate, + "27")}, + {}); + CopyConstraintsIntoRtcConfiguration(&constraints_screencast, &configuration); + EXPECT_TRUE(configuration.screencast_min_bitrate); + EXPECT_EQ(27, *(configuration.screencast_min_bitrate)); + + // An empty set of constraints will not overwrite + // values that are already present. + configuration = old_configuration; + configuration.audio_jitter_buffer_max_packets = 34; + CopyConstraintsIntoRtcConfiguration(&constraints_empty, &configuration); + EXPECT_EQ(34, configuration.audio_jitter_buffer_max_packets); +} + +} // namespace + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/DEPS b/third_party/libwebrtc/sdk/objc/DEPS new file mode 100644 index 0000000000..4cff92caf1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/DEPS @@ -0,0 +1,18 @@ +include_rules = [ + "+base", + "+components", + "+helpers", + "+sdk", + "+common_video/h264", + "+common_video/include", + "+common_video/libyuv/include", + "+logging/rtc_event_log/rtc_event_log_factory.h", + "+media", + "+modules/video_coding", + "+pc", + "+system_wrappers", + "+modules/audio_device", + "+modules/audio_processing", + "+native", + "+third_party/libyuv", +] diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/NSString+StdString.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/NSString+StdString.h new file mode 100644 index 0000000000..3ec1b613ef --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/NSString+StdString.h @@ -0,0 +1,11 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "helpers/NSString+StdString.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/scoped_cftyperef.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/scoped_cftyperef.h new file mode 100644 index 0000000000..e5e376b0bc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/Common/scoped_cftyperef.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "helpers/scoped_cftyperef.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration+Native.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration+Native.h new file mode 100644 index 0000000000..529aa8dcf5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration+Native.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "api/peerconnection/RTCConfiguration+Native.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory+Native.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory+Native.h new file mode 100644 index 0000000000..222e06ef33 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnectionFactory+Native.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "api/peerconnection/RTCPeerConnectionFactory+Native.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h new file mode 100644 index 0000000000..136d7003c6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCDefaultShader.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "components/renderer/opengl/RTCDefaultShader.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h new file mode 100644 index 0000000000..4ba1caa41d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "components/renderer/opengl/RTCNV12TextureCache.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h b/third_party/libwebrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h new file mode 100644 index 0000000000..21281f36ac --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "components/video_codec/nalu_rewriter.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/api/audio_device_module.h b/third_party/libwebrtc/sdk/objc/Framework/Native/api/audio_device_module.h new file mode 100644 index 0000000000..7b448024de --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/api/audio_device_module.h @@ -0,0 +1,11 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/api/audio_device_module.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_decoder_factory.h b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_decoder_factory.h new file mode 100644 index 0000000000..ca9371c54d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_decoder_factory.h @@ -0,0 +1,11 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/api/video_decoder_factory.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_encoder_factory.h b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_encoder_factory.h new file mode 100644 index 0000000000..35e1e6c99f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_encoder_factory.h @@ -0,0 +1,11 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/api/video_encoder_factory.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_frame_buffer.h b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_frame_buffer.h new file mode 100644 index 0000000000..0e862cfa07 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/api/video_frame_buffer.h @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/api/video_frame_buffer.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_decoder_factory.h b/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_decoder_factory.h new file mode 100644 index 0000000000..bd8513c342 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_decoder_factory.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/src/objc_video_decoder_factory.h" diff --git a/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_encoder_factory.h b/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_encoder_factory.h new file mode 100644 index 0000000000..b6bd650a4f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Framework/Native/src/objc_video_encoder_factory.h @@ -0,0 +1,11 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "native/src/objc_video_encoder_factory.h" diff --git a/third_party/libwebrtc/sdk/objc/Info.plist b/third_party/libwebrtc/sdk/objc/Info.plist new file mode 100644 index 0000000000..38c437e7fe --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/Info.plist @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>en</string> + <key>CFBundleExecutable</key> + <string>WebRTC</string> + <key>CFBundleIdentifier</key> + <string>org.webrtc.WebRTC</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleName</key> + <string>WebRTC</string> + <key>CFBundlePackageType</key> + <string>FMWK</string> + <key>CFBundleShortVersionString</key> + <string>1.0</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleVersion</key> + <string>1.0</string> + <key>NSPrincipalClass</key> + <string></string> +</dict> +</plist> diff --git a/third_party/libwebrtc/sdk/objc/OWNERS b/third_party/libwebrtc/sdk/objc/OWNERS new file mode 100644 index 0000000000..6af9062b2d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/OWNERS @@ -0,0 +1,9 @@ +# Normal code changes. +kthelgason@webrtc.org +andersc@webrtc.org +peterhanspers@webrtc.org +denicija@webrtc.org + +# Rubberstamps of e.g. reverts and critical bug fixes. +magjed@webrtc.org +tkchin@webrtc.org diff --git a/third_party/libwebrtc/sdk/objc/README.md b/third_party/libwebrtc/sdk/objc/README.md new file mode 100644 index 0000000000..ff294a266f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/README.md @@ -0,0 +1,37 @@ +# WebRTC Obj-C SDK + +This directory contains the Obj-C SDK for WebRTC. This includes wrappers for the +C++ PeerConnection API and some platform specific components for iOS and macOS. + +## Organization + +- api/ + + Wrappers around classes and functions in the C++ API for creating and + configuring peer connections, etc. + +- base/ + + This directory contains some base protocols and classes that are used by both + the platform specific components and the SDK wrappers. + +- components/ + + These are the platform specific components. Contains components for handling + audio, capturing and rendering video, encoding and decoding using the + platform's hardware codec implementation and for representing video frames + in the platform's native format. + +- helpers/ + + These files are not WebRTC specific, but are general helper classes and + utilities for the Cocoa platforms. + +- native/ + + APIs for wrapping the platform specific components and using them with the + C++ API. + +- unittests/ + + This directory contains the tests. diff --git a/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter+Private.h b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter+Private.h new file mode 100644 index 0000000000..9b123d2d05 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter+Private.h @@ -0,0 +1,41 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoRendererAdapter.h" + +#import "base/RTCVideoRenderer.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCVideoRendererAdapter () + +/** + * The Objective-C video renderer passed to this adapter during construction. + * Calls made to the webrtc::VideoRenderInterface will be adapted and passed to + * this video renderer. + */ +@property(nonatomic, readonly) id<RTC_OBJC_TYPE(RTCVideoRenderer)> videoRenderer; + +/** + * The native VideoSinkInterface surface exposed by this adapter. Calls made + * to this interface will be adapted and passed to the RTCVideoRenderer supplied + * during construction. This pointer is unsafe and owned by this class. + */ +@property(nonatomic, readonly) rtc::VideoSinkInterface<webrtc::VideoFrame> *nativeVideoRenderer; + +/** Initialize an RTCVideoRendererAdapter with an RTCVideoRenderer. */ +- (instancetype)initWithNativeRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)videoRenderer + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.h b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.h new file mode 100644 index 0000000000..b0b6f04488 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.h @@ -0,0 +1,27 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +NS_ASSUME_NONNULL_BEGIN + +/* + * Creates a rtc::VideoSinkInterface surface for an RTCVideoRenderer. The + * rtc::VideoSinkInterface is used by WebRTC rendering code - this + * adapter adapts calls made to that interface to the RTCVideoRenderer supplied + * during construction. + */ +@interface RTCVideoRendererAdapter : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.mm b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.mm new file mode 100644 index 0000000000..ef02f72f60 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/RTCVideoRendererAdapter.mm @@ -0,0 +1,67 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoRendererAdapter+Private.h" +#import "base/RTCVideoFrame.h" + +#include <memory> + +#include "sdk/objc/native/api/video_frame.h" + +namespace webrtc { + +class VideoRendererAdapter + : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + VideoRendererAdapter(RTCVideoRendererAdapter* adapter) { + adapter_ = adapter; + size_ = CGSizeZero; + } + + void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { + RTC_OBJC_TYPE(RTCVideoFrame)* videoFrame = NativeToObjCVideoFrame(nativeVideoFrame); + + CGSize current_size = (videoFrame.rotation % 180 == 0) + ? CGSizeMake(videoFrame.width, videoFrame.height) + : CGSizeMake(videoFrame.height, videoFrame.width); + + if (!CGSizeEqualToSize(size_, current_size)) { + size_ = current_size; + [adapter_.videoRenderer setSize:size_]; + } + [adapter_.videoRenderer renderFrame:videoFrame]; + } + + private: + __weak RTCVideoRendererAdapter *adapter_; + CGSize size_; +}; +} + +@implementation RTCVideoRendererAdapter { + std::unique_ptr<webrtc::VideoRendererAdapter> _adapter; +} + +@synthesize videoRenderer = _videoRenderer; + +- (instancetype)initWithNativeRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)videoRenderer { + NSParameterAssert(videoRenderer); + if (self = [super init]) { + _videoRenderer = videoRenderer; + _adapter.reset(new webrtc::VideoRendererAdapter(self)); + } + return self; +} + +- (rtc::VideoSinkInterface<webrtc::VideoFrame> *)nativeVideoRenderer { + return _adapter.get(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.h b/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.h new file mode 100644 index 0000000000..1d178b6d49 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.h @@ -0,0 +1,41 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCLogging.h" +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef void (^RTCCallbackLoggerMessageHandler)(NSString *message); +typedef void (^RTCCallbackLoggerMessageAndSeverityHandler)(NSString *message, + RTCLoggingSeverity severity); + +// This class intercepts WebRTC logs and forwards them to a registered block. +// This class is not threadsafe. +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCallbackLogger) : NSObject + +// The severity level to capture. The default is kRTCLoggingSeverityInfo. +@property(nonatomic, assign) RTCLoggingSeverity severity; + +// The callback handler will be called on the same thread that does the +// logging, so if the logging callback can be slow it may be a good idea +// to implement dispatching to some other queue. +- (void)start:(nullable RTCCallbackLoggerMessageHandler)handler; +- (void)startWithMessageAndSeverityHandler: + (nullable RTCCallbackLoggerMessageAndSeverityHandler)handler; + +- (void)stop; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.mm b/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.mm new file mode 100644 index 0000000000..ba6fe1b1cc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/logging/RTCCallbackLogger.mm @@ -0,0 +1,151 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCallbackLogger.h" + +#import "helpers/NSString+StdString.h" + +#include <memory> + +#include "absl/strings/string_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/log_sinks.h" +#include "rtc_base/logging.h" + +namespace { + +class CallbackLogSink final : public rtc::LogSink { + public: + CallbackLogSink(RTCCallbackLoggerMessageHandler callbackHandler) + : callback_handler_(callbackHandler) {} + + void OnLogMessage(const std::string& message) override { + OnLogMessage(absl::string_view(message)); + } + + void OnLogMessage(absl::string_view message) override { + if (callback_handler_) { + callback_handler_([NSString stringForAbslStringView:message]); + } + } + + private: + RTCCallbackLoggerMessageHandler callback_handler_; +}; + +class CallbackWithSeverityLogSink final : public rtc::LogSink { + public: + CallbackWithSeverityLogSink(RTCCallbackLoggerMessageAndSeverityHandler callbackHandler) + : callback_handler_(callbackHandler) {} + + void OnLogMessage(const std::string& message) override { RTC_DCHECK_NOTREACHED(); } + + void OnLogMessage(const std::string& message, rtc::LoggingSeverity severity) override { + OnLogMessage(absl::string_view(message), severity); + } + + void OnLogMessage(absl::string_view message, rtc::LoggingSeverity severity) override { + if (callback_handler_) { + RTCLoggingSeverity loggingSeverity = NativeSeverityToObjcSeverity(severity); + callback_handler_([NSString stringForAbslStringView:message], loggingSeverity); + } + } + + private: + static RTCLoggingSeverity NativeSeverityToObjcSeverity(rtc::LoggingSeverity severity) { + switch (severity) { + case rtc::LS_VERBOSE: + return RTCLoggingSeverityVerbose; + case rtc::LS_INFO: + return RTCLoggingSeverityInfo; + case rtc::LS_WARNING: + return RTCLoggingSeverityWarning; + case rtc::LS_ERROR: + return RTCLoggingSeverityError; + case rtc::LS_NONE: + return RTCLoggingSeverityNone; + } + } + + RTCCallbackLoggerMessageAndSeverityHandler callback_handler_; +}; + +} + +@implementation RTC_OBJC_TYPE (RTCCallbackLogger) { + BOOL _hasStarted; + std::unique_ptr<rtc::LogSink> _logSink; +} + +@synthesize severity = _severity; + +- (instancetype)init { + self = [super init]; + if (self != nil) { + _severity = RTCLoggingSeverityInfo; + } + return self; +} + +- (void)dealloc { + [self stop]; +} + +- (void)start:(nullable RTCCallbackLoggerMessageHandler)handler { + if (_hasStarted) { + return; + } + + _logSink.reset(new CallbackLogSink(handler)); + + rtc::LogMessage::AddLogToStream(_logSink.get(), [self rtcSeverity]); + _hasStarted = YES; +} + +- (void)startWithMessageAndSeverityHandler: + (nullable RTCCallbackLoggerMessageAndSeverityHandler)handler { + if (_hasStarted) { + return; + } + + _logSink.reset(new CallbackWithSeverityLogSink(handler)); + + rtc::LogMessage::AddLogToStream(_logSink.get(), [self rtcSeverity]); + _hasStarted = YES; +} + +- (void)stop { + if (!_hasStarted) { + return; + } + RTC_DCHECK(_logSink); + rtc::LogMessage::RemoveLogToStream(_logSink.get()); + _hasStarted = NO; + _logSink.reset(); +} + +#pragma mark - Private + +- (rtc::LoggingSeverity)rtcSeverity { + switch (_severity) { + case RTCLoggingSeverityVerbose: + return rtc::LS_VERBOSE; + case RTCLoggingSeverityInfo: + return rtc::LS_INFO; + case RTCLoggingSeverityWarning: + return rtc::LS_WARNING; + case RTCLoggingSeverityError: + return rtc::LS_ERROR; + case RTCLoggingSeverityNone: + return rtc::LS_NONE; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource+Private.h new file mode 100644 index 0000000000..2c333f9d73 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource+Private.h @@ -0,0 +1,34 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSource.h" + +#import "RTCMediaSource+Private.h" + +@interface RTC_OBJC_TYPE (RTCAudioSource) +() + + /** + * The AudioSourceInterface object passed to this RTCAudioSource during + * construction. + */ + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::AudioSourceInterface> nativeAudioSource; + +/** Initialize an RTCAudioSource from a native AudioSourceInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeAudioSource:(rtc::scoped_refptr<webrtc::AudioSourceInterface>)nativeAudioSource + NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type NS_UNAVAILABLE; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.h new file mode 100644 index 0000000000..9272fdf2d8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.h @@ -0,0 +1,32 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCMediaSource.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioSource) : RTC_OBJC_TYPE(RTCMediaSource) + +- (instancetype)init NS_UNAVAILABLE; + +// Sets the volume for the RTCMediaSource. `volume` is a gain value in the range +// [0, 10]. +// Temporary fix to be able to modify volume of remote audio tracks. +// TODO(kthelgason): Property stays here temporarily until a proper volume-api +// is available on the surface exposed by webrtc. +@property(nonatomic, assign) double volume; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.mm new file mode 100644 index 0000000000..1541045099 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioSource.mm @@ -0,0 +1,52 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSource+Private.h" + +#include "rtc_base/checks.h" + +@implementation RTC_OBJC_TYPE (RTCAudioSource) { +} + +@synthesize volume = _volume; +@synthesize nativeAudioSource = _nativeAudioSource; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeAudioSource: + (rtc::scoped_refptr<webrtc::AudioSourceInterface>)nativeAudioSource { + RTC_DCHECK(factory); + RTC_DCHECK(nativeAudioSource); + + if (self = [super initWithFactory:factory + nativeMediaSource:nativeAudioSource + type:RTCMediaSourceTypeAudio]) { + _nativeAudioSource = nativeAudioSource; + } + return self; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type { + RTC_DCHECK_NOTREACHED(); + return nil; +} + +- (NSString *)description { + NSString *stateString = [[self class] stringForState:self.state]; + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCAudioSource)( %p ): %@", self, stateString]; +} + +- (void)setVolume:(double)volume { + _volume = volume; + _nativeAudioSource->SetVolume(volume); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h new file mode 100644 index 0000000000..6495500484 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioTrack.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); +@interface RTC_OBJC_TYPE (RTCAudioTrack) +() + + /** AudioTrackInterface created or passed in at construction. */ + @property(nonatomic, readonly) rtc::scoped_refptr<webrtc::AudioTrackInterface> nativeAudioTrack; + +/** Initialize an RTCAudioTrack with an id. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + source:(RTC_OBJC_TYPE(RTCAudioSource) *)source + trackId:(NSString *)trackId; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.h new file mode 100644 index 0000000000..95eb5d3d48 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -0,0 +1,28 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMacros.h" +#import "RTCMediaStreamTrack.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioSource); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioTrack) : RTC_OBJC_TYPE(RTCMediaStreamTrack) + +- (instancetype)init NS_UNAVAILABLE; + +/** The audio source for this audio track. */ +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.mm new file mode 100644 index 0000000000..5c1736f436 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -0,0 +1,67 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioTrack+Private.h" + +#import "RTCAudioSource+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "helpers/NSString+StdString.h" + +#include "rtc_base/checks.h" + +@implementation RTC_OBJC_TYPE (RTCAudioTrack) + +@synthesize source = _source; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + source:(RTC_OBJC_TYPE(RTCAudioSource) *)source + trackId:(NSString *)trackId { + RTC_DCHECK(factory); + RTC_DCHECK(source); + RTC_DCHECK(trackId.length); + + std::string nativeId = [NSString stdStringForString:trackId]; + rtc::scoped_refptr<webrtc::AudioTrackInterface> track = + factory.nativeFactory->CreateAudioTrack(nativeId, source.nativeAudioSource.get()); + if (self = [self initWithFactory:factory nativeTrack:track type:RTCMediaStreamTrackTypeAudio]) { + _source = source; + } + return self; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack + type:(RTCMediaStreamTrackType)type { + NSParameterAssert(factory); + NSParameterAssert(nativeTrack); + NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); + return [super initWithFactory:factory nativeTrack:nativeTrack type:type]; +} + +- (RTC_OBJC_TYPE(RTCAudioSource) *)source { + if (!_source) { + rtc::scoped_refptr<webrtc::AudioSourceInterface> source(self.nativeAudioTrack->GetSource()); + if (source) { + _source = [[RTC_OBJC_TYPE(RTCAudioSource) alloc] initWithFactory:self.factory + nativeAudioSource:source]; + } + } + return _source; +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::AudioTrackInterface>)nativeAudioTrack { + return rtc::scoped_refptr<webrtc::AudioTrackInterface>( + static_cast<webrtc::AudioTrackInterface *>(self.nativeTrack.get())); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.h new file mode 100644 index 0000000000..5ac8984d4a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.h @@ -0,0 +1,44 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCertificate) : NSObject <NSCopying> + +/** Private key in PEM. */ +@property(nonatomic, readonly, copy) NSString *private_key; + +/** Public key in an x509 cert encoded in PEM. */ +@property(nonatomic, readonly, copy) NSString *certificate; + +/** + * Initialize an RTCCertificate with PEM strings for private_key and certificate. + */ +- (instancetype)initWithPrivateKey:(NSString *)private_key + certificate:(NSString *)certificate NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + +/** Generate a new certificate for 're' use. + * + * Optional dictionary of parameters. Defaults to KeyType ECDSA if none are + * provided. + * - name: "ECDSA" or "RSASSA-PKCS1-v1_5" + */ ++ (nullable RTC_OBJC_TYPE(RTCCertificate) *)generateCertificateWithParams:(NSDictionary *)params; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.mm new file mode 100644 index 0000000000..e5c33e407c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCertificate.mm @@ -0,0 +1,72 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCertificate.h" + +#import "base/RTCLogging.h" + +#include "rtc_base/logging.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_identity.h" + +@implementation RTC_OBJC_TYPE (RTCCertificate) + +@synthesize private_key = _private_key; +@synthesize certificate = _certificate; + +- (id)copyWithZone:(NSZone *)zone { + id copy = [[[self class] alloc] initWithPrivateKey:[self.private_key copyWithZone:zone] + certificate:[self.certificate copyWithZone:zone]]; + return copy; +} + +- (instancetype)initWithPrivateKey:(NSString *)private_key certificate:(NSString *)certificate { + if (self = [super init]) { + _private_key = [private_key copy]; + _certificate = [certificate copy]; + } + return self; +} + ++ (nullable RTC_OBJC_TYPE(RTCCertificate) *)generateCertificateWithParams:(NSDictionary *)params { + rtc::KeyType keyType = rtc::KT_ECDSA; + NSString *keyTypeString = [params valueForKey:@"name"]; + if (keyTypeString && [keyTypeString isEqualToString:@"RSASSA-PKCS1-v1_5"]) { + keyType = rtc::KT_RSA; + } + + NSNumber *expires = [params valueForKey:@"expires"]; + rtc::scoped_refptr<rtc::RTCCertificate> cc_certificate = nullptr; + if (expires != nil) { + uint64_t expirationTimestamp = [expires unsignedLongLongValue]; + cc_certificate = rtc::RTCCertificateGenerator::GenerateCertificate(rtc::KeyParams(keyType), + expirationTimestamp); + } else { + cc_certificate = + rtc::RTCCertificateGenerator::GenerateCertificate(rtc::KeyParams(keyType), absl::nullopt); + } + if (!cc_certificate) { + RTCLogError(@"Failed to generate certificate."); + return nullptr; + } + // grab PEMs and create an NS RTCCerticicate + rtc::RTCCertificatePEM pem = cc_certificate->ToPEM(); + std::string pem_private_key = pem.private_key(); + std::string pem_certificate = pem.certificate(); + RTC_LOG(LS_INFO) << "CERT PEM "; + RTC_LOG(LS_INFO) << pem_certificate; + + RTC_OBJC_TYPE(RTCCertificate) *cert = + [[RTC_OBJC_TYPE(RTCCertificate) alloc] initWithPrivateKey:@(pem_private_key.c_str()) + certificate:@(pem_certificate.c_str())]; + return cert; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Native.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Native.h new file mode 100644 index 0000000000..07c0da6041 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Native.h @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCConfiguration.h" + +#include "api/peer_connection_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCConfiguration) +() + + /** Optional TurnCustomizer. + * With this class one can modify outgoing TURN messages. + * The object passed in must remain valid until PeerConnection::Close() is + * called. + */ + @property(nonatomic, nullable) webrtc::TurnCustomizer* turnCustomizer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Private.h new file mode 100644 index 0000000000..6ad780acdc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration+Private.h @@ -0,0 +1,79 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCConfiguration.h" + +#include "api/peer_connection_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCConfiguration) +() + + + (webrtc::PeerConnectionInterface::IceTransportsType)nativeTransportsTypeForTransportPolicy + : (RTCIceTransportPolicy)policy; + ++ (RTCIceTransportPolicy)transportPolicyForTransportsType: + (webrtc::PeerConnectionInterface::IceTransportsType)nativeType; + ++ (NSString *)stringForTransportPolicy:(RTCIceTransportPolicy)policy; + ++ (webrtc::PeerConnectionInterface::BundlePolicy)nativeBundlePolicyForPolicy: + (RTCBundlePolicy)policy; + ++ (RTCBundlePolicy)bundlePolicyForNativePolicy: + (webrtc::PeerConnectionInterface::BundlePolicy)nativePolicy; + ++ (NSString *)stringForBundlePolicy:(RTCBundlePolicy)policy; + ++ (webrtc::PeerConnectionInterface::RtcpMuxPolicy)nativeRtcpMuxPolicyForPolicy: + (RTCRtcpMuxPolicy)policy; + ++ (RTCRtcpMuxPolicy)rtcpMuxPolicyForNativePolicy: + (webrtc::PeerConnectionInterface::RtcpMuxPolicy)nativePolicy; + ++ (NSString *)stringForRtcpMuxPolicy:(RTCRtcpMuxPolicy)policy; + ++ (webrtc::PeerConnectionInterface::TcpCandidatePolicy)nativeTcpCandidatePolicyForPolicy: + (RTCTcpCandidatePolicy)policy; + ++ (RTCTcpCandidatePolicy)tcpCandidatePolicyForNativePolicy: + (webrtc::PeerConnectionInterface::TcpCandidatePolicy)nativePolicy; + ++ (NSString *)stringForTcpCandidatePolicy:(RTCTcpCandidatePolicy)policy; + ++ (webrtc::PeerConnectionInterface::CandidateNetworkPolicy)nativeCandidateNetworkPolicyForPolicy: + (RTCCandidateNetworkPolicy)policy; + ++ (RTCCandidateNetworkPolicy)candidateNetworkPolicyForNativePolicy: + (webrtc::PeerConnectionInterface::CandidateNetworkPolicy)nativePolicy; + ++ (NSString *)stringForCandidateNetworkPolicy:(RTCCandidateNetworkPolicy)policy; + ++ (rtc::KeyType)nativeEncryptionKeyTypeForKeyType:(RTCEncryptionKeyType)keyType; + ++ (webrtc::SdpSemantics)nativeSdpSemanticsForSdpSemantics:(RTCSdpSemantics)sdpSemantics; + ++ (RTCSdpSemantics)sdpSemanticsForNativeSdpSemantics:(webrtc::SdpSemantics)sdpSemantics; + ++ (NSString *)stringForSdpSemantics:(RTCSdpSemantics)sdpSemantics; + +/** + * RTCConfiguration struct representation of this RTCConfiguration. + * This is needed to pass to the underlying C++ APIs. + */ +- (nullable webrtc::PeerConnectionInterface::RTCConfiguration *)createNativeConfiguration; + +- (instancetype)initWithNativeConfiguration: + (const webrtc::PeerConnectionInterface::RTCConfiguration &)config NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.h new file mode 100644 index 0000000000..011eaa613d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.h @@ -0,0 +1,262 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCertificate.h" +#import "RTCCryptoOptions.h" +#import "RTCMacros.h" + +@class RTC_OBJC_TYPE(RTCIceServer); + +/** + * Represents the ice transport policy. This exposes the same states in C++, + * which include one more state than what exists in the W3C spec. + */ +typedef NS_ENUM(NSInteger, RTCIceTransportPolicy) { + RTCIceTransportPolicyNone, + RTCIceTransportPolicyRelay, + RTCIceTransportPolicyNoHost, + RTCIceTransportPolicyAll +}; + +/** Represents the bundle policy. */ +typedef NS_ENUM(NSInteger, RTCBundlePolicy) { + RTCBundlePolicyBalanced, + RTCBundlePolicyMaxCompat, + RTCBundlePolicyMaxBundle +}; + +/** Represents the rtcp mux policy. */ +typedef NS_ENUM(NSInteger, RTCRtcpMuxPolicy) { RTCRtcpMuxPolicyNegotiate, RTCRtcpMuxPolicyRequire }; + +/** Represents the tcp candidate policy. */ +typedef NS_ENUM(NSInteger, RTCTcpCandidatePolicy) { + RTCTcpCandidatePolicyEnabled, + RTCTcpCandidatePolicyDisabled +}; + +/** Represents the candidate network policy. */ +typedef NS_ENUM(NSInteger, RTCCandidateNetworkPolicy) { + RTCCandidateNetworkPolicyAll, + RTCCandidateNetworkPolicyLowCost +}; + +/** Represents the continual gathering policy. */ +typedef NS_ENUM(NSInteger, RTCContinualGatheringPolicy) { + RTCContinualGatheringPolicyGatherOnce, + RTCContinualGatheringPolicyGatherContinually +}; + +/** Represents the encryption key type. */ +typedef NS_ENUM(NSInteger, RTCEncryptionKeyType) { + RTCEncryptionKeyTypeRSA, + RTCEncryptionKeyTypeECDSA, +}; + +/** Represents the chosen SDP semantics for the RTCPeerConnection. */ +typedef NS_ENUM(NSInteger, RTCSdpSemantics) { + // TODO(https://crbug.com/webrtc/13528): Remove support for Plan B. + RTCSdpSemanticsPlanB, + RTCSdpSemanticsUnifiedPlan, +}; + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCConfiguration) : NSObject + +/** If true, allows DSCP codes to be set on outgoing packets, configured using + * networkPriority field of RTCRtpEncodingParameters. Defaults to false. + */ +@property(nonatomic, assign) BOOL enableDscp; + +/** An array of Ice Servers available to be used by ICE. */ +@property(nonatomic, copy) NSArray<RTC_OBJC_TYPE(RTCIceServer) *> *iceServers; + +/** An RTCCertificate for 're' use. */ +@property(nonatomic, nullable) RTC_OBJC_TYPE(RTCCertificate) * certificate; + +/** Which candidates the ICE agent is allowed to use. The W3C calls it + * `iceTransportPolicy`, while in C++ it is called `type`. */ +@property(nonatomic, assign) RTCIceTransportPolicy iceTransportPolicy; + +/** The media-bundling policy to use when gathering ICE candidates. */ +@property(nonatomic, assign) RTCBundlePolicy bundlePolicy; + +/** The rtcp-mux policy to use when gathering ICE candidates. */ +@property(nonatomic, assign) RTCRtcpMuxPolicy rtcpMuxPolicy; +@property(nonatomic, assign) RTCTcpCandidatePolicy tcpCandidatePolicy; +@property(nonatomic, assign) RTCCandidateNetworkPolicy candidateNetworkPolicy; +@property(nonatomic, assign) RTCContinualGatheringPolicy continualGatheringPolicy; + +/** If set to YES, don't gather IPv6 ICE candidates on Wi-Fi. + * Only intended to be used on specific devices. Certain phones disable IPv6 + * when the screen is turned off and it would be better to just disable the + * IPv6 ICE candidates on Wi-Fi in those cases. + * Default is NO. + */ +@property(nonatomic, assign) BOOL disableIPV6OnWiFi; + +/** By default, the PeerConnection will use a limited number of IPv6 network + * interfaces, in order to avoid too many ICE candidate pairs being created + * and delaying ICE completion. + * + * Can be set to INT_MAX to effectively disable the limit. + */ +@property(nonatomic, assign) int maxIPv6Networks; + +/** Exclude link-local network interfaces + * from considertaion for gathering ICE candidates. + * Defaults to NO. + */ +@property(nonatomic, assign) BOOL disableLinkLocalNetworks; + +@property(nonatomic, assign) int audioJitterBufferMaxPackets; +@property(nonatomic, assign) BOOL audioJitterBufferFastAccelerate; +@property(nonatomic, assign) int iceConnectionReceivingTimeout; +@property(nonatomic, assign) int iceBackupCandidatePairPingInterval; + +/** Key type used to generate SSL identity. Default is ECDSA. */ +@property(nonatomic, assign) RTCEncryptionKeyType keyType; + +/** ICE candidate pool size as defined in JSEP. Default is 0. */ +@property(nonatomic, assign) int iceCandidatePoolSize; + +/** Prune turn ports on the same network to the same turn server. + * Default is NO. + */ +@property(nonatomic, assign) BOOL shouldPruneTurnPorts; + +/** If set to YES, this means the ICE transport should presume TURN-to-TURN + * candidate pairs will succeed, even before a binding response is received. + */ +@property(nonatomic, assign) BOOL shouldPresumeWritableWhenFullyRelayed; + +/* This flag is only effective when `continualGatheringPolicy` is + * RTCContinualGatheringPolicyGatherContinually. + * + * If YES, after the ICE transport type is changed such that new types of + * ICE candidates are allowed by the new transport type, e.g. from + * RTCIceTransportPolicyRelay to RTCIceTransportPolicyAll, candidates that + * have been gathered by the ICE transport but not matching the previous + * transport type and as a result not observed by PeerConnectionDelegateAdapter, + * will be surfaced to the delegate. + */ +@property(nonatomic, assign) BOOL shouldSurfaceIceCandidatesOnIceTransportTypeChanged; + +/** If set to non-nil, controls the minimal interval between consecutive ICE + * check packets. + */ +@property(nonatomic, copy, nullable) NSNumber *iceCheckMinInterval; + +/** + * Configure the SDP semantics used by this PeerConnection. By default, this + * is RTCSdpSemanticsUnifiedPlan which is compliant to the WebRTC 1.0 + * specification. It is possible to overrwite this to the deprecated + * RTCSdpSemanticsPlanB SDP format, but note that RTCSdpSemanticsPlanB will be + * deleted at some future date, see https://crbug.com/webrtc/13528. + * + * RTCSdpSemanticsUnifiedPlan will cause RTCPeerConnection to create offers and + * answers with multiple m= sections where each m= section maps to one + * RTCRtpSender and one RTCRtpReceiver (an RTCRtpTransceiver), either both audio + * or both video. This will also cause RTCPeerConnection to ignore all but the + * first a=ssrc lines that form a Plan B stream. + * + * RTCSdpSemanticsPlanB will cause RTCPeerConnection to create offers and + * answers with at most one audio and one video m= section with multiple + * RTCRtpSenders and RTCRtpReceivers specified as multiple a=ssrc lines within + * the section. This will also cause RTCPeerConnection to ignore all but the + * first m= section of the same media type. + */ +@property(nonatomic, assign) RTCSdpSemantics sdpSemantics; + +/** Actively reset the SRTP parameters when the DTLS transports underneath are + * changed after offer/answer negotiation. This is only intended to be a + * workaround for crbug.com/835958 + */ +@property(nonatomic, assign) BOOL activeResetSrtpParams; + +/** + * Defines advanced optional cryptographic settings related to SRTP and + * frame encryption for native WebRTC. Setting this will overwrite any + * options set through the PeerConnectionFactory (which is deprecated). + */ +@property(nonatomic, nullable) RTC_OBJC_TYPE(RTCCryptoOptions) * cryptoOptions; + +/** + * An optional string that will be attached to the TURN_ALLOCATE_REQUEST which + * which can be used to correlate client logs with backend logs. + */ +@property(nonatomic, nullable, copy) NSString *turnLoggingId; + +/** + * Time interval between audio RTCP reports. + */ +@property(nonatomic, assign) int rtcpAudioReportIntervalMs; + +/** + * Time interval between video RTCP reports. + */ +@property(nonatomic, assign) int rtcpVideoReportIntervalMs; + +/** + * Allow implicit rollback of local description when remote description + * conflicts with local description. + * See: https://w3c.github.io/webrtc-pc/#dom-peerconnection-setremotedescription + */ +@property(nonatomic, assign) BOOL enableImplicitRollback; + +/** + * Control if "a=extmap-allow-mixed" is included in the offer. + * See: https://www.chromestatus.com/feature/6269234631933952 + */ +@property(nonatomic, assign) BOOL offerExtmapAllowMixed; + +/** + * Defines the interval applied to ALL candidate pairs + * when ICE is strongly connected, and it overrides the + * default value of this interval in the ICE implementation; + */ +@property(nonatomic, copy, nullable) NSNumber *iceCheckIntervalStrongConnectivity; + +/** + * Defines the counterpart for ALL pairs when ICE is + * weakly connected, and it overrides the default value of + * this interval in the ICE implementation + */ +@property(nonatomic, copy, nullable) NSNumber *iceCheckIntervalWeakConnectivity; + +/** + * The min time period for which a candidate pair must wait for response to + * connectivity checks before it becomes unwritable. This parameter + * overrides the default value in the ICE implementation if set. + */ +@property(nonatomic, copy, nullable) NSNumber *iceUnwritableTimeout; + +/** + * The min number of connectivity checks that a candidate pair must sent + * without receiving response before it becomes unwritable. This parameter + * overrides the default value in the ICE implementation if set. + */ +@property(nonatomic, copy, nullable) NSNumber *iceUnwritableMinChecks; + +/** + * The min time period for which a candidate pair must wait for response to + * connectivity checks it becomes inactive. This parameter overrides the + * default value in the ICE implementation if set. + */ +@property(nonatomic, copy, nullable) NSNumber *iceInactiveTimeout; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.mm new file mode 100644 index 0000000000..86ecbabf8d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCConfiguration.mm @@ -0,0 +1,541 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCConfiguration+Private.h" + +#include <memory> + +#import "RTCCertificate.h" +#import "RTCConfiguration+Native.h" +#import "RTCIceServer+Private.h" +#import "base/RTCLogging.h" + +#include "rtc_base/checks.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_identity.h" + +@implementation RTC_OBJC_TYPE (RTCConfiguration) + +@synthesize enableDscp = _enableDscp; +@synthesize iceServers = _iceServers; +@synthesize certificate = _certificate; +@synthesize iceTransportPolicy = _iceTransportPolicy; +@synthesize bundlePolicy = _bundlePolicy; +@synthesize rtcpMuxPolicy = _rtcpMuxPolicy; +@synthesize tcpCandidatePolicy = _tcpCandidatePolicy; +@synthesize candidateNetworkPolicy = _candidateNetworkPolicy; +@synthesize continualGatheringPolicy = _continualGatheringPolicy; +@synthesize disableIPV6OnWiFi = _disableIPV6OnWiFi; +@synthesize maxIPv6Networks = _maxIPv6Networks; +@synthesize disableLinkLocalNetworks = _disableLinkLocalNetworks; +@synthesize audioJitterBufferMaxPackets = _audioJitterBufferMaxPackets; +@synthesize audioJitterBufferFastAccelerate = _audioJitterBufferFastAccelerate; +@synthesize iceConnectionReceivingTimeout = _iceConnectionReceivingTimeout; +@synthesize iceBackupCandidatePairPingInterval = + _iceBackupCandidatePairPingInterval; +@synthesize keyType = _keyType; +@synthesize iceCandidatePoolSize = _iceCandidatePoolSize; +@synthesize shouldPruneTurnPorts = _shouldPruneTurnPorts; +@synthesize shouldPresumeWritableWhenFullyRelayed = + _shouldPresumeWritableWhenFullyRelayed; +@synthesize shouldSurfaceIceCandidatesOnIceTransportTypeChanged = + _shouldSurfaceIceCandidatesOnIceTransportTypeChanged; +@synthesize iceCheckMinInterval = _iceCheckMinInterval; +@synthesize sdpSemantics = _sdpSemantics; +@synthesize turnCustomizer = _turnCustomizer; +@synthesize activeResetSrtpParams = _activeResetSrtpParams; +@synthesize cryptoOptions = _cryptoOptions; +@synthesize turnLoggingId = _turnLoggingId; +@synthesize rtcpAudioReportIntervalMs = _rtcpAudioReportIntervalMs; +@synthesize rtcpVideoReportIntervalMs = _rtcpVideoReportIntervalMs; +@synthesize enableImplicitRollback = _enableImplicitRollback; +@synthesize offerExtmapAllowMixed = _offerExtmapAllowMixed; +@synthesize iceCheckIntervalStrongConnectivity = _iceCheckIntervalStrongConnectivity; +@synthesize iceCheckIntervalWeakConnectivity = _iceCheckIntervalWeakConnectivity; +@synthesize iceUnwritableTimeout = _iceUnwritableTimeout; +@synthesize iceUnwritableMinChecks = _iceUnwritableMinChecks; +@synthesize iceInactiveTimeout = _iceInactiveTimeout; + +- (instancetype)init { + // Copy defaults. + webrtc::PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + return [self initWithNativeConfiguration:config]; +} + +- (instancetype)initWithNativeConfiguration: + (const webrtc::PeerConnectionInterface::RTCConfiguration &)config { + if (self = [super init]) { + _enableDscp = config.dscp(); + NSMutableArray *iceServers = [NSMutableArray array]; + for (const webrtc::PeerConnectionInterface::IceServer& server : config.servers) { + RTC_OBJC_TYPE(RTCIceServer) *iceServer = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithNativeServer:server]; + [iceServers addObject:iceServer]; + } + _iceServers = iceServers; + if (!config.certificates.empty()) { + rtc::scoped_refptr<rtc::RTCCertificate> native_cert; + native_cert = config.certificates[0]; + rtc::RTCCertificatePEM native_pem = native_cert->ToPEM(); + _certificate = [[RTC_OBJC_TYPE(RTCCertificate) alloc] + initWithPrivateKey:@(native_pem.private_key().c_str()) + certificate:@(native_pem.certificate().c_str())]; + } + _iceTransportPolicy = + [[self class] transportPolicyForTransportsType:config.type]; + _bundlePolicy = + [[self class] bundlePolicyForNativePolicy:config.bundle_policy]; + _rtcpMuxPolicy = + [[self class] rtcpMuxPolicyForNativePolicy:config.rtcp_mux_policy]; + _tcpCandidatePolicy = [[self class] tcpCandidatePolicyForNativePolicy: + config.tcp_candidate_policy]; + _candidateNetworkPolicy = [[self class] + candidateNetworkPolicyForNativePolicy:config.candidate_network_policy]; + webrtc::PeerConnectionInterface::ContinualGatheringPolicy nativePolicy = + config.continual_gathering_policy; + _continualGatheringPolicy = [[self class] continualGatheringPolicyForNativePolicy:nativePolicy]; + _disableIPV6OnWiFi = config.disable_ipv6_on_wifi; + _maxIPv6Networks = config.max_ipv6_networks; + _disableLinkLocalNetworks = config.disable_link_local_networks; + _audioJitterBufferMaxPackets = config.audio_jitter_buffer_max_packets; + _audioJitterBufferFastAccelerate = config.audio_jitter_buffer_fast_accelerate; + _iceConnectionReceivingTimeout = config.ice_connection_receiving_timeout; + _iceBackupCandidatePairPingInterval = + config.ice_backup_candidate_pair_ping_interval; + _keyType = RTCEncryptionKeyTypeECDSA; + _iceCandidatePoolSize = config.ice_candidate_pool_size; + _shouldPruneTurnPorts = config.prune_turn_ports; + _shouldPresumeWritableWhenFullyRelayed = + config.presume_writable_when_fully_relayed; + _shouldSurfaceIceCandidatesOnIceTransportTypeChanged = + config.surface_ice_candidates_on_ice_transport_type_changed; + if (config.ice_check_min_interval) { + _iceCheckMinInterval = + [NSNumber numberWithInt:*config.ice_check_min_interval]; + } + _sdpSemantics = [[self class] sdpSemanticsForNativeSdpSemantics:config.sdp_semantics]; + _turnCustomizer = config.turn_customizer; + _activeResetSrtpParams = config.active_reset_srtp_params; + if (config.crypto_options) { + _cryptoOptions = [[RTC_OBJC_TYPE(RTCCryptoOptions) alloc] + initWithSrtpEnableGcmCryptoSuites:config.crypto_options->srtp + .enable_gcm_crypto_suites + srtpEnableAes128Sha1_32CryptoCipher:config.crypto_options->srtp + .enable_aes128_sha1_32_crypto_cipher + srtpEnableEncryptedRtpHeaderExtensions:config.crypto_options->srtp + .enable_encrypted_rtp_header_extensions + sframeRequireFrameEncryption:config.crypto_options->sframe + .require_frame_encryption]; + } + _turnLoggingId = [NSString stringWithUTF8String:config.turn_logging_id.c_str()]; + _rtcpAudioReportIntervalMs = config.audio_rtcp_report_interval_ms(); + _rtcpVideoReportIntervalMs = config.video_rtcp_report_interval_ms(); + _enableImplicitRollback = config.enable_implicit_rollback; + _offerExtmapAllowMixed = config.offer_extmap_allow_mixed; + _iceCheckIntervalStrongConnectivity = + config.ice_check_interval_strong_connectivity.has_value() ? + [NSNumber numberWithInt:*config.ice_check_interval_strong_connectivity] : + nil; + _iceCheckIntervalWeakConnectivity = config.ice_check_interval_weak_connectivity.has_value() ? + [NSNumber numberWithInt:*config.ice_check_interval_weak_connectivity] : + nil; + _iceUnwritableTimeout = config.ice_unwritable_timeout.has_value() ? + [NSNumber numberWithInt:*config.ice_unwritable_timeout] : + nil; + _iceUnwritableMinChecks = config.ice_unwritable_min_checks.has_value() ? + [NSNumber numberWithInt:*config.ice_unwritable_min_checks] : + nil; + _iceInactiveTimeout = config.ice_inactive_timeout.has_value() ? + [NSNumber numberWithInt:*config.ice_inactive_timeout] : + nil; + } + return self; +} + +- (NSString *)description { + static NSString *formatString = @"RTC_OBJC_TYPE(RTCConfiguration): " + @"{\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%@\n%d\n%d\n%d\n%d\n%d\n%d\n" + @"%d\n%@\n%d\n%d\n%d\n%d\n%d\n%d\n%d\n}\n"; + + return [NSString + stringWithFormat:formatString, + _iceServers, + [[self class] stringForTransportPolicy:_iceTransportPolicy], + [[self class] stringForBundlePolicy:_bundlePolicy], + [[self class] stringForRtcpMuxPolicy:_rtcpMuxPolicy], + [[self class] stringForTcpCandidatePolicy:_tcpCandidatePolicy], + [[self class] stringForCandidateNetworkPolicy:_candidateNetworkPolicy], + [[self class] stringForContinualGatheringPolicy:_continualGatheringPolicy], + [[self class] stringForSdpSemantics:_sdpSemantics], + _audioJitterBufferMaxPackets, + _audioJitterBufferFastAccelerate, + _iceConnectionReceivingTimeout, + _iceBackupCandidatePairPingInterval, + _iceCandidatePoolSize, + _shouldPruneTurnPorts, + _shouldPresumeWritableWhenFullyRelayed, + _shouldSurfaceIceCandidatesOnIceTransportTypeChanged, + _iceCheckMinInterval, + _disableLinkLocalNetworks, + _disableIPV6OnWiFi, + _maxIPv6Networks, + _activeResetSrtpParams, + _enableDscp, + _enableImplicitRollback]; +} + +#pragma mark - Private + +- (webrtc::PeerConnectionInterface::RTCConfiguration *) + createNativeConfiguration { + std::unique_ptr<webrtc::PeerConnectionInterface::RTCConfiguration> + nativeConfig(new webrtc::PeerConnectionInterface::RTCConfiguration( + webrtc::PeerConnectionInterface::RTCConfigurationType::kAggressive)); + + nativeConfig->set_dscp(_enableDscp); + for (RTC_OBJC_TYPE(RTCIceServer) * iceServer in _iceServers) { + nativeConfig->servers.push_back(iceServer.nativeServer); + } + nativeConfig->type = + [[self class] nativeTransportsTypeForTransportPolicy:_iceTransportPolicy]; + nativeConfig->bundle_policy = + [[self class] nativeBundlePolicyForPolicy:_bundlePolicy]; + nativeConfig->rtcp_mux_policy = + [[self class] nativeRtcpMuxPolicyForPolicy:_rtcpMuxPolicy]; + nativeConfig->tcp_candidate_policy = + [[self class] nativeTcpCandidatePolicyForPolicy:_tcpCandidatePolicy]; + nativeConfig->candidate_network_policy = [[self class] + nativeCandidateNetworkPolicyForPolicy:_candidateNetworkPolicy]; + nativeConfig->continual_gathering_policy = + [[self class] nativeContinualGatheringPolicyForPolicy:_continualGatheringPolicy]; + nativeConfig->disable_ipv6_on_wifi = _disableIPV6OnWiFi; + nativeConfig->max_ipv6_networks = _maxIPv6Networks; + nativeConfig->disable_link_local_networks = _disableLinkLocalNetworks; + nativeConfig->audio_jitter_buffer_max_packets = _audioJitterBufferMaxPackets; + nativeConfig->audio_jitter_buffer_fast_accelerate = + _audioJitterBufferFastAccelerate ? true : false; + nativeConfig->ice_connection_receiving_timeout = + _iceConnectionReceivingTimeout; + nativeConfig->ice_backup_candidate_pair_ping_interval = + _iceBackupCandidatePairPingInterval; + rtc::KeyType keyType = + [[self class] nativeEncryptionKeyTypeForKeyType:_keyType]; + if (_certificate != nullptr) { + // if offered a pemcert use it... + RTC_LOG(LS_INFO) << "Have configured cert - using it."; + std::string pem_private_key = [[_certificate private_key] UTF8String]; + std::string pem_certificate = [[_certificate certificate] UTF8String]; + rtc::RTCCertificatePEM pem = rtc::RTCCertificatePEM(pem_private_key, pem_certificate); + rtc::scoped_refptr<rtc::RTCCertificate> certificate = rtc::RTCCertificate::FromPEM(pem); + RTC_LOG(LS_INFO) << "Created cert from PEM strings."; + if (!certificate) { + RTC_LOG(LS_ERROR) << "Failed to generate certificate from PEM."; + return nullptr; + } + nativeConfig->certificates.push_back(certificate); + } else { + RTC_LOG(LS_INFO) << "Don't have configured cert."; + // Generate non-default certificate. + if (keyType != rtc::KT_DEFAULT) { + rtc::scoped_refptr<rtc::RTCCertificate> certificate = + rtc::RTCCertificateGenerator::GenerateCertificate(rtc::KeyParams(keyType), + absl::optional<uint64_t>()); + if (!certificate) { + RTCLogError(@"Failed to generate certificate."); + return nullptr; + } + nativeConfig->certificates.push_back(certificate); + } + } + nativeConfig->ice_candidate_pool_size = _iceCandidatePoolSize; + nativeConfig->prune_turn_ports = _shouldPruneTurnPorts ? true : false; + nativeConfig->presume_writable_when_fully_relayed = + _shouldPresumeWritableWhenFullyRelayed ? true : false; + nativeConfig->surface_ice_candidates_on_ice_transport_type_changed = + _shouldSurfaceIceCandidatesOnIceTransportTypeChanged ? true : false; + if (_iceCheckMinInterval != nil) { + nativeConfig->ice_check_min_interval = absl::optional<int>(_iceCheckMinInterval.intValue); + } + nativeConfig->sdp_semantics = [[self class] nativeSdpSemanticsForSdpSemantics:_sdpSemantics]; + if (_turnCustomizer) { + nativeConfig->turn_customizer = _turnCustomizer; + } + nativeConfig->active_reset_srtp_params = _activeResetSrtpParams ? true : false; + if (_cryptoOptions) { + webrtc::CryptoOptions nativeCryptoOptions; + nativeCryptoOptions.srtp.enable_gcm_crypto_suites = + _cryptoOptions.srtpEnableGcmCryptoSuites ? true : false; + nativeCryptoOptions.srtp.enable_aes128_sha1_32_crypto_cipher = + _cryptoOptions.srtpEnableAes128Sha1_32CryptoCipher ? true : false; + nativeCryptoOptions.srtp.enable_encrypted_rtp_header_extensions = + _cryptoOptions.srtpEnableEncryptedRtpHeaderExtensions ? true : false; + nativeCryptoOptions.sframe.require_frame_encryption = + _cryptoOptions.sframeRequireFrameEncryption ? true : false; + nativeConfig->crypto_options = absl::optional<webrtc::CryptoOptions>(nativeCryptoOptions); + } + nativeConfig->turn_logging_id = [_turnLoggingId UTF8String]; + nativeConfig->set_audio_rtcp_report_interval_ms(_rtcpAudioReportIntervalMs); + nativeConfig->set_video_rtcp_report_interval_ms(_rtcpVideoReportIntervalMs); + nativeConfig->enable_implicit_rollback = _enableImplicitRollback; + nativeConfig->offer_extmap_allow_mixed = _offerExtmapAllowMixed; + if (_iceCheckIntervalStrongConnectivity != nil) { + nativeConfig->ice_check_interval_strong_connectivity = + absl::optional<int>(_iceCheckIntervalStrongConnectivity.intValue); + } + if (_iceCheckIntervalWeakConnectivity != nil) { + nativeConfig->ice_check_interval_weak_connectivity = + absl::optional<int>(_iceCheckIntervalWeakConnectivity.intValue); + } + if (_iceUnwritableTimeout != nil) { + nativeConfig->ice_unwritable_timeout = absl::optional<int>(_iceUnwritableTimeout.intValue); + } + if (_iceUnwritableMinChecks != nil) { + nativeConfig->ice_unwritable_min_checks = absl::optional<int>(_iceUnwritableMinChecks.intValue); + } + if (_iceInactiveTimeout != nil) { + nativeConfig->ice_inactive_timeout = absl::optional<int>(_iceInactiveTimeout.intValue); + } + return nativeConfig.release(); +} + ++ (webrtc::PeerConnectionInterface::IceTransportsType) + nativeTransportsTypeForTransportPolicy:(RTCIceTransportPolicy)policy { + switch (policy) { + case RTCIceTransportPolicyNone: + return webrtc::PeerConnectionInterface::kNone; + case RTCIceTransportPolicyRelay: + return webrtc::PeerConnectionInterface::kRelay; + case RTCIceTransportPolicyNoHost: + return webrtc::PeerConnectionInterface::kNoHost; + case RTCIceTransportPolicyAll: + return webrtc::PeerConnectionInterface::kAll; + } +} + ++ (RTCIceTransportPolicy)transportPolicyForTransportsType: + (webrtc::PeerConnectionInterface::IceTransportsType)nativeType { + switch (nativeType) { + case webrtc::PeerConnectionInterface::kNone: + return RTCIceTransportPolicyNone; + case webrtc::PeerConnectionInterface::kRelay: + return RTCIceTransportPolicyRelay; + case webrtc::PeerConnectionInterface::kNoHost: + return RTCIceTransportPolicyNoHost; + case webrtc::PeerConnectionInterface::kAll: + return RTCIceTransportPolicyAll; + } +} + ++ (NSString *)stringForTransportPolicy:(RTCIceTransportPolicy)policy { + switch (policy) { + case RTCIceTransportPolicyNone: + return @"NONE"; + case RTCIceTransportPolicyRelay: + return @"RELAY"; + case RTCIceTransportPolicyNoHost: + return @"NO_HOST"; + case RTCIceTransportPolicyAll: + return @"ALL"; + } +} + ++ (webrtc::PeerConnectionInterface::BundlePolicy)nativeBundlePolicyForPolicy: + (RTCBundlePolicy)policy { + switch (policy) { + case RTCBundlePolicyBalanced: + return webrtc::PeerConnectionInterface::kBundlePolicyBalanced; + case RTCBundlePolicyMaxCompat: + return webrtc::PeerConnectionInterface::kBundlePolicyMaxCompat; + case RTCBundlePolicyMaxBundle: + return webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; + } +} + ++ (RTCBundlePolicy)bundlePolicyForNativePolicy: + (webrtc::PeerConnectionInterface::BundlePolicy)nativePolicy { + switch (nativePolicy) { + case webrtc::PeerConnectionInterface::kBundlePolicyBalanced: + return RTCBundlePolicyBalanced; + case webrtc::PeerConnectionInterface::kBundlePolicyMaxCompat: + return RTCBundlePolicyMaxCompat; + case webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle: + return RTCBundlePolicyMaxBundle; + } +} + ++ (NSString *)stringForBundlePolicy:(RTCBundlePolicy)policy { + switch (policy) { + case RTCBundlePolicyBalanced: + return @"BALANCED"; + case RTCBundlePolicyMaxCompat: + return @"MAX_COMPAT"; + case RTCBundlePolicyMaxBundle: + return @"MAX_BUNDLE"; + } +} + ++ (webrtc::PeerConnectionInterface::RtcpMuxPolicy)nativeRtcpMuxPolicyForPolicy: + (RTCRtcpMuxPolicy)policy { + switch (policy) { + case RTCRtcpMuxPolicyNegotiate: + return webrtc::PeerConnectionInterface::kRtcpMuxPolicyNegotiate; + case RTCRtcpMuxPolicyRequire: + return webrtc::PeerConnectionInterface::kRtcpMuxPolicyRequire; + } +} + ++ (RTCRtcpMuxPolicy)rtcpMuxPolicyForNativePolicy: + (webrtc::PeerConnectionInterface::RtcpMuxPolicy)nativePolicy { + switch (nativePolicy) { + case webrtc::PeerConnectionInterface::kRtcpMuxPolicyNegotiate: + return RTCRtcpMuxPolicyNegotiate; + case webrtc::PeerConnectionInterface::kRtcpMuxPolicyRequire: + return RTCRtcpMuxPolicyRequire; + } +} + ++ (NSString *)stringForRtcpMuxPolicy:(RTCRtcpMuxPolicy)policy { + switch (policy) { + case RTCRtcpMuxPolicyNegotiate: + return @"NEGOTIATE"; + case RTCRtcpMuxPolicyRequire: + return @"REQUIRE"; + } +} + ++ (webrtc::PeerConnectionInterface::TcpCandidatePolicy) + nativeTcpCandidatePolicyForPolicy:(RTCTcpCandidatePolicy)policy { + switch (policy) { + case RTCTcpCandidatePolicyEnabled: + return webrtc::PeerConnectionInterface::kTcpCandidatePolicyEnabled; + case RTCTcpCandidatePolicyDisabled: + return webrtc::PeerConnectionInterface::kTcpCandidatePolicyDisabled; + } +} + ++ (webrtc::PeerConnectionInterface::CandidateNetworkPolicy) + nativeCandidateNetworkPolicyForPolicy:(RTCCandidateNetworkPolicy)policy { + switch (policy) { + case RTCCandidateNetworkPolicyAll: + return webrtc::PeerConnectionInterface::kCandidateNetworkPolicyAll; + case RTCCandidateNetworkPolicyLowCost: + return webrtc::PeerConnectionInterface::kCandidateNetworkPolicyLowCost; + } +} + ++ (RTCTcpCandidatePolicy)tcpCandidatePolicyForNativePolicy: + (webrtc::PeerConnectionInterface::TcpCandidatePolicy)nativePolicy { + switch (nativePolicy) { + case webrtc::PeerConnectionInterface::kTcpCandidatePolicyEnabled: + return RTCTcpCandidatePolicyEnabled; + case webrtc::PeerConnectionInterface::kTcpCandidatePolicyDisabled: + return RTCTcpCandidatePolicyDisabled; + } +} + ++ (NSString *)stringForTcpCandidatePolicy:(RTCTcpCandidatePolicy)policy { + switch (policy) { + case RTCTcpCandidatePolicyEnabled: + return @"TCP_ENABLED"; + case RTCTcpCandidatePolicyDisabled: + return @"TCP_DISABLED"; + } +} + ++ (RTCCandidateNetworkPolicy)candidateNetworkPolicyForNativePolicy: + (webrtc::PeerConnectionInterface::CandidateNetworkPolicy)nativePolicy { + switch (nativePolicy) { + case webrtc::PeerConnectionInterface::kCandidateNetworkPolicyAll: + return RTCCandidateNetworkPolicyAll; + case webrtc::PeerConnectionInterface::kCandidateNetworkPolicyLowCost: + return RTCCandidateNetworkPolicyLowCost; + } +} + ++ (NSString *)stringForCandidateNetworkPolicy: + (RTCCandidateNetworkPolicy)policy { + switch (policy) { + case RTCCandidateNetworkPolicyAll: + return @"CANDIDATE_ALL_NETWORKS"; + case RTCCandidateNetworkPolicyLowCost: + return @"CANDIDATE_LOW_COST_NETWORKS"; + } +} + ++ (webrtc::PeerConnectionInterface::ContinualGatheringPolicy) + nativeContinualGatheringPolicyForPolicy: + (RTCContinualGatheringPolicy)policy { + switch (policy) { + case RTCContinualGatheringPolicyGatherOnce: + return webrtc::PeerConnectionInterface::GATHER_ONCE; + case RTCContinualGatheringPolicyGatherContinually: + return webrtc::PeerConnectionInterface::GATHER_CONTINUALLY; + } +} + ++ (RTCContinualGatheringPolicy)continualGatheringPolicyForNativePolicy: + (webrtc::PeerConnectionInterface::ContinualGatheringPolicy)nativePolicy { + switch (nativePolicy) { + case webrtc::PeerConnectionInterface::GATHER_ONCE: + return RTCContinualGatheringPolicyGatherOnce; + case webrtc::PeerConnectionInterface::GATHER_CONTINUALLY: + return RTCContinualGatheringPolicyGatherContinually; + } +} + ++ (NSString *)stringForContinualGatheringPolicy: + (RTCContinualGatheringPolicy)policy { + switch (policy) { + case RTCContinualGatheringPolicyGatherOnce: + return @"GATHER_ONCE"; + case RTCContinualGatheringPolicyGatherContinually: + return @"GATHER_CONTINUALLY"; + } +} + ++ (rtc::KeyType)nativeEncryptionKeyTypeForKeyType: + (RTCEncryptionKeyType)keyType { + switch (keyType) { + case RTCEncryptionKeyTypeRSA: + return rtc::KT_RSA; + case RTCEncryptionKeyTypeECDSA: + return rtc::KT_ECDSA; + } +} + ++ (webrtc::SdpSemantics)nativeSdpSemanticsForSdpSemantics:(RTCSdpSemantics)sdpSemantics { + switch (sdpSemantics) { + case RTCSdpSemanticsPlanB: + return webrtc::SdpSemantics::kPlanB_DEPRECATED; + case RTCSdpSemanticsUnifiedPlan: + return webrtc::SdpSemantics::kUnifiedPlan; + } +} + ++ (RTCSdpSemantics)sdpSemanticsForNativeSdpSemantics:(webrtc::SdpSemantics)sdpSemantics { + switch (sdpSemantics) { + case webrtc::SdpSemantics::kPlanB_DEPRECATED: + return RTCSdpSemanticsPlanB; + case webrtc::SdpSemantics::kUnifiedPlan: + return RTCSdpSemanticsUnifiedPlan; + } +} + ++ (NSString *)stringForSdpSemantics:(RTCSdpSemantics)sdpSemantics { + switch (sdpSemantics) { + case RTCSdpSemanticsPlanB: + return @"PLAN_B"; + case RTCSdpSemanticsUnifiedPlan: + return @"UNIFIED_PLAN"; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.h new file mode 100644 index 0000000000..7894c8d50c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.h @@ -0,0 +1,63 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Objective-C bindings for webrtc::CryptoOptions. This API had to be flattened + * as Objective-C doesn't support nested structures. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCryptoOptions) : NSObject + +/** + * Enable GCM crypto suites from RFC 7714 for SRTP. GCM will only be used + * if both sides enable it + */ +@property(nonatomic, assign) BOOL srtpEnableGcmCryptoSuites; +/** + * If set to true, the (potentially insecure) crypto cipher + * kSrtpAes128CmSha1_32 will be included in the list of supported ciphers + * during negotiation. It will only be used if both peers support it and no + * other ciphers get preferred. + */ +@property(nonatomic, assign) BOOL srtpEnableAes128Sha1_32CryptoCipher; +/** + * If set to true, encrypted RTP header extensions as defined in RFC 6904 + * will be negotiated. They will only be used if both peers support them. + */ +@property(nonatomic, assign) BOOL srtpEnableEncryptedRtpHeaderExtensions; + +/** + * If set all RtpSenders must have an FrameEncryptor attached to them before + * they are allowed to send packets. All RtpReceivers must have a + * FrameDecryptor attached to them before they are able to receive packets. + */ +@property(nonatomic, assign) BOOL sframeRequireFrameEncryption; + +/** + * Initializes CryptoOptions with all possible options set explicitly. This + * is done when converting from a native RTCConfiguration.crypto_options. + */ +- (instancetype)initWithSrtpEnableGcmCryptoSuites:(BOOL)srtpEnableGcmCryptoSuites + srtpEnableAes128Sha1_32CryptoCipher:(BOOL)srtpEnableAes128Sha1_32CryptoCipher + srtpEnableEncryptedRtpHeaderExtensions:(BOOL)srtpEnableEncryptedRtpHeaderExtensions + sframeRequireFrameEncryption:(BOOL)sframeRequireFrameEncryption + NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.mm new file mode 100644 index 0000000000..fbaa1de58d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCCryptoOptions.mm @@ -0,0 +1,33 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCryptoOptions.h" + +@implementation RTC_OBJC_TYPE (RTCCryptoOptions) + +@synthesize srtpEnableGcmCryptoSuites = _srtpEnableGcmCryptoSuites; +@synthesize srtpEnableAes128Sha1_32CryptoCipher = _srtpEnableAes128Sha1_32CryptoCipher; +@synthesize srtpEnableEncryptedRtpHeaderExtensions = _srtpEnableEncryptedRtpHeaderExtensions; +@synthesize sframeRequireFrameEncryption = _sframeRequireFrameEncryption; + +- (instancetype)initWithSrtpEnableGcmCryptoSuites:(BOOL)srtpEnableGcmCryptoSuites + srtpEnableAes128Sha1_32CryptoCipher:(BOOL)srtpEnableAes128Sha1_32CryptoCipher + srtpEnableEncryptedRtpHeaderExtensions:(BOOL)srtpEnableEncryptedRtpHeaderExtensions + sframeRequireFrameEncryption:(BOOL)sframeRequireFrameEncryption { + if (self = [super init]) { + _srtpEnableGcmCryptoSuites = srtpEnableGcmCryptoSuites; + _srtpEnableAes128Sha1_32CryptoCipher = srtpEnableAes128Sha1_32CryptoCipher; + _srtpEnableEncryptedRtpHeaderExtensions = srtpEnableEncryptedRtpHeaderExtensions; + _sframeRequireFrameEncryption = sframeRequireFrameEncryption; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel+Private.h new file mode 100644 index 0000000000..d903b0c002 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel+Private.h @@ -0,0 +1,52 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDataChannel.h" + +#include "api/data_channel_interface.h" +#include "api/scoped_refptr.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +@interface RTC_OBJC_TYPE (RTCDataBuffer) +() + + /** + * The native DataBuffer representation of this RTCDatabuffer object. This is + * needed to pass to the underlying C++ APIs. + */ + @property(nonatomic, readonly) const webrtc::DataBuffer *nativeDataBuffer; + +/** Initialize an RTCDataBuffer from a native DataBuffer. */ +- (instancetype)initWithNativeBuffer:(const webrtc::DataBuffer &)nativeBuffer; + +@end + +@interface RTC_OBJC_TYPE (RTCDataChannel) +() + + /** Initialize an RTCDataChannel from a native DataChannelInterface. */ + - (instancetype)initWithFactory + : (RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory nativeDataChannel + : (rtc::scoped_refptr<webrtc::DataChannelInterface>)nativeDataChannel NS_DESIGNATED_INITIALIZER; + ++ (webrtc::DataChannelInterface::DataState)nativeDataChannelStateForState: + (RTCDataChannelState)state; + ++ (RTCDataChannelState)dataChannelStateForNativeState: + (webrtc::DataChannelInterface::DataState)nativeState; + ++ (NSString *)stringForState:(RTCDataChannelState)state; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.h new file mode 100644 index 0000000000..89eb58bc3f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.h @@ -0,0 +1,132 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AvailabilityMacros.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDataBuffer) : NSObject + +/** NSData representation of the underlying buffer. */ +@property(nonatomic, readonly) NSData *data; + +/** Indicates whether `data` contains UTF-8 or binary data. */ +@property(nonatomic, readonly) BOOL isBinary; + +- (instancetype)init NS_UNAVAILABLE; + +/** + * Initialize an RTCDataBuffer from NSData. `isBinary` indicates whether `data` + * contains UTF-8 or binary data. + */ +- (instancetype)initWithData:(NSData *)data isBinary:(BOOL)isBinary; + +@end + +@class RTC_OBJC_TYPE(RTCDataChannel); +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDataChannelDelegate)<NSObject> + + /** The data channel state changed. */ + - (void)dataChannelDidChangeState : (RTC_OBJC_TYPE(RTCDataChannel) *)dataChannel; + +/** The data channel successfully received a data buffer. */ +- (void)dataChannel:(RTC_OBJC_TYPE(RTCDataChannel) *)dataChannel + didReceiveMessageWithBuffer:(RTC_OBJC_TYPE(RTCDataBuffer) *)buffer; + +@optional +/** The data channel's `bufferedAmount` changed. */ +- (void)dataChannel:(RTC_OBJC_TYPE(RTCDataChannel) *)dataChannel + didChangeBufferedAmount:(uint64_t)amount; + +@end + +/** Represents the state of the data channel. */ +typedef NS_ENUM(NSInteger, RTCDataChannelState) { + RTCDataChannelStateConnecting, + RTCDataChannelStateOpen, + RTCDataChannelStateClosing, + RTCDataChannelStateClosed, +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDataChannel) : NSObject + +/** + * A label that can be used to distinguish this data channel from other data + * channel objects. + */ +@property(nonatomic, readonly) NSString *label; + +/** Whether the data channel can send messages in unreliable mode. */ +@property(nonatomic, readonly) BOOL isReliable DEPRECATED_ATTRIBUTE; + +/** Returns whether this data channel is ordered or not. */ +@property(nonatomic, readonly) BOOL isOrdered; + +/** Deprecated. Use maxPacketLifeTime. */ +@property(nonatomic, readonly) NSUInteger maxRetransmitTime DEPRECATED_ATTRIBUTE; + +/** + * The length of the time window (in milliseconds) during which transmissions + * and retransmissions may occur in unreliable mode. + */ +@property(nonatomic, readonly) uint16_t maxPacketLifeTime; + +/** + * The maximum number of retransmissions that are attempted in unreliable mode. + */ +@property(nonatomic, readonly) uint16_t maxRetransmits; + +/** + * The name of the sub-protocol used with this data channel, if any. Otherwise + * this returns an empty string. + */ +@property(nonatomic, readonly) NSString *protocol; + +/** + * Returns whether this data channel was negotiated by the application or not. + */ +@property(nonatomic, readonly) BOOL isNegotiated; + +/** Deprecated. Use channelId. */ +@property(nonatomic, readonly) NSInteger streamId DEPRECATED_ATTRIBUTE; + +/** The identifier for this data channel. */ +@property(nonatomic, readonly) int channelId; + +/** The state of the data channel. */ +@property(nonatomic, readonly) RTCDataChannelState readyState; + +/** + * The number of bytes of application data that have been queued using + * `sendData:` but that have not yet been transmitted to the network. + */ +@property(nonatomic, readonly) uint64_t bufferedAmount; + +/** The delegate for this data channel. */ +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCDataChannelDelegate)> delegate; + +- (instancetype)init NS_UNAVAILABLE; + +/** Closes the data channel. */ +- (void)close; + +/** Attempt to send `data` on this data channel's underlying data transport. */ +- (BOOL)sendData:(RTC_OBJC_TYPE(RTCDataBuffer) *)data; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.mm new file mode 100644 index 0000000000..4a79cefdb4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannel.mm @@ -0,0 +1,220 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDataChannel+Private.h" + +#import "helpers/NSString+StdString.h" + +#include <memory> + +namespace webrtc { + +class DataChannelDelegateAdapter : public DataChannelObserver { + public: + DataChannelDelegateAdapter(RTC_OBJC_TYPE(RTCDataChannel) * channel) { channel_ = channel; } + + void OnStateChange() override { + [channel_.delegate dataChannelDidChangeState:channel_]; + } + + void OnMessage(const DataBuffer& buffer) override { + RTC_OBJC_TYPE(RTCDataBuffer) *data_buffer = + [[RTC_OBJC_TYPE(RTCDataBuffer) alloc] initWithNativeBuffer:buffer]; + [channel_.delegate dataChannel:channel_ + didReceiveMessageWithBuffer:data_buffer]; + } + + void OnBufferedAmountChange(uint64_t previousAmount) override { + id<RTC_OBJC_TYPE(RTCDataChannelDelegate)> delegate = channel_.delegate; + SEL sel = @selector(dataChannel:didChangeBufferedAmount:); + if ([delegate respondsToSelector:sel]) { + [delegate dataChannel:channel_ didChangeBufferedAmount:previousAmount]; + } + } + + private: + __weak RTC_OBJC_TYPE(RTCDataChannel) * channel_; +}; +} + +@implementation RTC_OBJC_TYPE (RTCDataBuffer) { + std::unique_ptr<webrtc::DataBuffer> _dataBuffer; +} + +- (instancetype)initWithData:(NSData *)data isBinary:(BOOL)isBinary { + NSParameterAssert(data); + if (self = [super init]) { + rtc::CopyOnWriteBuffer buffer( + reinterpret_cast<const uint8_t*>(data.bytes), data.length); + _dataBuffer.reset(new webrtc::DataBuffer(buffer, isBinary)); + } + return self; +} + +- (NSData *)data { + return [NSData dataWithBytes:_dataBuffer->data.data() + length:_dataBuffer->data.size()]; +} + +- (BOOL)isBinary { + return _dataBuffer->binary; +} + +#pragma mark - Private + +- (instancetype)initWithNativeBuffer:(const webrtc::DataBuffer&)nativeBuffer { + if (self = [super init]) { + _dataBuffer.reset(new webrtc::DataBuffer(nativeBuffer)); + } + return self; +} + +- (const webrtc::DataBuffer *)nativeDataBuffer { + return _dataBuffer.get(); +} + +@end + +@implementation RTC_OBJC_TYPE (RTCDataChannel) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::scoped_refptr<webrtc::DataChannelInterface> _nativeDataChannel; + std::unique_ptr<webrtc::DataChannelDelegateAdapter> _observer; + BOOL _isObserverRegistered; +} + +@synthesize delegate = _delegate; + +- (void)dealloc { + // Handles unregistering the observer properly. We need to do this because + // there may still be other references to the underlying data channel. + _nativeDataChannel->UnregisterObserver(); +} + +- (NSString *)label { + return [NSString stringForStdString:_nativeDataChannel->label()]; +} + +- (BOOL)isReliable { + return _nativeDataChannel->reliable(); +} + +- (BOOL)isOrdered { + return _nativeDataChannel->ordered(); +} + +- (NSUInteger)maxRetransmitTime { + return self.maxPacketLifeTime; +} + +- (uint16_t)maxPacketLifeTime { + return _nativeDataChannel->maxRetransmitTime(); +} + +- (uint16_t)maxRetransmits { + return _nativeDataChannel->maxRetransmits(); +} + +- (NSString *)protocol { + return [NSString stringForStdString:_nativeDataChannel->protocol()]; +} + +- (BOOL)isNegotiated { + return _nativeDataChannel->negotiated(); +} + +- (NSInteger)streamId { + return self.channelId; +} + +- (int)channelId { + return _nativeDataChannel->id(); +} + +- (RTCDataChannelState)readyState { + return [[self class] dataChannelStateForNativeState: + _nativeDataChannel->state()]; +} + +- (uint64_t)bufferedAmount { + return _nativeDataChannel->buffered_amount(); +} + +- (void)close { + _nativeDataChannel->Close(); +} + +- (BOOL)sendData:(RTC_OBJC_TYPE(RTCDataBuffer) *)data { + return _nativeDataChannel->Send(*data.nativeDataBuffer); +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCDataChannel):\n%ld\n%@\n%@", + (long)self.channelId, + self.label, + [[self class] stringForState:self.readyState]]; +} + +#pragma mark - Private + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeDataChannel: + (rtc::scoped_refptr<webrtc::DataChannelInterface>)nativeDataChannel { + NSParameterAssert(nativeDataChannel); + if (self = [super init]) { + _factory = factory; + _nativeDataChannel = nativeDataChannel; + _observer.reset(new webrtc::DataChannelDelegateAdapter(self)); + _nativeDataChannel->RegisterObserver(_observer.get()); + } + return self; +} + ++ (webrtc::DataChannelInterface::DataState) + nativeDataChannelStateForState:(RTCDataChannelState)state { + switch (state) { + case RTCDataChannelStateConnecting: + return webrtc::DataChannelInterface::DataState::kConnecting; + case RTCDataChannelStateOpen: + return webrtc::DataChannelInterface::DataState::kOpen; + case RTCDataChannelStateClosing: + return webrtc::DataChannelInterface::DataState::kClosing; + case RTCDataChannelStateClosed: + return webrtc::DataChannelInterface::DataState::kClosed; + } +} + ++ (RTCDataChannelState)dataChannelStateForNativeState: + (webrtc::DataChannelInterface::DataState)nativeState { + switch (nativeState) { + case webrtc::DataChannelInterface::DataState::kConnecting: + return RTCDataChannelStateConnecting; + case webrtc::DataChannelInterface::DataState::kOpen: + return RTCDataChannelStateOpen; + case webrtc::DataChannelInterface::DataState::kClosing: + return RTCDataChannelStateClosing; + case webrtc::DataChannelInterface::DataState::kClosed: + return RTCDataChannelStateClosed; + } +} + ++ (NSString *)stringForState:(RTCDataChannelState)state { + switch (state) { + case RTCDataChannelStateConnecting: + return @"Connecting"; + case RTCDataChannelStateOpen: + return @"Open"; + case RTCDataChannelStateClosing: + return @"Closing"; + case RTCDataChannelStateClosed: + return @"Closed"; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration+Private.h new file mode 100644 index 0000000000..5aef10fcef --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration+Private.h @@ -0,0 +1,24 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDataChannelConfiguration.h" + +#include "api/data_channel_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCDataChannelConfiguration) +() + + @property(nonatomic, readonly) webrtc::DataChannelInit nativeDataChannelInit; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.h new file mode 100644 index 0000000000..9459ae0a13 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.h @@ -0,0 +1,52 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AvailabilityMacros.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDataChannelConfiguration) : NSObject + +/** Set to YES if ordered delivery is required. */ +@property(nonatomic, assign) BOOL isOrdered; + +/** Deprecated. Use maxPacketLifeTime. */ +@property(nonatomic, assign) NSInteger maxRetransmitTimeMs DEPRECATED_ATTRIBUTE; + +/** + * Max period in milliseconds in which retransmissions will be sent. After this + * time, no more retransmissions will be sent. -1 if unset. + */ +@property(nonatomic, assign) int maxPacketLifeTime; + +/** The max number of retransmissions. -1 if unset. */ +@property(nonatomic, assign) int maxRetransmits; + +/** Set to YES if the channel has been externally negotiated and we do not send + * an in-band signalling in the form of an "open" message. + */ +@property(nonatomic, assign) BOOL isNegotiated; + +/** Deprecated. Use channelId. */ +@property(nonatomic, assign) int streamId DEPRECATED_ATTRIBUTE; + +/** The id of the data channel. */ +@property(nonatomic, assign) int channelId; + +/** Set by the application and opaque to the WebRTC implementation. */ +@property(nonatomic) NSString* protocol; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.mm new file mode 100644 index 0000000000..bf775b1afd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDataChannelConfiguration.mm @@ -0,0 +1,87 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDataChannelConfiguration+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCDataChannelConfiguration) + +@synthesize nativeDataChannelInit = _nativeDataChannelInit; + +- (BOOL)isOrdered { + return _nativeDataChannelInit.ordered; +} + +- (void)setIsOrdered:(BOOL)isOrdered { + _nativeDataChannelInit.ordered = isOrdered; +} + +- (NSInteger)maxRetransmitTimeMs { + return self.maxPacketLifeTime; +} + +- (void)setMaxRetransmitTimeMs:(NSInteger)maxRetransmitTimeMs { + self.maxPacketLifeTime = maxRetransmitTimeMs; +} + +- (int)maxPacketLifeTime { + return *_nativeDataChannelInit.maxRetransmitTime; +} + +- (void)setMaxPacketLifeTime:(int)maxPacketLifeTime { + _nativeDataChannelInit.maxRetransmitTime = maxPacketLifeTime; +} + +- (int)maxRetransmits { + if (_nativeDataChannelInit.maxRetransmits) { + return *_nativeDataChannelInit.maxRetransmits; + } else { + return -1; + } +} + +- (void)setMaxRetransmits:(int)maxRetransmits { + _nativeDataChannelInit.maxRetransmits = maxRetransmits; +} + +- (NSString *)protocol { + return [NSString stringForStdString:_nativeDataChannelInit.protocol]; +} + +- (void)setProtocol:(NSString *)protocol { + _nativeDataChannelInit.protocol = [NSString stdStringForString:protocol]; +} + +- (BOOL)isNegotiated { + return _nativeDataChannelInit.negotiated; +} + +- (void)setIsNegotiated:(BOOL)isNegotiated { + _nativeDataChannelInit.negotiated = isNegotiated; +} + +- (int)streamId { + return self.channelId; +} + +- (void)setStreamId:(int)streamId { + self.channelId = streamId; +} + +- (int)channelId { + return _nativeDataChannelInit.id; +} + +- (void)setChannelId:(int)channelId { + _nativeDataChannelInit.id = channelId; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender+Private.h new file mode 100644 index 0000000000..627d02a6c8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDtmfSender.h" + +#include "api/dtmf_sender_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCDtmfSender) : NSObject <RTC_OBJC_TYPE(RTCDtmfSender)> + +@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::DtmfSenderInterface> nativeDtmfSender; + +- (instancetype)init NS_UNAVAILABLE; + +/** Initialize an RTCDtmfSender with a native DtmfSenderInterface. */ +- (instancetype)initWithNativeDtmfSender: + (rtc::scoped_refptr<webrtc::DtmfSenderInterface>)nativeDtmfSender NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.h new file mode 100644 index 0000000000..0f1b6ba4da --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.h @@ -0,0 +1,71 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCDtmfSender)<NSObject> + + /** + * Returns true if this RTCDtmfSender is capable of sending DTMF. Otherwise + * returns false. To be able to send DTMF, the associated RTCRtpSender must be + * able to send packets, and a "telephone-event" codec must be negotiated. + */ + @property(nonatomic, readonly) BOOL canInsertDtmf; + +/** + * Queues a task that sends the DTMF tones. The tones parameter is treated + * as a series of characters. The characters 0 through 9, A through D, #, and * + * generate the associated DTMF tones. The characters a to d are equivalent + * to A to D. The character ',' indicates a delay of 2 seconds before + * processing the next character in the tones parameter. + * + * Unrecognized characters are ignored. + * + * @param duration The parameter indicates the duration to use for each + * character passed in the tones parameter. The duration cannot be more + * than 6000 or less than 70 ms. + * + * @param interToneGap The parameter indicates the gap between tones. + * This parameter must be at least 50 ms but should be as short as + * possible. + * + * If InsertDtmf is called on the same object while an existing task for this + * object to generate DTMF is still running, the previous task is canceled. + * Returns true on success and false on failure. + */ +- (BOOL)insertDtmf:(nonnull NSString *)tones + duration:(NSTimeInterval)duration + interToneGap:(NSTimeInterval)interToneGap; + +/** The tones remaining to be played out */ +- (nonnull NSString *)remainingTones; + +/** + * The current tone duration value. This value will be the value last set via the + * insertDtmf method, or the default value of 100 ms if insertDtmf was never called. + */ +- (NSTimeInterval)duration; + +/** + * The current value of the between-tone gap. This value will be the value last set + * via the insertDtmf() method, or the default value of 50 ms if insertDtmf() was never + * called. + */ +- (NSTimeInterval)interToneGap; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.mm new file mode 100644 index 0000000000..ee3b79cd37 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCDtmfSender.mm @@ -0,0 +1,74 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDtmfSender+Private.h" + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "rtc_base/time_utils.h" + +@implementation RTC_OBJC_TYPE (RTCDtmfSender) { + rtc::scoped_refptr<webrtc::DtmfSenderInterface> _nativeDtmfSender; +} + +- (BOOL)canInsertDtmf { + return _nativeDtmfSender->CanInsertDtmf(); +} + +- (BOOL)insertDtmf:(nonnull NSString *)tones + duration:(NSTimeInterval)duration + interToneGap:(NSTimeInterval)interToneGap { + RTC_DCHECK(tones != nil); + + int durationMs = static_cast<int>(duration * rtc::kNumMillisecsPerSec); + int interToneGapMs = static_cast<int>(interToneGap * rtc::kNumMillisecsPerSec); + return _nativeDtmfSender->InsertDtmf( + [NSString stdStringForString:tones], durationMs, interToneGapMs); +} + +- (nonnull NSString *)remainingTones { + return [NSString stringForStdString:_nativeDtmfSender->tones()]; +} + +- (NSTimeInterval)duration { + return static_cast<NSTimeInterval>(_nativeDtmfSender->duration()) / rtc::kNumMillisecsPerSec; +} + +- (NSTimeInterval)interToneGap { + return static_cast<NSTimeInterval>(_nativeDtmfSender->inter_tone_gap()) / + rtc::kNumMillisecsPerSec; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCDtmfSender) {\n remainingTones: %@\n " + @"duration: %f sec\n interToneGap: %f sec\n}", + [self remainingTones], + [self duration], + [self interToneGap]]; +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::DtmfSenderInterface>)nativeDtmfSender { + return _nativeDtmfSender; +} + +- (instancetype)initWithNativeDtmfSender: + (rtc::scoped_refptr<webrtc::DtmfSenderInterface>)nativeDtmfSender { + NSParameterAssert(nativeDtmfSender); + if (self = [super init]) { + _nativeDtmfSender = nativeDtmfSender; + RTCLogInfo( + @"RTC_OBJC_TYPE(RTCDtmfSender)(%p): created DTMF sender: %@", self, self.description); + } + return self; +} +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.h new file mode 100644 index 0000000000..a078b0aded --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "base/RTCEncodedImage.h" + +#include "api/video/encoded_image.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interfaces for converting to/from internal C++ formats. */ +@interface RTC_OBJC_TYPE (RTCEncodedImage) +(Private) + + - (instancetype)initWithNativeEncodedImage : (const webrtc::EncodedImage &)encodedImage; +- (webrtc::EncodedImage)nativeEncodedImage; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm new file mode 100644 index 0000000000..c8936d3ad5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCEncodedImage+Private.mm @@ -0,0 +1,130 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCEncodedImage+Private.h" + +#import <objc/runtime.h> + +#include "rtc_base/numerics/safe_conversions.h" + +namespace { +// An implementation of EncodedImageBufferInterface that doesn't perform any copies. +class ObjCEncodedImageBuffer : public webrtc::EncodedImageBufferInterface { + public: + static rtc::scoped_refptr<ObjCEncodedImageBuffer> Create(NSData *data) { + return rtc::make_ref_counted<ObjCEncodedImageBuffer>(data); + } + const uint8_t *data() const override { return static_cast<const uint8_t *>(data_.bytes); } + // TODO(bugs.webrtc.org/9378): delete this non-const data method. + uint8_t *data() override { + return const_cast<uint8_t *>(static_cast<const uint8_t *>(data_.bytes)); + } + size_t size() const override { return data_.length; } + + protected: + explicit ObjCEncodedImageBuffer(NSData *data) : data_(data) {} + ~ObjCEncodedImageBuffer() {} + + NSData *data_; +}; +} + +// A simple wrapper around webrtc::EncodedImageBufferInterface to make it usable with associated +// objects. +@interface RTCWrappedEncodedImageBuffer : NSObject +@property(nonatomic) rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> buffer; +- (instancetype)initWithEncodedImageBuffer: + (rtc::scoped_refptr<webrtc::EncodedImageBufferInterface>)buffer; +@end +@implementation RTCWrappedEncodedImageBuffer +@synthesize buffer = _buffer; +- (instancetype)initWithEncodedImageBuffer: + (rtc::scoped_refptr<webrtc::EncodedImageBufferInterface>)buffer { + self = [super init]; + if (self) { + _buffer = buffer; + } + return self; +} +@end + +@implementation RTC_OBJC_TYPE (RTCEncodedImage) +(Private) + + - (rtc::scoped_refptr<webrtc::EncodedImageBufferInterface>)encodedData { + RTCWrappedEncodedImageBuffer *wrappedBuffer = + objc_getAssociatedObject(self, @selector(encodedData)); + return wrappedBuffer.buffer; +} + +- (void)setEncodedData:(rtc::scoped_refptr<webrtc::EncodedImageBufferInterface>)buffer { + return objc_setAssociatedObject( + self, + @selector(encodedData), + [[RTCWrappedEncodedImageBuffer alloc] initWithEncodedImageBuffer:buffer], + OBJC_ASSOCIATION_RETAIN_NONATOMIC); +} + +- (instancetype)initWithNativeEncodedImage:(const webrtc::EncodedImage &)encodedImage { + if (self = [super init]) { + // A reference to the encodedData must be stored so that it's kept alive as long + // self.buffer references its underlying data. + self.encodedData = encodedImage.GetEncodedData(); + // Wrap the buffer in NSData without copying, do not take ownership. + self.buffer = [NSData dataWithBytesNoCopy:self.encodedData->data() + length:encodedImage.size() + freeWhenDone:NO]; + self.encodedWidth = rtc::dchecked_cast<int32_t>(encodedImage._encodedWidth); + self.encodedHeight = rtc::dchecked_cast<int32_t>(encodedImage._encodedHeight); + self.timeStamp = encodedImage.RtpTimestamp(); + self.captureTimeMs = encodedImage.capture_time_ms_; + self.ntpTimeMs = encodedImage.ntp_time_ms_; + self.flags = encodedImage.timing_.flags; + self.encodeStartMs = encodedImage.timing_.encode_start_ms; + self.encodeFinishMs = encodedImage.timing_.encode_finish_ms; + self.frameType = static_cast<RTCFrameType>(encodedImage._frameType); + self.rotation = static_cast<RTCVideoRotation>(encodedImage.rotation_); + self.qp = @(encodedImage.qp_); + self.contentType = (encodedImage.content_type_ == webrtc::VideoContentType::SCREENSHARE) ? + RTCVideoContentTypeScreenshare : + RTCVideoContentTypeUnspecified; + } + + return self; +} + +- (webrtc::EncodedImage)nativeEncodedImage { + // Return the pointer without copying. + webrtc::EncodedImage encodedImage; + if (self.encodedData) { + encodedImage.SetEncodedData(self.encodedData); + } else if (self.buffer) { + encodedImage.SetEncodedData(ObjCEncodedImageBuffer::Create(self.buffer)); + } + encodedImage.set_size(self.buffer.length); + encodedImage._encodedWidth = rtc::dchecked_cast<uint32_t>(self.encodedWidth); + encodedImage._encodedHeight = rtc::dchecked_cast<uint32_t>(self.encodedHeight); + encodedImage.SetRtpTimestamp(self.timeStamp); + encodedImage.capture_time_ms_ = self.captureTimeMs; + encodedImage.ntp_time_ms_ = self.ntpTimeMs; + encodedImage.timing_.flags = self.flags; + encodedImage.timing_.encode_start_ms = self.encodeStartMs; + encodedImage.timing_.encode_finish_ms = self.encodeFinishMs; + encodedImage._frameType = webrtc::VideoFrameType(self.frameType); + encodedImage.rotation_ = webrtc::VideoRotation(self.rotation); + encodedImage.qp_ = self.qp ? self.qp.intValue : -1; + encodedImage.content_type_ = (self.contentType == RTCVideoContentTypeScreenshare) ? + webrtc::VideoContentType::SCREENSHARE : + webrtc::VideoContentType::UNSPECIFIED; + + return encodedImage; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.h new file mode 100644 index 0000000000..2c00e11721 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.h @@ -0,0 +1,30 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +/** The only valid value for the following if set is kRTCFieldTrialEnabledValue. */ +RTC_EXTERN NSString *const kRTCFieldTrialAudioForceABWENoTWCCKey; +RTC_EXTERN NSString *const kRTCFieldTrialFlexFec03AdvertisedKey; +RTC_EXTERN NSString *const kRTCFieldTrialFlexFec03Key; +RTC_EXTERN NSString *const kRTCFieldTrialH264HighProfileKey; +RTC_EXTERN NSString *const kRTCFieldTrialMinimizeResamplingOnMobileKey; +RTC_EXTERN NSString *const kRTCFieldTrialUseNWPathMonitor; + +/** The valid value for field trials above. */ +RTC_EXTERN NSString *const kRTCFieldTrialEnabledValue; + +/** Initialize field trials using a dictionary mapping field trial keys to their + * values. See above for valid keys and values. Must be called before any other + * call into WebRTC. See: webrtc/system_wrappers/include/field_trial.h + */ +RTC_EXTERN void RTCInitFieldTrialDictionary(NSDictionary<NSString *, NSString *> *fieldTrials); diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.mm new file mode 100644 index 0000000000..193da9e4f7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFieldTrials.mm @@ -0,0 +1,56 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCFieldTrials.h" + +#include <memory> + +#import "base/RTCLogging.h" + +#include "system_wrappers/include/field_trial.h" + +NSString *const kRTCFieldTrialAudioForceABWENoTWCCKey = @"WebRTC-Audio-ABWENoTWCC"; +NSString * const kRTCFieldTrialFlexFec03AdvertisedKey = @"WebRTC-FlexFEC-03-Advertised"; +NSString * const kRTCFieldTrialFlexFec03Key = @"WebRTC-FlexFEC-03"; +NSString * const kRTCFieldTrialH264HighProfileKey = @"WebRTC-H264HighProfile"; +NSString * const kRTCFieldTrialMinimizeResamplingOnMobileKey = + @"WebRTC-Audio-MinimizeResamplingOnMobile"; +NSString *const kRTCFieldTrialUseNWPathMonitor = @"WebRTC-Network-UseNWPathMonitor"; +NSString * const kRTCFieldTrialEnabledValue = @"Enabled"; + +// InitFieldTrialsFromString stores the char*, so the char array must outlive +// the application. +static char *gFieldTrialInitString = nullptr; + +void RTCInitFieldTrialDictionary(NSDictionary<NSString *, NSString *> *fieldTrials) { + if (!fieldTrials) { + RTCLogWarning(@"No fieldTrials provided."); + return; + } + // Assemble the keys and values into the field trial string. + // We don't perform any extra format checking. That should be done by the underlying WebRTC calls. + NSMutableString *fieldTrialInitString = [NSMutableString string]; + for (NSString *key in fieldTrials) { + NSString *fieldTrialEntry = [NSString stringWithFormat:@"%@/%@/", key, fieldTrials[key]]; + [fieldTrialInitString appendString:fieldTrialEntry]; + } + size_t len = fieldTrialInitString.length + 1; + if (gFieldTrialInitString != nullptr) { + delete[] gFieldTrialInitString; + } + gFieldTrialInitString = new char[len]; + if (![fieldTrialInitString getCString:gFieldTrialInitString + maxLength:len + encoding:NSUTF8StringEncoding]) { + RTCLogError(@"Failed to convert field trial string."); + return; + } + webrtc::field_trial::InitFieldTrialsFromString(gFieldTrialInitString); +} diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.h new file mode 100644 index 0000000000..cb397c9633 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.h @@ -0,0 +1,74 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +typedef NS_ENUM(NSUInteger, RTCFileLoggerSeverity) { + RTCFileLoggerSeverityVerbose, + RTCFileLoggerSeverityInfo, + RTCFileLoggerSeverityWarning, + RTCFileLoggerSeverityError +}; + +typedef NS_ENUM(NSUInteger, RTCFileLoggerRotationType) { + RTCFileLoggerTypeCall, + RTCFileLoggerTypeApp, +}; + +NS_ASSUME_NONNULL_BEGIN + +// This class intercepts WebRTC logs and saves them to a file. The file size +// will not exceed the given maximum bytesize. When the maximum bytesize is +// reached, logs are rotated according to the rotationType specified. +// For kRTCFileLoggerTypeCall, logs from the beginning and the end +// are preserved while the middle section is overwritten instead. +// For kRTCFileLoggerTypeApp, the oldest log is overwritten. +// This class is not threadsafe. +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCFileLogger) : NSObject + +// The severity level to capture. The default is kRTCFileLoggerSeverityInfo. +@property(nonatomic, assign) RTCFileLoggerSeverity severity; + +// The rotation type for this file logger. The default is +// kRTCFileLoggerTypeCall. +@property(nonatomic, readonly) RTCFileLoggerRotationType rotationType; + +// Disables buffering disk writes. Should be set before `start`. Buffering +// is enabled by default for performance. +@property(nonatomic, assign) BOOL shouldDisableBuffering; + +// Default constructor provides default settings for dir path, file size and +// rotation type. +- (instancetype)init; + +// Create file logger with default rotation type. +- (instancetype)initWithDirPath:(NSString *)dirPath maxFileSize:(NSUInteger)maxFileSize; + +- (instancetype)initWithDirPath:(NSString *)dirPath + maxFileSize:(NSUInteger)maxFileSize + rotationType:(RTCFileLoggerRotationType)rotationType NS_DESIGNATED_INITIALIZER; + +// Starts writing WebRTC logs to disk if not already started. Overwrites any +// existing file(s). +- (void)start; + +// Stops writing WebRTC logs to disk. This method is also called on dealloc. +- (void)stop; + +// Returns the current contents of the logs, or nil if start has been called +// without a stop. +- (nullable NSData *)logData; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.mm new file mode 100644 index 0000000000..9562245611 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCFileLogger.mm @@ -0,0 +1,170 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCFileLogger.h" + +#include <memory> + +#include "rtc_base/checks.h" +#include "rtc_base/file_rotating_stream.h" +#include "rtc_base/log_sinks.h" +#include "rtc_base/logging.h" + +NSString *const kDefaultLogDirName = @"webrtc_logs"; +NSUInteger const kDefaultMaxFileSize = 10 * 1024 * 1024; // 10MB. +const char *kRTCFileLoggerRotatingLogPrefix = "rotating_log"; + +@implementation RTC_OBJC_TYPE (RTCFileLogger) { + BOOL _hasStarted; + NSString *_dirPath; + NSUInteger _maxFileSize; + std::unique_ptr<rtc::FileRotatingLogSink> _logSink; +} + +@synthesize severity = _severity; +@synthesize rotationType = _rotationType; +@synthesize shouldDisableBuffering = _shouldDisableBuffering; + +- (instancetype)init { + NSArray *paths = NSSearchPathForDirectoriesInDomains( + NSDocumentDirectory, NSUserDomainMask, YES); + NSString *documentsDirPath = [paths firstObject]; + NSString *defaultDirPath = + [documentsDirPath stringByAppendingPathComponent:kDefaultLogDirName]; + return [self initWithDirPath:defaultDirPath + maxFileSize:kDefaultMaxFileSize]; +} + +- (instancetype)initWithDirPath:(NSString *)dirPath + maxFileSize:(NSUInteger)maxFileSize { + return [self initWithDirPath:dirPath + maxFileSize:maxFileSize + rotationType:RTCFileLoggerTypeCall]; +} + +- (instancetype)initWithDirPath:(NSString *)dirPath + maxFileSize:(NSUInteger)maxFileSize + rotationType:(RTCFileLoggerRotationType)rotationType { + NSParameterAssert(dirPath.length); + NSParameterAssert(maxFileSize); + if (self = [super init]) { + BOOL isDir = NO; + NSFileManager *fileManager = [NSFileManager defaultManager]; + if ([fileManager fileExistsAtPath:dirPath isDirectory:&isDir]) { + if (!isDir) { + // Bail if something already exists there. + return nil; + } + } else { + if (![fileManager createDirectoryAtPath:dirPath + withIntermediateDirectories:NO + attributes:nil + error:nil]) { + // Bail if we failed to create a directory. + return nil; + } + } + _dirPath = dirPath; + _maxFileSize = maxFileSize; + _severity = RTCFileLoggerSeverityInfo; + } + return self; +} + +- (void)dealloc { + [self stop]; +} + +- (void)start { + if (_hasStarted) { + return; + } + switch (_rotationType) { + case RTCFileLoggerTypeApp: + _logSink.reset( + new rtc::FileRotatingLogSink(_dirPath.UTF8String, + kRTCFileLoggerRotatingLogPrefix, + _maxFileSize, + _maxFileSize / 10)); + break; + case RTCFileLoggerTypeCall: + _logSink.reset( + new rtc::CallSessionFileRotatingLogSink(_dirPath.UTF8String, + _maxFileSize)); + break; + } + if (!_logSink->Init()) { + RTC_LOG(LS_ERROR) << "Failed to open log files at path: " << _dirPath.UTF8String; + _logSink.reset(); + return; + } + if (_shouldDisableBuffering) { + _logSink->DisableBuffering(); + } + rtc::LogMessage::LogThreads(true); + rtc::LogMessage::LogTimestamps(true); + rtc::LogMessage::AddLogToStream(_logSink.get(), [self rtcSeverity]); + _hasStarted = YES; +} + +- (void)stop { + if (!_hasStarted) { + return; + } + RTC_DCHECK(_logSink); + rtc::LogMessage::RemoveLogToStream(_logSink.get()); + _hasStarted = NO; + _logSink.reset(); +} + +- (nullable NSData *)logData { + if (_hasStarted) { + return nil; + } + NSMutableData* logData = [NSMutableData data]; + std::unique_ptr<rtc::FileRotatingStreamReader> stream; + switch(_rotationType) { + case RTCFileLoggerTypeApp: + stream = std::make_unique<rtc::FileRotatingStreamReader>(_dirPath.UTF8String, + kRTCFileLoggerRotatingLogPrefix); + break; + case RTCFileLoggerTypeCall: + stream = std::make_unique<rtc::CallSessionFileRotatingStreamReader>(_dirPath.UTF8String); + break; + } + size_t bufferSize = stream->GetSize(); + if (bufferSize == 0) { + return logData; + } + // Allocate memory using malloc so we can pass it direcly to NSData without + // copying. + std::unique_ptr<uint8_t[]> buffer(static_cast<uint8_t*>(malloc(bufferSize))); + size_t read = stream->ReadAll(buffer.get(), bufferSize); + logData = [[NSMutableData alloc] initWithBytesNoCopy:buffer.release() + length:read]; + return logData; +} + +#pragma mark - Private + +- (rtc::LoggingSeverity)rtcSeverity { + switch (_severity) { + case RTCFileLoggerSeverityVerbose: + return rtc::LS_VERBOSE; + case RTCFileLoggerSeverityInfo: + return rtc::LS_INFO; + case RTCFileLoggerSeverityWarning: + return rtc::LS_WARNING; + case RTCFileLoggerSeverityError: + return rtc::LS_ERROR; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate+Private.h new file mode 100644 index 0000000000..409e16b608 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate+Private.h @@ -0,0 +1,36 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceCandidate.h" + +#include <memory> + +#include "api/jsep.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCIceCandidate) +() + + /** + * The native IceCandidateInterface representation of this RTCIceCandidate + * object. This is needed to pass to the underlying C++ APIs. + */ + @property(nonatomic, readonly) std::unique_ptr<webrtc::IceCandidateInterface> nativeCandidate; + +/** + * Initialize an RTCIceCandidate from a native IceCandidateInterface. No + * ownership is taken of the native candidate. + */ +- (instancetype)initWithNativeCandidate:(const webrtc::IceCandidateInterface *)candidate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.h new file mode 100644 index 0000000000..f84843af6c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.h @@ -0,0 +1,49 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCIceCandidate) : NSObject + +/** + * If present, the identifier of the "media stream identification" for the media + * component this candidate is associated with. + */ +@property(nonatomic, readonly, nullable) NSString *sdpMid; + +/** + * The index (starting at zero) of the media description this candidate is + * associated with in the SDP. + */ +@property(nonatomic, readonly) int sdpMLineIndex; + +/** The SDP string for this candidate. */ +@property(nonatomic, readonly) NSString *sdp; + +/** The URL of the ICE server which this candidate is gathered from. */ +@property(nonatomic, readonly, nullable) NSString *serverUrl; + +- (instancetype)init NS_UNAVAILABLE; + +/** + * Initialize an RTCIceCandidate from SDP. + */ +- (instancetype)initWithSdp:(NSString *)sdp + sdpMLineIndex:(int)sdpMLineIndex + sdpMid:(nullable NSString *)sdpMid NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.mm new file mode 100644 index 0000000000..48385ef5b4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidate.mm @@ -0,0 +1,76 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceCandidate+Private.h" + +#include <memory> + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCIceCandidate) + +@synthesize sdpMid = _sdpMid; +@synthesize sdpMLineIndex = _sdpMLineIndex; +@synthesize sdp = _sdp; +@synthesize serverUrl = _serverUrl; + +- (instancetype)initWithSdp:(NSString *)sdp + sdpMLineIndex:(int)sdpMLineIndex + sdpMid:(NSString *)sdpMid { + NSParameterAssert(sdp.length); + if (self = [super init]) { + _sdpMid = [sdpMid copy]; + _sdpMLineIndex = sdpMLineIndex; + _sdp = [sdp copy]; + } + return self; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCIceCandidate):\n%@\n%d\n%@\n%@", + _sdpMid, + _sdpMLineIndex, + _sdp, + _serverUrl]; +} + +#pragma mark - Private + +- (instancetype)initWithNativeCandidate: + (const webrtc::IceCandidateInterface *)candidate { + NSParameterAssert(candidate); + std::string sdp; + candidate->ToString(&sdp); + + RTC_OBJC_TYPE(RTCIceCandidate) *rtcCandidate = + [self initWithSdp:[NSString stringForStdString:sdp] + sdpMLineIndex:candidate->sdp_mline_index() + sdpMid:[NSString stringForStdString:candidate->sdp_mid()]]; + rtcCandidate->_serverUrl = [NSString stringForStdString:candidate->server_url()]; + return rtcCandidate; +} + +- (std::unique_ptr<webrtc::IceCandidateInterface>)nativeCandidate { + webrtc::SdpParseError error; + + webrtc::IceCandidateInterface *candidate = webrtc::CreateIceCandidate( + _sdpMid.stdString, _sdpMLineIndex, _sdp.stdString, &error); + + if (!candidate) { + RTCLog(@"Failed to create ICE candidate: %s\nline: %s", + error.description.c_str(), + error.line.c_str()); + } + + return std::unique_ptr<webrtc::IceCandidateInterface>(candidate); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent+Private.h new file mode 100644 index 0000000000..8502da08a8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceCandidateErrorEvent.h" + +#include <string> + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCIceCandidateErrorEvent) +() + + - (instancetype)initWithAddress : (const std::string&)address port : (const int)port url + : (const std::string&)url errorCode : (const int)errorCode errorText + : (const std::string&)errorText; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.h new file mode 100644 index 0000000000..e0906fdbdd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCIceCandidateErrorEvent) : NSObject + +/** The local IP address used to communicate with the STUN or TURN server. */ +@property(nonatomic, readonly) NSString *address; + +/** The port used to communicate with the STUN or TURN server. */ +@property(nonatomic, readonly) int port; + +/** The STUN or TURN URL that identifies the STUN or TURN server for which the failure occurred. */ +@property(nonatomic, readonly) NSString *url; + +/** The numeric STUN error code returned by the STUN or TURN server. If no host candidate can reach + * the server, errorCode will be set to the value 701 which is outside the STUN error code range. + * This error is only fired once per server URL while in the RTCIceGatheringState of "gathering". */ +@property(nonatomic, readonly) int errorCode; + +/** The STUN reason text returned by the STUN or TURN server. If the server could not be reached, + * errorText will be set to an implementation-specific value providing details about the error. */ +@property(nonatomic, readonly) NSString *errorText; + +- (instancetype)init NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.mm new file mode 100644 index 0000000000..573e30642b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceCandidateErrorEvent.mm @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceCandidateErrorEvent+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCIceCandidateErrorEvent) + +@synthesize address = _address; +@synthesize port = _port; +@synthesize url = _url; +@synthesize errorCode = _errorCode; +@synthesize errorText = _errorText; + +- (instancetype)init { + return [super init]; +} + +- (instancetype)initWithAddress:(const std::string&)address + port:(const int)port + url:(const std::string&)url + errorCode:(const int)errorCode + errorText:(const std::string&)errorText { + if (self = [self init]) { + _address = [NSString stringForStdString:address]; + _port = port; + _url = [NSString stringForStdString:url]; + _errorCode = errorCode; + _errorText = [NSString stringForStdString:errorText]; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer+Private.h new file mode 100644 index 0000000000..3eee819965 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer+Private.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceServer.h" + +#include "api/peer_connection_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCIceServer) +() + + /** + * IceServer struct representation of this RTCIceServer object's data. + * This is needed to pass to the underlying C++ APIs. + */ + @property(nonatomic, readonly) webrtc::PeerConnectionInterface::IceServer nativeServer; + +/** Initialize an RTCIceServer from a native IceServer. */ +- (instancetype)initWithNativeServer:(webrtc::PeerConnectionInterface::IceServer)nativeServer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.h new file mode 100644 index 0000000000..7ddcbc1a1f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.h @@ -0,0 +1,114 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +typedef NS_ENUM(NSUInteger, RTCTlsCertPolicy) { + RTCTlsCertPolicySecure, + RTCTlsCertPolicyInsecureNoCheck +}; + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCIceServer) : NSObject + +/** URI(s) for this server represented as NSStrings. */ +@property(nonatomic, readonly) NSArray<NSString *> *urlStrings; + +/** Username to use if this RTCIceServer object is a TURN server. */ +@property(nonatomic, readonly, nullable) NSString *username; + +/** Credential to use if this RTCIceServer object is a TURN server. */ +@property(nonatomic, readonly, nullable) NSString *credential; + +/** + * TLS certificate policy to use if this RTCIceServer object is a TURN server. + */ +@property(nonatomic, readonly) RTCTlsCertPolicy tlsCertPolicy; + +/** + If the URIs in `urls` only contain IP addresses, this field can be used + to indicate the hostname, which may be necessary for TLS (using the SNI + extension). If `urls` itself contains the hostname, this isn't necessary. + */ +@property(nonatomic, readonly, nullable) NSString *hostname; + +/** List of protocols to be used in the TLS ALPN extension. */ +@property(nonatomic, readonly) NSArray<NSString *> *tlsAlpnProtocols; + +/** + List elliptic curves to be used in the TLS elliptic curves extension. + Only curve names supported by OpenSSL should be used (eg. "P-256","X25519"). + */ +@property(nonatomic, readonly) NSArray<NSString *> *tlsEllipticCurves; + +- (nonnull instancetype)init NS_UNAVAILABLE; + +/** Convenience initializer for a server with no authentication (e.g. STUN). */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings; + +/** + * Initialize an RTCIceServer with its associated URLs, optional username, + * optional credential, and credentialType. + */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(nullable NSString *)username + credential:(nullable NSString *)credential; + +/** + * Initialize an RTCIceServer with its associated URLs, optional username, + * optional credential, and TLS cert policy. + */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(nullable NSString *)username + credential:(nullable NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy; + +/** + * Initialize an RTCIceServer with its associated URLs, optional username, + * optional credential, TLS cert policy and hostname. + */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(nullable NSString *)username + credential:(nullable NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(nullable NSString *)hostname; + +/** + * Initialize an RTCIceServer with its associated URLs, optional username, + * optional credential, TLS cert policy, hostname and ALPN protocols. + */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(nullable NSString *)username + credential:(nullable NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(nullable NSString *)hostname + tlsAlpnProtocols:(NSArray<NSString *> *)tlsAlpnProtocols; + +/** + * Initialize an RTCIceServer with its associated URLs, optional username, + * optional credential, TLS cert policy, hostname, ALPN protocols and + * elliptic curves. + */ +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(nullable NSString *)username + credential:(nullable NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(nullable NSString *)hostname + tlsAlpnProtocols:(nullable NSArray<NSString *> *)tlsAlpnProtocols + tlsEllipticCurves:(nullable NSArray<NSString *> *)tlsEllipticCurves + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.mm new file mode 100644 index 0000000000..19a0a7e9e8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCIceServer.mm @@ -0,0 +1,196 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCIceServer+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCIceServer) + +@synthesize urlStrings = _urlStrings; +@synthesize username = _username; +@synthesize credential = _credential; +@synthesize tlsCertPolicy = _tlsCertPolicy; +@synthesize hostname = _hostname; +@synthesize tlsAlpnProtocols = _tlsAlpnProtocols; +@synthesize tlsEllipticCurves = _tlsEllipticCurves; + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings { + return [self initWithURLStrings:urlStrings + username:nil + credential:nil]; +} + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(NSString *)username + credential:(NSString *)credential { + return [self initWithURLStrings:urlStrings + username:username + credential:credential + tlsCertPolicy:RTCTlsCertPolicySecure]; +} + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(NSString *)username + credential:(NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy { + return [self initWithURLStrings:urlStrings + username:username + credential:credential + tlsCertPolicy:tlsCertPolicy + hostname:nil]; +} + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(NSString *)username + credential:(NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(NSString *)hostname { + return [self initWithURLStrings:urlStrings + username:username + credential:credential + tlsCertPolicy:tlsCertPolicy + hostname:hostname + tlsAlpnProtocols:[NSArray array]]; +} + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(NSString *)username + credential:(NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(NSString *)hostname + tlsAlpnProtocols:(NSArray<NSString *> *)tlsAlpnProtocols { + return [self initWithURLStrings:urlStrings + username:username + credential:credential + tlsCertPolicy:tlsCertPolicy + hostname:hostname + tlsAlpnProtocols:tlsAlpnProtocols + tlsEllipticCurves:[NSArray array]]; +} + +- (instancetype)initWithURLStrings:(NSArray<NSString *> *)urlStrings + username:(NSString *)username + credential:(NSString *)credential + tlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy + hostname:(NSString *)hostname + tlsAlpnProtocols:(NSArray<NSString *> *)tlsAlpnProtocols + tlsEllipticCurves:(NSArray<NSString *> *)tlsEllipticCurves { + NSParameterAssert(urlStrings.count); + if (self = [super init]) { + _urlStrings = [[NSArray alloc] initWithArray:urlStrings copyItems:YES]; + _username = [username copy]; + _credential = [credential copy]; + _tlsCertPolicy = tlsCertPolicy; + _hostname = [hostname copy]; + _tlsAlpnProtocols = [[NSArray alloc] initWithArray:tlsAlpnProtocols copyItems:YES]; + _tlsEllipticCurves = [[NSArray alloc] initWithArray:tlsEllipticCurves copyItems:YES]; + } + return self; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCIceServer):\n%@\n%@\n%@\n%@\n%@\n%@\n%@", + _urlStrings, + _username, + _credential, + [self stringForTlsCertPolicy:_tlsCertPolicy], + _hostname, + _tlsAlpnProtocols, + _tlsEllipticCurves]; +} + +#pragma mark - Private + +- (NSString *)stringForTlsCertPolicy:(RTCTlsCertPolicy)tlsCertPolicy { + switch (tlsCertPolicy) { + case RTCTlsCertPolicySecure: + return @"RTCTlsCertPolicySecure"; + case RTCTlsCertPolicyInsecureNoCheck: + return @"RTCTlsCertPolicyInsecureNoCheck"; + } +} + +- (webrtc::PeerConnectionInterface::IceServer)nativeServer { + __block webrtc::PeerConnectionInterface::IceServer iceServer; + + iceServer.username = [NSString stdStringForString:_username]; + iceServer.password = [NSString stdStringForString:_credential]; + iceServer.hostname = [NSString stdStringForString:_hostname]; + + [_tlsAlpnProtocols enumerateObjectsUsingBlock:^(NSString *proto, NSUInteger idx, BOOL *stop) { + iceServer.tls_alpn_protocols.push_back(proto.stdString); + }]; + + [_tlsEllipticCurves enumerateObjectsUsingBlock:^(NSString *curve, NSUInteger idx, BOOL *stop) { + iceServer.tls_elliptic_curves.push_back(curve.stdString); + }]; + + [_urlStrings enumerateObjectsUsingBlock:^(NSString *url, + NSUInteger idx, + BOOL *stop) { + iceServer.urls.push_back(url.stdString); + }]; + + switch (_tlsCertPolicy) { + case RTCTlsCertPolicySecure: + iceServer.tls_cert_policy = + webrtc::PeerConnectionInterface::kTlsCertPolicySecure; + break; + case RTCTlsCertPolicyInsecureNoCheck: + iceServer.tls_cert_policy = + webrtc::PeerConnectionInterface::kTlsCertPolicyInsecureNoCheck; + break; + } + return iceServer; +} + +- (instancetype)initWithNativeServer: + (webrtc::PeerConnectionInterface::IceServer)nativeServer { + NSMutableArray *urls = + [NSMutableArray arrayWithCapacity:nativeServer.urls.size()]; + for (auto const &url : nativeServer.urls) { + [urls addObject:[NSString stringForStdString:url]]; + } + NSString *username = [NSString stringForStdString:nativeServer.username]; + NSString *credential = [NSString stringForStdString:nativeServer.password]; + NSString *hostname = [NSString stringForStdString:nativeServer.hostname]; + NSMutableArray *tlsAlpnProtocols = + [NSMutableArray arrayWithCapacity:nativeServer.tls_alpn_protocols.size()]; + for (auto const &proto : nativeServer.tls_alpn_protocols) { + [tlsAlpnProtocols addObject:[NSString stringForStdString:proto]]; + } + NSMutableArray *tlsEllipticCurves = + [NSMutableArray arrayWithCapacity:nativeServer.tls_elliptic_curves.size()]; + for (auto const &curve : nativeServer.tls_elliptic_curves) { + [tlsEllipticCurves addObject:[NSString stringForStdString:curve]]; + } + RTCTlsCertPolicy tlsCertPolicy; + + switch (nativeServer.tls_cert_policy) { + case webrtc::PeerConnectionInterface::kTlsCertPolicySecure: + tlsCertPolicy = RTCTlsCertPolicySecure; + break; + case webrtc::PeerConnectionInterface::kTlsCertPolicyInsecureNoCheck: + tlsCertPolicy = RTCTlsCertPolicyInsecureNoCheck; + break; + } + + self = [self initWithURLStrings:urls + username:username + credential:credential + tlsCertPolicy:tlsCertPolicy + hostname:hostname + tlsAlpnProtocols:tlsAlpnProtocols + tlsEllipticCurves:tlsEllipticCurves]; + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport+Private.h new file mode 100644 index 0000000000..7374b2b72f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport+Private.h @@ -0,0 +1,25 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCLegacyStatsReport.h" + +#include "api/legacy_stats_types.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCLegacyStatsReport) +() + + /** Initialize an RTCLegacyStatsReport object from a native StatsReport. */ + - (instancetype)initWithNativeReport : (const webrtc::StatsReport &)nativeReport; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.h new file mode 100644 index 0000000000..b3bd12c5d7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.h @@ -0,0 +1,37 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** This does not currently conform to the spec. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCLegacyStatsReport) : NSObject + +/** Time since 1970-01-01T00:00:00Z in milliseconds. */ +@property(nonatomic, readonly) CFTimeInterval timestamp; + +/** The type of stats held by this object. */ +@property(nonatomic, readonly) NSString *type; + +/** The identifier for this object. */ +@property(nonatomic, readonly) NSString *reportId; + +/** A dictionary holding the actual stats. */ +@property(nonatomic, readonly) NSDictionary<NSString *, NSString *> *values; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.mm new file mode 100644 index 0000000000..bd7a1ad9c9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCLegacyStatsReport.mm @@ -0,0 +1,60 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCLegacyStatsReport+Private.h" + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "rtc_base/checks.h" + +@implementation RTC_OBJC_TYPE (RTCLegacyStatsReport) + +@synthesize timestamp = _timestamp; +@synthesize type = _type; +@synthesize reportId = _reportId; +@synthesize values = _values; + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCLegacyStatsReport):\n%@\n%@\n%f\n%@", + _reportId, + _type, + _timestamp, + _values]; +} + +#pragma mark - Private + +- (instancetype)initWithNativeReport:(const webrtc::StatsReport &)nativeReport { + if (self = [super init]) { + _timestamp = nativeReport.timestamp(); + _type = [NSString stringForStdString:nativeReport.TypeToString()]; + _reportId = [NSString stringForStdString: + nativeReport.id()->ToString()]; + + NSUInteger capacity = nativeReport.values().size(); + NSMutableDictionary *values = + [NSMutableDictionary dictionaryWithCapacity:capacity]; + for (auto const &valuePair : nativeReport.values()) { + NSString *key = [NSString stringForStdString: + valuePair.second->display_name()]; + NSString *value = [NSString stringForStdString: + valuePair.second->ToString()]; + + // Not expecting duplicate keys. + RTC_DCHECK(![values objectForKey:key]); + [values setObject:value forKey:key]; + } + _values = values; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints+Private.h new file mode 100644 index 0000000000..97eee8307d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints+Private.h @@ -0,0 +1,34 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaConstraints.h" + +#include <memory> + +#include "sdk/media_constraints.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCMediaConstraints) +() + + /** + * A MediaConstraints representation of this RTCMediaConstraints object. This is + * needed to pass to the underlying C++ APIs. + */ + - (std::unique_ptr<webrtc::MediaConstraints>)nativeConstraints; + +/** Return a native Constraints object representing these constraints */ ++ (webrtc::MediaConstraints::Constraints)nativeConstraintsForConstraints: + (NSDictionary<NSString*, NSString*>*)constraints; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.h new file mode 100644 index 0000000000..c5baf20c1d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.h @@ -0,0 +1,46 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Constraint keys for media sources. */ +/** The value for this key should be a base64 encoded string containing + * the data from the serialized configuration proto. + */ +RTC_EXTERN NSString *const kRTCMediaConstraintsAudioNetworkAdaptorConfig; + +/** Constraint keys for generating offers and answers. */ +RTC_EXTERN NSString *const kRTCMediaConstraintsIceRestart; +RTC_EXTERN NSString *const kRTCMediaConstraintsOfferToReceiveAudio; +RTC_EXTERN NSString *const kRTCMediaConstraintsOfferToReceiveVideo; +RTC_EXTERN NSString *const kRTCMediaConstraintsVoiceActivityDetection; + +/** Constraint values for Boolean parameters. */ +RTC_EXTERN NSString *const kRTCMediaConstraintsValueTrue; +RTC_EXTERN NSString *const kRTCMediaConstraintsValueFalse; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMediaConstraints) : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +/** Initialize with mandatory and/or optional constraints. */ +- (instancetype) + initWithMandatoryConstraints:(nullable NSDictionary<NSString *, NSString *> *)mandatory + optionalConstraints:(nullable NSDictionary<NSString *, NSString *> *)optional + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.mm new file mode 100644 index 0000000000..0f46e4b8fe --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaConstraints.mm @@ -0,0 +1,90 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaConstraints+Private.h" + +#import "helpers/NSString+StdString.h" + +#include <memory> + +NSString *const kRTCMediaConstraintsAudioNetworkAdaptorConfig = + @(webrtc::MediaConstraints::kAudioNetworkAdaptorConfig); + +NSString *const kRTCMediaConstraintsIceRestart = @(webrtc::MediaConstraints::kIceRestart); +NSString *const kRTCMediaConstraintsOfferToReceiveAudio = + @(webrtc::MediaConstraints::kOfferToReceiveAudio); +NSString *const kRTCMediaConstraintsOfferToReceiveVideo = + @(webrtc::MediaConstraints::kOfferToReceiveVideo); +NSString *const kRTCMediaConstraintsVoiceActivityDetection = + @(webrtc::MediaConstraints::kVoiceActivityDetection); + +NSString *const kRTCMediaConstraintsValueTrue = @(webrtc::MediaConstraints::kValueTrue); +NSString *const kRTCMediaConstraintsValueFalse = @(webrtc::MediaConstraints::kValueFalse); + +@implementation RTC_OBJC_TYPE (RTCMediaConstraints) { + NSDictionary<NSString *, NSString *> *_mandatory; + NSDictionary<NSString *, NSString *> *_optional; +} + +- (instancetype)initWithMandatoryConstraints: + (NSDictionary<NSString *, NSString *> *)mandatory + optionalConstraints: + (NSDictionary<NSString *, NSString *> *)optional { + if (self = [super init]) { + _mandatory = [[NSDictionary alloc] initWithDictionary:mandatory + copyItems:YES]; + _optional = [[NSDictionary alloc] initWithDictionary:optional + copyItems:YES]; + } + return self; +} + +- (NSString *)description { + return [NSString + stringWithFormat:@"RTC_OBJC_TYPE(RTCMediaConstraints):\n%@\n%@", _mandatory, _optional]; +} + +#pragma mark - Private + +- (std::unique_ptr<webrtc::MediaConstraints>)nativeConstraints { + webrtc::MediaConstraints::Constraints mandatory = + [[self class] nativeConstraintsForConstraints:_mandatory]; + webrtc::MediaConstraints::Constraints optional = + [[self class] nativeConstraintsForConstraints:_optional]; + + webrtc::MediaConstraints *nativeConstraints = + new webrtc::MediaConstraints(mandatory, optional); + return std::unique_ptr<webrtc::MediaConstraints>(nativeConstraints); +} + ++ (webrtc::MediaConstraints::Constraints)nativeConstraintsForConstraints: + (NSDictionary<NSString *, NSString *> *)constraints { + webrtc::MediaConstraints::Constraints nativeConstraints; + for (NSString *key in constraints) { + NSAssert([key isKindOfClass:[NSString class]], + @"%@ is not an NSString.", key); + NSString *value = [constraints objectForKey:key]; + NSAssert([value isKindOfClass:[NSString class]], + @"%@ is not an NSString.", value); + if ([kRTCMediaConstraintsAudioNetworkAdaptorConfig isEqualToString:key]) { + // This value is base64 encoded. + NSData *charData = [[NSData alloc] initWithBase64EncodedString:value options:0]; + std::string configValue = + std::string(reinterpret_cast<const char *>(charData.bytes), charData.length); + nativeConstraints.push_back(webrtc::MediaConstraints::Constraint(key.stdString, configValue)); + } else { + nativeConstraints.push_back( + webrtc::MediaConstraints::Constraint(key.stdString, value.stdString)); + } + } + return nativeConstraints; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource+Private.h new file mode 100644 index 0000000000..edda892e50 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource+Private.h @@ -0,0 +1,42 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaSource.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +typedef NS_ENUM(NSInteger, RTCMediaSourceType) { + RTCMediaSourceTypeAudio, + RTCMediaSourceTypeVideo, +}; + +@interface RTC_OBJC_TYPE (RTCMediaSource) +() + + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::MediaSourceInterface> nativeMediaSource; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type NS_DESIGNATED_INITIALIZER; + ++ (webrtc::MediaSourceInterface::SourceState)nativeSourceStateForState:(RTCSourceState)state; + ++ (RTCSourceState)sourceStateForNativeState:(webrtc::MediaSourceInterface::SourceState)nativeState; + ++ (NSString *)stringForState:(RTCSourceState)state; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.h new file mode 100644 index 0000000000..ba19c2a352 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.h @@ -0,0 +1,34 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +typedef NS_ENUM(NSInteger, RTCSourceState) { + RTCSourceStateInitializing, + RTCSourceStateLive, + RTCSourceStateEnded, + RTCSourceStateMuted, +}; + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMediaSource) : NSObject + +/** The current state of the RTCMediaSource. */ +@property(nonatomic, readonly) RTCSourceState state; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.mm new file mode 100644 index 0000000000..61472a782a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaSource.mm @@ -0,0 +1,82 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaSource+Private.h" + +#include "rtc_base/checks.h" + +@implementation RTC_OBJC_TYPE (RTCMediaSource) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + RTCMediaSourceType _type; +} + +@synthesize nativeMediaSource = _nativeMediaSource; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type { + RTC_DCHECK(factory); + RTC_DCHECK(nativeMediaSource); + if (self = [super init]) { + _factory = factory; + _nativeMediaSource = nativeMediaSource; + _type = type; + } + return self; +} + +- (RTCSourceState)state { + return [[self class] sourceStateForNativeState:_nativeMediaSource->state()]; +} + +#pragma mark - Private + ++ (webrtc::MediaSourceInterface::SourceState)nativeSourceStateForState: + (RTCSourceState)state { + switch (state) { + case RTCSourceStateInitializing: + return webrtc::MediaSourceInterface::kInitializing; + case RTCSourceStateLive: + return webrtc::MediaSourceInterface::kLive; + case RTCSourceStateEnded: + return webrtc::MediaSourceInterface::kEnded; + case RTCSourceStateMuted: + return webrtc::MediaSourceInterface::kMuted; + } +} + ++ (RTCSourceState)sourceStateForNativeState: + (webrtc::MediaSourceInterface::SourceState)nativeState { + switch (nativeState) { + case webrtc::MediaSourceInterface::kInitializing: + return RTCSourceStateInitializing; + case webrtc::MediaSourceInterface::kLive: + return RTCSourceStateLive; + case webrtc::MediaSourceInterface::kEnded: + return RTCSourceStateEnded; + case webrtc::MediaSourceInterface::kMuted: + return RTCSourceStateMuted; + } +} + ++ (NSString *)stringForState:(RTCSourceState)state { + switch (state) { + case RTCSourceStateInitializing: + return @"Initializing"; + case RTCSourceStateLive: + return @"Live"; + case RTCSourceStateEnded: + return @"Ended"; + case RTCSourceStateMuted: + return @"Muted"; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream+Private.h new file mode 100644 index 0000000000..6c8a602766 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream+Private.h @@ -0,0 +1,37 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaStream.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCMediaStream) +() + + /** + * MediaStreamInterface representation of this RTCMediaStream object. This is + * needed to pass to the underlying C++ APIs. + */ + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::MediaStreamInterface> nativeMediaStream; + +/** Initialize an RTCMediaStream with an id. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + streamId:(NSString *)streamId; + +/** Initialize an RTCMediaStream from a native MediaStreamInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaStream:(rtc::scoped_refptr<webrtc::MediaStreamInterface>)nativeMediaStream; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.h new file mode 100644 index 0000000000..2d56f15c7d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.h @@ -0,0 +1,49 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioTrack); +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); +@class RTC_OBJC_TYPE(RTCVideoTrack); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMediaStream) : NSObject + +/** The audio tracks in this stream. */ +@property(nonatomic, strong, readonly) NSArray<RTC_OBJC_TYPE(RTCAudioTrack) *> *audioTracks; + +/** The video tracks in this stream. */ +@property(nonatomic, strong, readonly) NSArray<RTC_OBJC_TYPE(RTCVideoTrack) *> *videoTracks; + +/** An identifier for this media stream. */ +@property(nonatomic, readonly) NSString *streamId; + +- (instancetype)init NS_UNAVAILABLE; + +/** Adds the given audio track to this media stream. */ +- (void)addAudioTrack:(RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrack; + +/** Adds the given video track to this media stream. */ +- (void)addVideoTrack:(RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrack; + +/** Removes the given audio track to this media stream. */ +- (void)removeAudioTrack:(RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrack; + +/** Removes the given video track to this media stream. */ +- (void)removeVideoTrack:(RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrack; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.mm new file mode 100644 index 0000000000..0018dd6945 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStream.mm @@ -0,0 +1,155 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaStream+Private.h" + +#import "RTCAudioTrack+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "RTCVideoTrack+Private.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCMediaStream) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::Thread *_signalingThread; + NSMutableArray *_audioTracks /* accessed on _signalingThread */; + NSMutableArray *_videoTracks /* accessed on _signalingThread */; + rtc::scoped_refptr<webrtc::MediaStreamInterface> _nativeMediaStream; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + streamId:(NSString *)streamId { + NSParameterAssert(factory); + NSParameterAssert(streamId.length); + std::string nativeId = [NSString stdStringForString:streamId]; + rtc::scoped_refptr<webrtc::MediaStreamInterface> stream = + factory.nativeFactory->CreateLocalMediaStream(nativeId); + return [self initWithFactory:factory nativeMediaStream:stream]; +} + +- (NSArray<RTC_OBJC_TYPE(RTCAudioTrack) *> *)audioTracks { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall([self]() { return self.audioTracks; }); + } + return [_audioTracks copy]; +} + +- (NSArray<RTC_OBJC_TYPE(RTCVideoTrack) *> *)videoTracks { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall([self]() { return self.videoTracks; }); + } + return [_videoTracks copy]; +} + +- (NSString *)streamId { + return [NSString stringForStdString:_nativeMediaStream->id()]; +} + +- (void)addAudioTrack:(RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrack { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall( + [audioTrack, self]() { return [self addAudioTrack:audioTrack]; }); + } + if (_nativeMediaStream->AddTrack(audioTrack.nativeAudioTrack)) { + [_audioTracks addObject:audioTrack]; + } +} + +- (void)addVideoTrack:(RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrack { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall( + [videoTrack, self]() { return [self addVideoTrack:videoTrack]; }); + } + if (_nativeMediaStream->AddTrack(videoTrack.nativeVideoTrack)) { + [_videoTracks addObject:videoTrack]; + } +} + +- (void)removeAudioTrack:(RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrack { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall( + [audioTrack, self]() { return [self removeAudioTrack:audioTrack]; }); + } + NSUInteger index = [_audioTracks indexOfObjectIdenticalTo:audioTrack]; + if (index == NSNotFound) { + RTC_LOG(LS_INFO) << "|removeAudioTrack| called on unexpected RTC_OBJC_TYPE(RTCAudioTrack)"; + return; + } + if (_nativeMediaStream->RemoveTrack(audioTrack.nativeAudioTrack)) { + [_audioTracks removeObjectAtIndex:index]; + } +} + +- (void)removeVideoTrack:(RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrack { + if (!_signalingThread->IsCurrent()) { + return _signalingThread->BlockingCall( + [videoTrack, self]() { return [self removeVideoTrack:videoTrack]; }); + } + NSUInteger index = [_videoTracks indexOfObjectIdenticalTo:videoTrack]; + if (index == NSNotFound) { + RTC_LOG(LS_INFO) << "|removeVideoTrack| called on unexpected RTC_OBJC_TYPE(RTCVideoTrack)"; + return; + } + + if (_nativeMediaStream->RemoveTrack(videoTrack.nativeVideoTrack)) { + [_videoTracks removeObjectAtIndex:index]; + } +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCMediaStream):\n%@\nA=%lu\nV=%lu", + self.streamId, + (unsigned long)self.audioTracks.count, + (unsigned long)self.videoTracks.count]; +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::MediaStreamInterface>)nativeMediaStream { + return _nativeMediaStream; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaStream: + (rtc::scoped_refptr<webrtc::MediaStreamInterface>)nativeMediaStream { + NSParameterAssert(nativeMediaStream); + if (self = [super init]) { + _factory = factory; + _signalingThread = factory.signalingThread; + + webrtc::AudioTrackVector audioTracks = nativeMediaStream->GetAudioTracks(); + webrtc::VideoTrackVector videoTracks = nativeMediaStream->GetVideoTracks(); + + _audioTracks = [NSMutableArray arrayWithCapacity:audioTracks.size()]; + _videoTracks = [NSMutableArray arrayWithCapacity:videoTracks.size()]; + _nativeMediaStream = nativeMediaStream; + + for (auto &track : audioTracks) { + RTCMediaStreamTrackType type = RTCMediaStreamTrackTypeAudio; + RTC_OBJC_TYPE(RTCAudioTrack) *audioTrack = + [[RTC_OBJC_TYPE(RTCAudioTrack) alloc] initWithFactory:_factory + nativeTrack:track + type:type]; + [_audioTracks addObject:audioTrack]; + } + + for (auto &track : videoTracks) { + RTCMediaStreamTrackType type = RTCMediaStreamTrackTypeVideo; + RTC_OBJC_TYPE(RTCVideoTrack) *videoTrack = + [[RTC_OBJC_TYPE(RTCVideoTrack) alloc] initWithFactory:_factory + nativeTrack:track + type:type]; + [_videoTracks addObject:videoTrack]; + } + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack+Private.h new file mode 100644 index 0000000000..df45c79f44 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack+Private.h @@ -0,0 +1,62 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaStreamTrack.h" + +#include "api/media_stream_interface.h" + +typedef NS_ENUM(NSInteger, RTCMediaStreamTrackType) { + RTCMediaStreamTrackTypeAudio, + RTCMediaStreamTrackTypeVideo, +}; + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +@interface RTC_OBJC_TYPE (RTCMediaStreamTrack) +() + + @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCPeerConnectionFactory) * + factory; + +/** + * The native MediaStreamTrackInterface passed in or created during + * construction. + */ +@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> nativeTrack; + +/** + * Initialize an RTCMediaStreamTrack from a native MediaStreamTrackInterface. + */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack + type:(RTCMediaStreamTrackType)type NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack; + +- (BOOL)isEqualToTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track; + ++ (webrtc::MediaStreamTrackInterface::TrackState)nativeTrackStateForState: + (RTCMediaStreamTrackState)state; + ++ (RTCMediaStreamTrackState)trackStateForNativeState: + (webrtc::MediaStreamTrackInterface::TrackState)nativeState; + ++ (NSString *)stringForState:(RTCMediaStreamTrackState)state; + ++ (RTC_OBJC_TYPE(RTCMediaStreamTrack) *) + mediaTrackForNativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack + factory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.h new file mode 100644 index 0000000000..2200122ccd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.h @@ -0,0 +1,50 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +/** + * Represents the state of the track. This exposes the same states in C++. + */ +typedef NS_ENUM(NSInteger, RTCMediaStreamTrackState) { + RTCMediaStreamTrackStateLive, + RTCMediaStreamTrackStateEnded +}; + +NS_ASSUME_NONNULL_BEGIN + +RTC_EXTERN NSString *const kRTCMediaStreamTrackKindAudio; +RTC_EXTERN NSString *const kRTCMediaStreamTrackKindVideo; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMediaStreamTrack) : NSObject + +/** + * The kind of track. For example, "audio" if this track represents an audio + * track and "video" if this track represents a video track. + */ +@property(nonatomic, readonly) NSString *kind; + +/** An identifier string. */ +@property(nonatomic, readonly) NSString *trackId; + +/** The enabled state of the track. */ +@property(nonatomic, assign) BOOL isEnabled; + +/** The state of the track. */ +@property(nonatomic, readonly) RTCMediaStreamTrackState readyState; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.mm new file mode 100644 index 0000000000..f1e128ca60 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMediaStreamTrack.mm @@ -0,0 +1,161 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioTrack+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCVideoTrack+Private.h" + +#import "helpers/NSString+StdString.h" + +NSString * const kRTCMediaStreamTrackKindAudio = + @(webrtc::MediaStreamTrackInterface::kAudioKind); +NSString * const kRTCMediaStreamTrackKindVideo = + @(webrtc::MediaStreamTrackInterface::kVideoKind); + +@implementation RTC_OBJC_TYPE (RTCMediaStreamTrack) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> _nativeTrack; + RTCMediaStreamTrackType _type; +} + +- (NSString *)kind { + return [NSString stringForStdString:_nativeTrack->kind()]; +} + +- (NSString *)trackId { + return [NSString stringForStdString:_nativeTrack->id()]; +} + +- (BOOL)isEnabled { + return _nativeTrack->enabled(); +} + +- (void)setIsEnabled:(BOOL)isEnabled { + _nativeTrack->set_enabled(isEnabled); +} + +- (RTCMediaStreamTrackState)readyState { + return [[self class] trackStateForNativeState:_nativeTrack->state()]; +} + +- (NSString *)description { + NSString *readyState = [[self class] stringForState:self.readyState]; + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCMediaStreamTrack):\n%@\n%@\n%@\n%@", + self.kind, + self.trackId, + self.isEnabled ? @"enabled" : @"disabled", + readyState]; +} + +- (BOOL)isEqual:(id)object { + if (self == object) { + return YES; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + return [self isEqualToTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)object]; +} + +- (NSUInteger)hash { + return (NSUInteger)_nativeTrack.get(); +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack { + return _nativeTrack; +} + +@synthesize factory = _factory; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack + type:(RTCMediaStreamTrackType)type { + NSParameterAssert(nativeTrack); + NSParameterAssert(factory); + if (self = [super init]) { + _factory = factory; + _nativeTrack = nativeTrack; + _type = type; + } + return self; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack { + NSParameterAssert(nativeTrack); + if (nativeTrack->kind() == + std::string(webrtc::MediaStreamTrackInterface::kAudioKind)) { + return [self initWithFactory:factory nativeTrack:nativeTrack type:RTCMediaStreamTrackTypeAudio]; + } + if (nativeTrack->kind() == + std::string(webrtc::MediaStreamTrackInterface::kVideoKind)) { + return [self initWithFactory:factory nativeTrack:nativeTrack type:RTCMediaStreamTrackTypeVideo]; + } + return nil; +} + +- (BOOL)isEqualToTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track { + if (!track) { + return NO; + } + return _nativeTrack == track.nativeTrack; +} + ++ (webrtc::MediaStreamTrackInterface::TrackState)nativeTrackStateForState: + (RTCMediaStreamTrackState)state { + switch (state) { + case RTCMediaStreamTrackStateLive: + return webrtc::MediaStreamTrackInterface::kLive; + case RTCMediaStreamTrackStateEnded: + return webrtc::MediaStreamTrackInterface::kEnded; + } +} + ++ (RTCMediaStreamTrackState)trackStateForNativeState: + (webrtc::MediaStreamTrackInterface::TrackState)nativeState { + switch (nativeState) { + case webrtc::MediaStreamTrackInterface::kLive: + return RTCMediaStreamTrackStateLive; + case webrtc::MediaStreamTrackInterface::kEnded: + return RTCMediaStreamTrackStateEnded; + } +} + ++ (NSString *)stringForState:(RTCMediaStreamTrackState)state { + switch (state) { + case RTCMediaStreamTrackStateLive: + return @"Live"; + case RTCMediaStreamTrackStateEnded: + return @"Ended"; + } +} + ++ (RTC_OBJC_TYPE(RTCMediaStreamTrack) *) + mediaTrackForNativeTrack:(rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeTrack + factory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory { + NSParameterAssert(nativeTrack); + NSParameterAssert(factory); + if (nativeTrack->kind() == webrtc::MediaStreamTrackInterface::kAudioKind) { + return [[RTC_OBJC_TYPE(RTCAudioTrack) alloc] initWithFactory:factory + nativeTrack:nativeTrack + type:RTCMediaStreamTrackTypeAudio]; + } else if (nativeTrack->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + return [[RTC_OBJC_TYPE(RTCVideoTrack) alloc] initWithFactory:factory + nativeTrack:nativeTrack + type:RTCMediaStreamTrackTypeVideo]; + } else { + return [[RTC_OBJC_TYPE(RTCMediaStreamTrack) alloc] initWithFactory:factory + nativeTrack:nativeTrack]; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.h new file mode 100644 index 0000000000..fddbb27c90 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.h @@ -0,0 +1,23 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCMetricsSampleInfo.h" + +/** + * Enables gathering of metrics (which can be fetched with + * RTCGetAndResetMetrics). Must be called before any other call into WebRTC. + */ +RTC_EXTERN void RTCEnableMetrics(void); + +/** Gets and clears native histograms. */ +RTC_EXTERN NSArray<RTC_OBJC_TYPE(RTCMetricsSampleInfo) *>* RTCGetAndResetMetrics(void); diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.mm new file mode 100644 index 0000000000..87eb8c0210 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetrics.mm @@ -0,0 +1,34 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMetrics.h" + +#import "RTCMetricsSampleInfo+Private.h" + +#include "rtc_base/string_utils.h" + +void RTCEnableMetrics(void) { + webrtc::metrics::Enable(); +} + +NSArray<RTC_OBJC_TYPE(RTCMetricsSampleInfo) *> *RTCGetAndResetMetrics(void) { + std::map<std::string, std::unique_ptr<webrtc::metrics::SampleInfo>, rtc::AbslStringViewCmp> + histograms; + webrtc::metrics::GetAndReset(&histograms); + + NSMutableArray *metrics = + [NSMutableArray arrayWithCapacity:histograms.size()]; + for (auto const &histogram : histograms) { + RTC_OBJC_TYPE(RTCMetricsSampleInfo) *metric = + [[RTC_OBJC_TYPE(RTCMetricsSampleInfo) alloc] initWithNativeSampleInfo:*histogram.second]; + [metrics addObject:metric]; + } + return metrics; +} diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo+Private.h new file mode 100644 index 0000000000..e4aa41f6c7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo+Private.h @@ -0,0 +1,25 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMetricsSampleInfo.h" + +#include "system_wrappers/include/metrics.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCMetricsSampleInfo) +() + + /** Initialize an RTCMetricsSampleInfo object from native SampleInfo. */ + - (instancetype)initWithNativeSampleInfo : (const webrtc::metrics::SampleInfo &)info; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.h new file mode 100644 index 0000000000..47a877b6fb --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.h @@ -0,0 +1,48 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMetricsSampleInfo) : NSObject + +/** + * Example of RTCMetricsSampleInfo: + * name: "WebRTC.Video.InputFramesPerSecond" + * min: 1 + * max: 100 + * bucketCount: 50 + * samples: [29]:2 [30]:1 + */ + +/** The name of the histogram. */ +@property(nonatomic, readonly) NSString *name; + +/** The minimum bucket value. */ +@property(nonatomic, readonly) int min; + +/** The maximum bucket value. */ +@property(nonatomic, readonly) int max; + +/** The number of buckets. */ +@property(nonatomic, readonly) int bucketCount; + +/** A dictionary holding the samples <value, # of events>. */ +@property(nonatomic, readonly) NSDictionary<NSNumber *, NSNumber *> *samples; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.mm new file mode 100644 index 0000000000..e4be94e90a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCMetricsSampleInfo.mm @@ -0,0 +1,43 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMetricsSampleInfo+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCMetricsSampleInfo) + +@synthesize name = _name; +@synthesize min = _min; +@synthesize max = _max; +@synthesize bucketCount = _bucketCount; +@synthesize samples = _samples; + +#pragma mark - Private + +- (instancetype)initWithNativeSampleInfo: + (const webrtc::metrics::SampleInfo &)info { + if (self = [super init]) { + _name = [NSString stringForStdString:info.name]; + _min = info.min; + _max = info.max; + _bucketCount = info.bucket_count; + + NSMutableDictionary *samples = + [NSMutableDictionary dictionaryWithCapacity:info.samples.size()]; + for (auto const &sample : info.samples) { + [samples setObject:@(sample.second) forKey:@(sample.first)]; + } + _samples = samples; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+DataChannel.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+DataChannel.mm new file mode 100644 index 0000000000..cb75f061d8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+DataChannel.mm @@ -0,0 +1,34 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnection+Private.h" + +#import "RTCDataChannel+Private.h" +#import "RTCDataChannelConfiguration+Private.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCPeerConnection) +(DataChannel) + + - (nullable RTC_OBJC_TYPE(RTCDataChannel) *)dataChannelForLabel + : (NSString *)label configuration + : (RTC_OBJC_TYPE(RTCDataChannelConfiguration) *)configuration { + std::string labelString = [NSString stdStringForString:label]; + const webrtc::DataChannelInit nativeInit = + configuration.nativeDataChannelInit; + auto result = self.nativePeerConnection->CreateDataChannelOrError(labelString, &nativeInit); + if (!result.ok()) { + return nil; + } + return [[RTC_OBJC_TYPE(RTCDataChannel) alloc] initWithFactory:self.factory + nativeDataChannel:result.MoveValue()]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Private.h new file mode 100644 index 0000000000..9714f504ac --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Private.h @@ -0,0 +1,143 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnection.h" + +#include "api/peer_connection_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +namespace webrtc { + +/** + * These objects are created by RTCPeerConnectionFactory to wrap an + * id<RTCPeerConnectionDelegate> and call methods on that interface. + */ +class PeerConnectionDelegateAdapter : public PeerConnectionObserver { + public: + PeerConnectionDelegateAdapter(RTC_OBJC_TYPE(RTCPeerConnection) * peerConnection); + ~PeerConnectionDelegateAdapter() override; + + void OnSignalingChange(PeerConnectionInterface::SignalingState new_state) override; + + void OnAddStream(rtc::scoped_refptr<MediaStreamInterface> stream) override; + + void OnRemoveStream(rtc::scoped_refptr<MediaStreamInterface> stream) override; + + void OnTrack(rtc::scoped_refptr<RtpTransceiverInterface> transceiver) override; + + void OnDataChannel(rtc::scoped_refptr<DataChannelInterface> data_channel) override; + + void OnRenegotiationNeeded() override; + + void OnIceConnectionChange(PeerConnectionInterface::IceConnectionState new_state) override; + + void OnStandardizedIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) override; + + void OnConnectionChange(PeerConnectionInterface::PeerConnectionState new_state) override; + + void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState new_state) override; + + void OnIceCandidate(const IceCandidateInterface *candidate) override; + + void OnIceCandidateError(const std::string &address, + int port, + const std::string &url, + int error_code, + const std::string &error_text) override; + + void OnIceCandidatesRemoved(const std::vector<cricket::Candidate> &candidates) override; + + void OnIceSelectedCandidatePairChanged(const cricket::CandidatePairChangeEvent &event) override; + + void OnAddTrack(rtc::scoped_refptr<RtpReceiverInterface> receiver, + const std::vector<rtc::scoped_refptr<MediaStreamInterface>> &streams) override; + + void OnRemoveTrack(rtc::scoped_refptr<RtpReceiverInterface> receiver) override; + + private: + __weak RTC_OBJC_TYPE(RTCPeerConnection) * peer_connection_; +}; + +} // namespace webrtc +@protocol RTC_OBJC_TYPE +(RTCSSLCertificateVerifier); + +@interface RTC_OBJC_TYPE (RTCPeerConnection) +() + + /** The factory used to create this RTCPeerConnection */ + @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCPeerConnectionFactory) * + factory; + +/** The native PeerConnectionInterface created during construction. */ +@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::PeerConnectionInterface> + nativePeerConnection; + +/** Initialize an RTCPeerConnection with a configuration, constraints, and + * delegate. + */ +- (nullable instancetype) + initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + configuration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + certificateVerifier:(nullable id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)>)certificateVerifier + delegate:(nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate; + +/** Initialize an RTCPeerConnection with a configuration, constraints, + * delegate and PeerConnectionDependencies. + */ +- (nullable instancetype) + initWithDependencies:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + configuration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + dependencies:(std::unique_ptr<webrtc::PeerConnectionDependencies>)dependencies + delegate:(nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate + NS_DESIGNATED_INITIALIZER; + ++ (webrtc::PeerConnectionInterface::SignalingState)nativeSignalingStateForState: + (RTCSignalingState)state; + ++ (RTCSignalingState)signalingStateForNativeState: + (webrtc::PeerConnectionInterface::SignalingState)nativeState; + ++ (NSString *)stringForSignalingState:(RTCSignalingState)state; + ++ (webrtc::PeerConnectionInterface::IceConnectionState)nativeIceConnectionStateForState: + (RTCIceConnectionState)state; + ++ (webrtc::PeerConnectionInterface::PeerConnectionState)nativeConnectionStateForState: + (RTCPeerConnectionState)state; + ++ (RTCIceConnectionState)iceConnectionStateForNativeState: + (webrtc::PeerConnectionInterface::IceConnectionState)nativeState; + ++ (RTCPeerConnectionState)connectionStateForNativeState: + (webrtc::PeerConnectionInterface::PeerConnectionState)nativeState; + ++ (NSString *)stringForIceConnectionState:(RTCIceConnectionState)state; + ++ (NSString *)stringForConnectionState:(RTCPeerConnectionState)state; + ++ (webrtc::PeerConnectionInterface::IceGatheringState)nativeIceGatheringStateForState: + (RTCIceGatheringState)state; + ++ (RTCIceGatheringState)iceGatheringStateForNativeState: + (webrtc::PeerConnectionInterface::IceGatheringState)nativeState; + ++ (NSString *)stringForIceGatheringState:(RTCIceGatheringState)state; + ++ (webrtc::PeerConnectionInterface::StatsOutputLevel)nativeStatsOutputLevelForLevel: + (RTCStatsOutputLevel)level; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Stats.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Stats.mm new file mode 100644 index 0000000000..f8d38143f3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection+Stats.mm @@ -0,0 +1,102 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnection+Private.h" + +#import "RTCLegacyStatsReport+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCRtpReceiver+Private.h" +#import "RTCRtpSender+Private.h" +#import "RTCStatisticsReport+Private.h" +#import "helpers/NSString+StdString.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +class StatsCollectorCallbackAdapter : public RTCStatsCollectorCallback { + public: + StatsCollectorCallbackAdapter(RTCStatisticsCompletionHandler completion_handler) + : completion_handler_(completion_handler) {} + + void OnStatsDelivered(const rtc::scoped_refptr<const RTCStatsReport> &report) override { + RTC_DCHECK(completion_handler_); + RTC_OBJC_TYPE(RTCStatisticsReport) *statisticsReport = + [[RTC_OBJC_TYPE(RTCStatisticsReport) alloc] initWithReport:*report]; + completion_handler_(statisticsReport); + completion_handler_ = nil; + } + + private: + RTCStatisticsCompletionHandler completion_handler_; +}; + +class StatsObserverAdapter : public StatsObserver { + public: + StatsObserverAdapter( + void (^completionHandler)(NSArray<RTC_OBJC_TYPE(RTCLegacyStatsReport) *> *stats)) { + completion_handler_ = completionHandler; + } + + ~StatsObserverAdapter() override { completion_handler_ = nil; } + + void OnComplete(const StatsReports& reports) override { + RTC_DCHECK(completion_handler_); + NSMutableArray *stats = [NSMutableArray arrayWithCapacity:reports.size()]; + for (const auto* report : reports) { + RTC_OBJC_TYPE(RTCLegacyStatsReport) *statsReport = + [[RTC_OBJC_TYPE(RTCLegacyStatsReport) alloc] initWithNativeReport:*report]; + [stats addObject:statsReport]; + } + completion_handler_(stats); + completion_handler_ = nil; + } + + private: + void (^completion_handler_)(NSArray<RTC_OBJC_TYPE(RTCLegacyStatsReport) *> *stats); +}; +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCPeerConnection) +(Stats) + + - (void)statisticsForSender : (RTC_OBJC_TYPE(RTCRtpSender) *)sender completionHandler + : (RTCStatisticsCompletionHandler)completionHandler { + rtc::scoped_refptr<webrtc::StatsCollectorCallbackAdapter> collector = + rtc::make_ref_counted<webrtc::StatsCollectorCallbackAdapter>(completionHandler); + self.nativePeerConnection->GetStats(sender.nativeRtpSender, collector); +} + +- (void)statisticsForReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)receiver + completionHandler:(RTCStatisticsCompletionHandler)completionHandler { + rtc::scoped_refptr<webrtc::StatsCollectorCallbackAdapter> collector = + rtc::make_ref_counted<webrtc::StatsCollectorCallbackAdapter>(completionHandler); + self.nativePeerConnection->GetStats(receiver.nativeRtpReceiver, collector); +} + +- (void)statisticsWithCompletionHandler:(RTCStatisticsCompletionHandler)completionHandler { + rtc::scoped_refptr<webrtc::StatsCollectorCallbackAdapter> collector = + rtc::make_ref_counted<webrtc::StatsCollectorCallbackAdapter>(completionHandler); + self.nativePeerConnection->GetStats(collector.get()); +} + +- (void)statsForTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)mediaStreamTrack + statsOutputLevel:(RTCStatsOutputLevel)statsOutputLevel + completionHandler: + (void (^)(NSArray<RTC_OBJC_TYPE(RTCLegacyStatsReport) *> *stats))completionHandler { + rtc::scoped_refptr<webrtc::StatsObserverAdapter> observer = + rtc::make_ref_counted<webrtc::StatsObserverAdapter>(completionHandler); + webrtc::PeerConnectionInterface::StatsOutputLevel nativeOutputLevel = + [[self class] nativeStatsOutputLevelForLevel:statsOutputLevel]; + self.nativePeerConnection->GetStats( + observer.get(), mediaStreamTrack.nativeTrack.get(), nativeOutputLevel); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.h new file mode 100644 index 0000000000..466e053492 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.h @@ -0,0 +1,397 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +@class RTC_OBJC_TYPE(RTCConfiguration); +@class RTC_OBJC_TYPE(RTCDataChannel); +@class RTC_OBJC_TYPE(RTCDataChannelConfiguration); +@class RTC_OBJC_TYPE(RTCIceCandidate); +@class RTC_OBJC_TYPE(RTCIceCandidateErrorEvent); +@class RTC_OBJC_TYPE(RTCMediaConstraints); +@class RTC_OBJC_TYPE(RTCMediaStream); +@class RTC_OBJC_TYPE(RTCMediaStreamTrack); +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); +@class RTC_OBJC_TYPE(RTCRtpReceiver); +@class RTC_OBJC_TYPE(RTCRtpSender); +@class RTC_OBJC_TYPE(RTCRtpTransceiver); +@class RTC_OBJC_TYPE(RTCRtpTransceiverInit); +@class RTC_OBJC_TYPE(RTCSessionDescription); +@class RTC_OBJC_TYPE(RTCStatisticsReport); +@class RTC_OBJC_TYPE(RTCLegacyStatsReport); + +typedef NS_ENUM(NSInteger, RTCRtpMediaType); + +NS_ASSUME_NONNULL_BEGIN + +extern NSString *const kRTCPeerConnectionErrorDomain; +extern int const kRTCSessionDescriptionErrorCode; + +/** Represents the signaling state of the peer connection. */ +typedef NS_ENUM(NSInteger, RTCSignalingState) { + RTCSignalingStateStable, + RTCSignalingStateHaveLocalOffer, + RTCSignalingStateHaveLocalPrAnswer, + RTCSignalingStateHaveRemoteOffer, + RTCSignalingStateHaveRemotePrAnswer, + // Not an actual state, represents the total number of states. + RTCSignalingStateClosed, +}; + +/** Represents the ice connection state of the peer connection. */ +typedef NS_ENUM(NSInteger, RTCIceConnectionState) { + RTCIceConnectionStateNew, + RTCIceConnectionStateChecking, + RTCIceConnectionStateConnected, + RTCIceConnectionStateCompleted, + RTCIceConnectionStateFailed, + RTCIceConnectionStateDisconnected, + RTCIceConnectionStateClosed, + RTCIceConnectionStateCount, +}; + +/** Represents the combined ice+dtls connection state of the peer connection. */ +typedef NS_ENUM(NSInteger, RTCPeerConnectionState) { + RTCPeerConnectionStateNew, + RTCPeerConnectionStateConnecting, + RTCPeerConnectionStateConnected, + RTCPeerConnectionStateDisconnected, + RTCPeerConnectionStateFailed, + RTCPeerConnectionStateClosed, +}; + +/** Represents the ice gathering state of the peer connection. */ +typedef NS_ENUM(NSInteger, RTCIceGatheringState) { + RTCIceGatheringStateNew, + RTCIceGatheringStateGathering, + RTCIceGatheringStateComplete, +}; + +/** Represents the stats output level. */ +typedef NS_ENUM(NSInteger, RTCStatsOutputLevel) { + RTCStatsOutputLevelStandard, + RTCStatsOutputLevelDebug, +}; + +typedef void (^RTCCreateSessionDescriptionCompletionHandler)( + RTC_OBJC_TYPE(RTCSessionDescription) *_Nullable sdp, NSError *_Nullable error); + +typedef void (^RTCSetSessionDescriptionCompletionHandler)(NSError *_Nullable error); + +@class RTC_OBJC_TYPE(RTCPeerConnection); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCPeerConnectionDelegate)<NSObject> + + /** Called when the SignalingState changed. */ + - (void)peerConnection + : (RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection didChangeSignalingState + : (RTCSignalingState)stateChanged; + +/** Called when media is received on a new stream from remote peer. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didAddStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream; + +/** Called when a remote peer closes a stream. + * This is not called when RTCSdpSemanticsUnifiedPlan is specified. + */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didRemoveStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream; + +/** Called when negotiation is needed, for example ICE has restarted. */ +- (void)peerConnectionShouldNegotiate:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection; + +/** Called any time the IceConnectionState changes. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didChangeIceConnectionState:(RTCIceConnectionState)newState; + +/** Called any time the IceGatheringState changes. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didChangeIceGatheringState:(RTCIceGatheringState)newState; + +/** New ice candidate has been found. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didGenerateIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)candidate; + +/** Called when a group of local Ice candidates have been removed. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didRemoveIceCandidates:(NSArray<RTC_OBJC_TYPE(RTCIceCandidate) *> *)candidates; + +/** New data channel has been opened. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didOpenDataChannel:(RTC_OBJC_TYPE(RTCDataChannel) *)dataChannel; + +/** Called when signaling indicates a transceiver will be receiving media from + * the remote endpoint. + * This is only called with RTCSdpSemanticsUnifiedPlan specified. + */ +@optional +/** Called any time the IceConnectionState changes following standardized + * transition. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didChangeStandardizedIceConnectionState:(RTCIceConnectionState)newState; + +/** Called any time the PeerConnectionState changes. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didChangeConnectionState:(RTCPeerConnectionState)newState; + +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didStartReceivingOnTransceiver:(RTC_OBJC_TYPE(RTCRtpTransceiver) *)transceiver; + +/** Called when a receiver and its track are created. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didAddReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)rtpReceiver + streams:(NSArray<RTC_OBJC_TYPE(RTCMediaStream) *> *)mediaStreams; + +/** Called when the receiver and its track are removed. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didRemoveReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)rtpReceiver; + +/** Called when the selected ICE candidate pair is changed. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didChangeLocalCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)local + remoteCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)remote + lastReceivedMs:(int)lastDataReceivedMs + changeReason:(NSString *)reason; + +/** Called when gathering of an ICE candidate failed. */ +- (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection + didFailToGatherIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidateErrorEvent) *)event; + +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCPeerConnection) : NSObject + +/** The object that will be notifed about events such as state changes and + * streams being added or removed. + */ +@property(nonatomic, weak, nullable) id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)> delegate; +/** This property is not available with RTCSdpSemanticsUnifiedPlan. Please use + * `senders` instead. + */ +@property(nonatomic, readonly) NSArray<RTC_OBJC_TYPE(RTCMediaStream) *> *localStreams; +@property(nonatomic, readonly, nullable) RTC_OBJC_TYPE(RTCSessionDescription) * localDescription; +@property(nonatomic, readonly, nullable) RTC_OBJC_TYPE(RTCSessionDescription) * remoteDescription; +@property(nonatomic, readonly) RTCSignalingState signalingState; +@property(nonatomic, readonly) RTCIceConnectionState iceConnectionState; +@property(nonatomic, readonly) RTCPeerConnectionState connectionState; +@property(nonatomic, readonly) RTCIceGatheringState iceGatheringState; +@property(nonatomic, readonly, copy) RTC_OBJC_TYPE(RTCConfiguration) * configuration; + +/** Gets all RTCRtpSenders associated with this peer connection. + * Note: reading this property returns different instances of RTCRtpSender. + * Use isEqual: instead of == to compare RTCRtpSender instances. + */ +@property(nonatomic, readonly) NSArray<RTC_OBJC_TYPE(RTCRtpSender) *> *senders; + +/** Gets all RTCRtpReceivers associated with this peer connection. + * Note: reading this property returns different instances of RTCRtpReceiver. + * Use isEqual: instead of == to compare RTCRtpReceiver instances. + */ +@property(nonatomic, readonly) NSArray<RTC_OBJC_TYPE(RTCRtpReceiver) *> *receivers; + +/** Gets all RTCRtpTransceivers associated with this peer connection. + * Note: reading this property returns different instances of + * RTCRtpTransceiver. Use isEqual: instead of == to compare + * RTCRtpTransceiver instances. This is only available with + * RTCSdpSemanticsUnifiedPlan specified. + */ +@property(nonatomic, readonly) NSArray<RTC_OBJC_TYPE(RTCRtpTransceiver) *> *transceivers; + +- (instancetype)init NS_UNAVAILABLE; + +/** Sets the PeerConnection's global configuration to `configuration`. + * Any changes to STUN/TURN servers or ICE candidate policy will affect the + * next gathering phase, and cause the next call to createOffer to generate + * new ICE credentials. Note that the BUNDLE and RTCP-multiplexing policies + * cannot be changed with this method. + */ +- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration; + +/** Terminate all media and close the transport. */ +- (void)close; + +/** Provide a remote candidate to the ICE Agent. */ +- (void)addIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)candidate + DEPRECATED_MSG_ATTRIBUTE("Please use addIceCandidate:completionHandler: instead"); + +/** Provide a remote candidate to the ICE Agent. */ +- (void)addIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)candidate + completionHandler:(void (^)(NSError *_Nullable error))completionHandler; + +/** Remove a group of remote candidates from the ICE Agent. */ +- (void)removeIceCandidates:(NSArray<RTC_OBJC_TYPE(RTCIceCandidate) *> *)candidates; + +/** Add a new media stream to be sent on this peer connection. + * This method is not supported with RTCSdpSemanticsUnifiedPlan. Please use + * addTrack instead. + */ +- (void)addStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream; + +/** Remove the given media stream from this peer connection. + * This method is not supported with RTCSdpSemanticsUnifiedPlan. Please use + * removeTrack instead. + */ +- (void)removeStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream; + +/** Add a new media stream track to be sent on this peer connection, and return + * the newly created RTCRtpSender. The RTCRtpSender will be + * associated with the streams specified in the `streamIds` list. + * + * Errors: If an error occurs, returns nil. An error can occur if: + * - A sender already exists for the track. + * - The peer connection is closed. + */ +- (nullable RTC_OBJC_TYPE(RTCRtpSender) *)addTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track + streamIds:(NSArray<NSString *> *)streamIds; + +/** With PlanB semantics, removes an RTCRtpSender from this peer connection. + * + * With UnifiedPlan semantics, sets sender's track to null and removes the + * send component from the associated RTCRtpTransceiver's direction. + * + * Returns YES on success. + */ +- (BOOL)removeTrack:(RTC_OBJC_TYPE(RTCRtpSender) *)sender; + +/** addTransceiver creates a new RTCRtpTransceiver and adds it to the set of + * transceivers. Adding a transceiver will cause future calls to CreateOffer + * to add a media description for the corresponding transceiver. + * + * The initial value of `mid` in the returned transceiver is nil. Setting a + * new session description may change it to a non-nil value. + * + * https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-addtransceiver + * + * Optionally, an RtpTransceiverInit structure can be specified to configure + * the transceiver from construction. If not specified, the transceiver will + * default to having a direction of kSendRecv and not be part of any streams. + * + * These methods are only available when Unified Plan is enabled (see + * RTCConfiguration). + */ + +/** Adds a transceiver with a sender set to transmit the given track. The kind + * of the transceiver (and sender/receiver) will be derived from the kind of + * the track. + */ +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *)addTransceiverWithTrack: + (RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track; +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *) + addTransceiverWithTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track + init:(RTC_OBJC_TYPE(RTCRtpTransceiverInit) *)init; + +/** Adds a transceiver with the given kind. Can either be RTCRtpMediaTypeAudio + * or RTCRtpMediaTypeVideo. + */ +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *)addTransceiverOfType:(RTCRtpMediaType)mediaType; +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *) + addTransceiverOfType:(RTCRtpMediaType)mediaType + init:(RTC_OBJC_TYPE(RTCRtpTransceiverInit) *)init; + +/** Tells the PeerConnection that ICE should be restarted. This triggers a need + * for negotiation and subsequent offerForConstraints:completionHandler call will act as if + * RTCOfferAnswerOptions::ice_restart is true. + */ +- (void)restartIce; + +/** Generate an SDP offer. */ +- (void)offerForConstraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + completionHandler:(RTCCreateSessionDescriptionCompletionHandler)completionHandler; + +/** Generate an SDP answer. */ +- (void)answerForConstraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + completionHandler:(RTCCreateSessionDescriptionCompletionHandler)completionHandler; + +/** Apply the supplied RTCSessionDescription as the local description. */ +- (void)setLocalDescription:(RTC_OBJC_TYPE(RTCSessionDescription) *)sdp + completionHandler:(RTCSetSessionDescriptionCompletionHandler)completionHandler; + +/** Creates an offer or answer (depending on current signaling state) and sets + * it as the local session description. */ +- (void)setLocalDescriptionWithCompletionHandler: + (RTCSetSessionDescriptionCompletionHandler)completionHandler; + +/** Apply the supplied RTCSessionDescription as the remote description. */ +- (void)setRemoteDescription:(RTC_OBJC_TYPE(RTCSessionDescription) *)sdp + completionHandler:(RTCSetSessionDescriptionCompletionHandler)completionHandler; + +/** Limits the bandwidth allocated for all RTP streams sent by this + * PeerConnection. Nil parameters will be unchanged. Setting + * `currentBitrateBps` will force the available bitrate estimate to the given + * value. Returns YES if the parameters were successfully updated. + */ +- (BOOL)setBweMinBitrateBps:(nullable NSNumber *)minBitrateBps + currentBitrateBps:(nullable NSNumber *)currentBitrateBps + maxBitrateBps:(nullable NSNumber *)maxBitrateBps; + +/** Start or stop recording an Rtc EventLog. */ +- (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes; +- (void)stopRtcEventLog; + +@end + +@interface RTC_OBJC_TYPE (RTCPeerConnection) +(Media) + + /** Create an RTCRtpSender with the specified kind and media stream ID. + * See RTCMediaStreamTrack.h for available kinds. + * This method is not supported with RTCSdpSemanticsUnifiedPlan. Please use + * addTransceiver instead. + */ + - (RTC_OBJC_TYPE(RTCRtpSender) *)senderWithKind : (NSString *)kind streamId + : (NSString *)streamId; + +@end + +@interface RTC_OBJC_TYPE (RTCPeerConnection) +(DataChannel) + + /** Create a new data channel with the given label and configuration. */ + - (nullable RTC_OBJC_TYPE(RTCDataChannel) *)dataChannelForLabel + : (NSString *)label configuration : (RTC_OBJC_TYPE(RTCDataChannelConfiguration) *)configuration; + +@end + +typedef void (^RTCStatisticsCompletionHandler)(RTC_OBJC_TYPE(RTCStatisticsReport) *); + +@interface RTC_OBJC_TYPE (RTCPeerConnection) +(Stats) + + /** Gather stats for the given RTCMediaStreamTrack. If `mediaStreamTrack` is nil + * statistics are gathered for all tracks. + */ + - (void)statsForTrack + : (nullable RTC_OBJC_TYPE(RTCMediaStreamTrack) *)mediaStreamTrack statsOutputLevel + : (RTCStatsOutputLevel)statsOutputLevel completionHandler + : (nullable void (^)(NSArray<RTC_OBJC_TYPE(RTCLegacyStatsReport) *> *stats))completionHandler; + +/** Gather statistic through the v2 statistics API. */ +- (void)statisticsWithCompletionHandler:(RTCStatisticsCompletionHandler)completionHandler; + +/** Spec-compliant getStats() performing the stats selection algorithm with the + * sender. + */ +- (void)statisticsForSender:(RTC_OBJC_TYPE(RTCRtpSender) *)sender + completionHandler:(RTCStatisticsCompletionHandler)completionHandler; + +/** Spec-compliant getStats() performing the stats selection algorithm with the + * receiver. + */ +- (void)statisticsForReceiver:(RTC_OBJC_TYPE(RTCRtpReceiver) *)receiver + completionHandler:(RTCStatisticsCompletionHandler)completionHandler; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.mm new file mode 100644 index 0000000000..e55c8a4a3e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnection.mm @@ -0,0 +1,934 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnection+Private.h" + +#import "RTCConfiguration+Private.h" +#import "RTCDataChannel+Private.h" +#import "RTCIceCandidate+Private.h" +#import "RTCIceCandidateErrorEvent+Private.h" +#import "RTCLegacyStatsReport+Private.h" +#import "RTCMediaConstraints+Private.h" +#import "RTCMediaStream+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "RTCRtpReceiver+Private.h" +#import "RTCRtpSender+Private.h" +#import "RTCRtpTransceiver+Private.h" +#import "RTCSessionDescription+Private.h" +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include <memory> + +#include "api/jsep_ice_candidate.h" +#include "api/rtc_event_log_output_file.h" +#include "api/set_local_description_observer_interface.h" +#include "api/set_remote_description_observer_interface.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "sdk/objc/native/api/ssl_certificate_verifier.h" + +NSString *const kRTCPeerConnectionErrorDomain = @"org.webrtc.RTC_OBJC_TYPE(RTCPeerConnection)"; +int const kRTCPeerConnnectionSessionDescriptionError = -1; + +namespace { + +class SetSessionDescriptionObserver : public webrtc::SetLocalDescriptionObserverInterface, + public webrtc::SetRemoteDescriptionObserverInterface { + public: + SetSessionDescriptionObserver(RTCSetSessionDescriptionCompletionHandler completionHandler) { + completion_handler_ = completionHandler; + } + + virtual void OnSetLocalDescriptionComplete(webrtc::RTCError error) override { + OnCompelete(error); + } + + virtual void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override { + OnCompelete(error); + } + + private: + void OnCompelete(webrtc::RTCError error) { + RTC_DCHECK(completion_handler_ != nil); + if (error.ok()) { + completion_handler_(nil); + } else { + // TODO(hta): Add handling of error.type() + NSString *str = [NSString stringForStdString:error.message()]; + NSError *err = [NSError errorWithDomain:kRTCPeerConnectionErrorDomain + code:kRTCPeerConnnectionSessionDescriptionError + userInfo:@{NSLocalizedDescriptionKey : str}]; + completion_handler_(err); + } + completion_handler_ = nil; + } + RTCSetSessionDescriptionCompletionHandler completion_handler_; +}; + +} // anonymous namespace + +namespace webrtc { + +class CreateSessionDescriptionObserverAdapter + : public CreateSessionDescriptionObserver { + public: + CreateSessionDescriptionObserverAdapter(void (^completionHandler)( + RTC_OBJC_TYPE(RTCSessionDescription) * sessionDescription, NSError *error)) { + completion_handler_ = completionHandler; + } + + ~CreateSessionDescriptionObserverAdapter() override { completion_handler_ = nil; } + + void OnSuccess(SessionDescriptionInterface *desc) override { + RTC_DCHECK(completion_handler_); + std::unique_ptr<webrtc::SessionDescriptionInterface> description = + std::unique_ptr<webrtc::SessionDescriptionInterface>(desc); + RTC_OBJC_TYPE(RTCSessionDescription) *session = + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithNativeDescription:description.get()]; + completion_handler_(session, nil); + completion_handler_ = nil; + } + + void OnFailure(RTCError error) override { + RTC_DCHECK(completion_handler_); + // TODO(hta): Add handling of error.type() + NSString *str = [NSString stringForStdString:error.message()]; + NSError* err = + [NSError errorWithDomain:kRTCPeerConnectionErrorDomain + code:kRTCPeerConnnectionSessionDescriptionError + userInfo:@{ NSLocalizedDescriptionKey : str }]; + completion_handler_(nil, err); + completion_handler_ = nil; + } + + private: + void (^completion_handler_)(RTC_OBJC_TYPE(RTCSessionDescription) * sessionDescription, + NSError *error); +}; + +PeerConnectionDelegateAdapter::PeerConnectionDelegateAdapter(RTC_OBJC_TYPE(RTCPeerConnection) * + peerConnection) { + peer_connection_ = peerConnection; +} + +PeerConnectionDelegateAdapter::~PeerConnectionDelegateAdapter() { + peer_connection_ = nil; +} + +void PeerConnectionDelegateAdapter::OnSignalingChange( + PeerConnectionInterface::SignalingState new_state) { + RTCSignalingState state = + [[RTC_OBJC_TYPE(RTCPeerConnection) class] signalingStateForNativeState:new_state]; + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + [peer_connection.delegate peerConnection:peer_connection + didChangeSignalingState:state]; +} + +void PeerConnectionDelegateAdapter::OnAddStream( + rtc::scoped_refptr<MediaStreamInterface> stream) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + RTC_OBJC_TYPE(RTCMediaStream) *mediaStream = + [[RTC_OBJC_TYPE(RTCMediaStream) alloc] initWithFactory:peer_connection.factory + nativeMediaStream:stream]; + [peer_connection.delegate peerConnection:peer_connection + didAddStream:mediaStream]; +} + +void PeerConnectionDelegateAdapter::OnRemoveStream( + rtc::scoped_refptr<MediaStreamInterface> stream) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + RTC_OBJC_TYPE(RTCMediaStream) *mediaStream = + [[RTC_OBJC_TYPE(RTCMediaStream) alloc] initWithFactory:peer_connection.factory + nativeMediaStream:stream]; + + [peer_connection.delegate peerConnection:peer_connection + didRemoveStream:mediaStream]; +} + +void PeerConnectionDelegateAdapter::OnTrack( + rtc::scoped_refptr<RtpTransceiverInterface> nativeTransceiver) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + RTC_OBJC_TYPE(RTCRtpTransceiver) *transceiver = + [[RTC_OBJC_TYPE(RTCRtpTransceiver) alloc] initWithFactory:peer_connection.factory + nativeRtpTransceiver:nativeTransceiver]; + if ([peer_connection.delegate + respondsToSelector:@selector(peerConnection:didStartReceivingOnTransceiver:)]) { + [peer_connection.delegate peerConnection:peer_connection + didStartReceivingOnTransceiver:transceiver]; + } +} + +void PeerConnectionDelegateAdapter::OnDataChannel( + rtc::scoped_refptr<DataChannelInterface> data_channel) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + RTC_OBJC_TYPE(RTCDataChannel) *dataChannel = + [[RTC_OBJC_TYPE(RTCDataChannel) alloc] initWithFactory:peer_connection.factory + nativeDataChannel:data_channel]; + [peer_connection.delegate peerConnection:peer_connection + didOpenDataChannel:dataChannel]; +} + +void PeerConnectionDelegateAdapter::OnRenegotiationNeeded() { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + [peer_connection.delegate peerConnectionShouldNegotiate:peer_connection]; +} + +void PeerConnectionDelegateAdapter::OnIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) { + RTCIceConnectionState state = + [RTC_OBJC_TYPE(RTCPeerConnection) iceConnectionStateForNativeState:new_state]; + [peer_connection_.delegate peerConnection:peer_connection_ didChangeIceConnectionState:state]; +} + +void PeerConnectionDelegateAdapter::OnStandardizedIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) { + if ([peer_connection_.delegate + respondsToSelector:@selector(peerConnection:didChangeStandardizedIceConnectionState:)]) { + RTCIceConnectionState state = + [RTC_OBJC_TYPE(RTCPeerConnection) iceConnectionStateForNativeState:new_state]; + [peer_connection_.delegate peerConnection:peer_connection_ + didChangeStandardizedIceConnectionState:state]; + } +} + +void PeerConnectionDelegateAdapter::OnConnectionChange( + PeerConnectionInterface::PeerConnectionState new_state) { + if ([peer_connection_.delegate + respondsToSelector:@selector(peerConnection:didChangeConnectionState:)]) { + RTCPeerConnectionState state = + [RTC_OBJC_TYPE(RTCPeerConnection) connectionStateForNativeState:new_state]; + [peer_connection_.delegate peerConnection:peer_connection_ didChangeConnectionState:state]; + } +} + +void PeerConnectionDelegateAdapter::OnIceGatheringChange( + PeerConnectionInterface::IceGatheringState new_state) { + RTCIceGatheringState state = + [[RTC_OBJC_TYPE(RTCPeerConnection) class] iceGatheringStateForNativeState:new_state]; + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + [peer_connection.delegate peerConnection:peer_connection + didChangeIceGatheringState:state]; +} + +void PeerConnectionDelegateAdapter::OnIceCandidate( + const IceCandidateInterface *candidate) { + RTC_OBJC_TYPE(RTCIceCandidate) *iceCandidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithNativeCandidate:candidate]; + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + [peer_connection.delegate peerConnection:peer_connection + didGenerateIceCandidate:iceCandidate]; +} + +void PeerConnectionDelegateAdapter::OnIceCandidateError(const std::string &address, + int port, + const std::string &url, + int error_code, + const std::string &error_text) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + RTC_OBJC_TYPE(RTCIceCandidateErrorEvent) *event = + [[RTC_OBJC_TYPE(RTCIceCandidateErrorEvent) alloc] initWithAddress:address + port:port + url:url + errorCode:error_code + errorText:error_text]; + if ([peer_connection.delegate respondsToSelector:@selector(peerConnection: + didFailToGatherIceCandidate:)]) { + [peer_connection.delegate peerConnection:peer_connection didFailToGatherIceCandidate:event]; + } +} + +void PeerConnectionDelegateAdapter::OnIceCandidatesRemoved( + const std::vector<cricket::Candidate>& candidates) { + NSMutableArray* ice_candidates = + [NSMutableArray arrayWithCapacity:candidates.size()]; + for (const auto& candidate : candidates) { + JsepIceCandidate candidate_wrapper(candidate.transport_name(), -1, candidate); + RTC_OBJC_TYPE(RTCIceCandidate) *ice_candidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithNativeCandidate:&candidate_wrapper]; + [ice_candidates addObject:ice_candidate]; + } + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + [peer_connection.delegate peerConnection:peer_connection + didRemoveIceCandidates:ice_candidates]; +} + +void PeerConnectionDelegateAdapter::OnIceSelectedCandidatePairChanged( + const cricket::CandidatePairChangeEvent &event) { + const auto &selected_pair = event.selected_candidate_pair; + JsepIceCandidate local_candidate_wrapper( + selected_pair.local_candidate().transport_name(), -1, selected_pair.local_candidate()); + RTC_OBJC_TYPE(RTCIceCandidate) *local_candidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithNativeCandidate:&local_candidate_wrapper]; + JsepIceCandidate remote_candidate_wrapper( + selected_pair.remote_candidate().transport_name(), -1, selected_pair.remote_candidate()); + RTC_OBJC_TYPE(RTCIceCandidate) *remote_candidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithNativeCandidate:&remote_candidate_wrapper]; + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + NSString *nsstr_reason = [NSString stringForStdString:event.reason]; + if ([peer_connection.delegate + respondsToSelector:@selector + (peerConnection:didChangeLocalCandidate:remoteCandidate:lastReceivedMs:changeReason:)]) { + [peer_connection.delegate peerConnection:peer_connection + didChangeLocalCandidate:local_candidate + remoteCandidate:remote_candidate + lastReceivedMs:event.last_data_received_ms + changeReason:nsstr_reason]; + } +} + +void PeerConnectionDelegateAdapter::OnAddTrack( + rtc::scoped_refptr<RtpReceiverInterface> receiver, + const std::vector<rtc::scoped_refptr<MediaStreamInterface>> &streams) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + if ([peer_connection.delegate respondsToSelector:@selector(peerConnection: + didAddReceiver:streams:)]) { + NSMutableArray *mediaStreams = [NSMutableArray arrayWithCapacity:streams.size()]; + for (const auto &nativeStream : streams) { + RTC_OBJC_TYPE(RTCMediaStream) *mediaStream = + [[RTC_OBJC_TYPE(RTCMediaStream) alloc] initWithFactory:peer_connection.factory + nativeMediaStream:nativeStream]; + [mediaStreams addObject:mediaStream]; + } + RTC_OBJC_TYPE(RTCRtpReceiver) *rtpReceiver = + [[RTC_OBJC_TYPE(RTCRtpReceiver) alloc] initWithFactory:peer_connection.factory + nativeRtpReceiver:receiver]; + + [peer_connection.delegate peerConnection:peer_connection + didAddReceiver:rtpReceiver + streams:mediaStreams]; + } +} + +void PeerConnectionDelegateAdapter::OnRemoveTrack( + rtc::scoped_refptr<RtpReceiverInterface> receiver) { + RTC_OBJC_TYPE(RTCPeerConnection) *peer_connection = peer_connection_; + if ([peer_connection.delegate respondsToSelector:@selector(peerConnection:didRemoveReceiver:)]) { + RTC_OBJC_TYPE(RTCRtpReceiver) *rtpReceiver = + [[RTC_OBJC_TYPE(RTCRtpReceiver) alloc] initWithFactory:peer_connection.factory + nativeRtpReceiver:receiver]; + [peer_connection.delegate peerConnection:peer_connection didRemoveReceiver:rtpReceiver]; + } +} + +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCPeerConnection) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + NSMutableArray<RTC_OBJC_TYPE(RTCMediaStream) *> *_localStreams; + std::unique_ptr<webrtc::PeerConnectionDelegateAdapter> _observer; + rtc::scoped_refptr<webrtc::PeerConnectionInterface> _peerConnection; + std::unique_ptr<webrtc::MediaConstraints> _nativeConstraints; + BOOL _hasStartedRtcEventLog; +} + +@synthesize delegate = _delegate; +@synthesize factory = _factory; + +- (nullable instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + configuration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + certificateVerifier: + (nullable id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)>)certificateVerifier + delegate:(id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate { + NSParameterAssert(factory); + std::unique_ptr<webrtc::PeerConnectionDependencies> dependencies = + std::make_unique<webrtc::PeerConnectionDependencies>(nullptr); + if (certificateVerifier != nil) { + dependencies->tls_cert_verifier = webrtc::ObjCToNativeCertificateVerifier(certificateVerifier); + } + return [self initWithDependencies:factory + configuration:configuration + constraints:constraints + dependencies:std::move(dependencies) + delegate:delegate]; +} + +- (nullable instancetype) + initWithDependencies:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + configuration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + dependencies:(std::unique_ptr<webrtc::PeerConnectionDependencies>)dependencies + delegate:(id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate { + NSParameterAssert(factory); + NSParameterAssert(dependencies.get()); + std::unique_ptr<webrtc::PeerConnectionInterface::RTCConfiguration> config( + [configuration createNativeConfiguration]); + if (!config) { + return nil; + } + if (self = [super init]) { + _observer.reset(new webrtc::PeerConnectionDelegateAdapter(self)); + _nativeConstraints = constraints.nativeConstraints; + CopyConstraintsIntoRtcConfiguration(_nativeConstraints.get(), config.get()); + + webrtc::PeerConnectionDependencies deps = std::move(*dependencies); + deps.observer = _observer.get(); + auto result = factory.nativeFactory->CreatePeerConnectionOrError(*config, std::move(deps)); + + if (!result.ok()) { + return nil; + } + _peerConnection = result.MoveValue(); + _factory = factory; + _localStreams = [[NSMutableArray alloc] init]; + _delegate = delegate; + } + return self; +} + +- (NSArray<RTC_OBJC_TYPE(RTCMediaStream) *> *)localStreams { + return [_localStreams copy]; +} + +- (RTC_OBJC_TYPE(RTCSessionDescription) *)localDescription { + // It's only safe to operate on SessionDescriptionInterface on the signaling thread. + return _peerConnection->signaling_thread()->BlockingCall([self] { + const webrtc::SessionDescriptionInterface *description = _peerConnection->local_description(); + return description ? + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithNativeDescription:description] : + nil; + }); +} + +- (RTC_OBJC_TYPE(RTCSessionDescription) *)remoteDescription { + // It's only safe to operate on SessionDescriptionInterface on the signaling thread. + return _peerConnection->signaling_thread()->BlockingCall([self] { + const webrtc::SessionDescriptionInterface *description = _peerConnection->remote_description(); + return description ? + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithNativeDescription:description] : + nil; + }); +} + +- (RTCSignalingState)signalingState { + return [[self class] + signalingStateForNativeState:_peerConnection->signaling_state()]; +} + +- (RTCIceConnectionState)iceConnectionState { + return [[self class] iceConnectionStateForNativeState: + _peerConnection->ice_connection_state()]; +} + +- (RTCPeerConnectionState)connectionState { + return [[self class] connectionStateForNativeState:_peerConnection->peer_connection_state()]; +} + +- (RTCIceGatheringState)iceGatheringState { + return [[self class] iceGatheringStateForNativeState: + _peerConnection->ice_gathering_state()]; +} + +- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration { + std::unique_ptr<webrtc::PeerConnectionInterface::RTCConfiguration> config( + [configuration createNativeConfiguration]); + if (!config) { + return NO; + } + CopyConstraintsIntoRtcConfiguration(_nativeConstraints.get(), + config.get()); + return _peerConnection->SetConfiguration(*config).ok(); +} + +- (RTC_OBJC_TYPE(RTCConfiguration) *)configuration { + webrtc::PeerConnectionInterface::RTCConfiguration config = + _peerConnection->GetConfiguration(); + return [[RTC_OBJC_TYPE(RTCConfiguration) alloc] initWithNativeConfiguration:config]; +} + +- (void)close { + _peerConnection->Close(); +} + +- (void)addIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)candidate { + std::unique_ptr<const webrtc::IceCandidateInterface> iceCandidate( + candidate.nativeCandidate); + _peerConnection->AddIceCandidate(iceCandidate.get()); +} +- (void)addIceCandidate:(RTC_OBJC_TYPE(RTCIceCandidate) *)candidate + completionHandler:(void (^)(NSError *_Nullable error))completionHandler { + RTC_DCHECK(completionHandler != nil); + _peerConnection->AddIceCandidate( + candidate.nativeCandidate, [completionHandler](const auto &error) { + if (error.ok()) { + completionHandler(nil); + } else { + NSString *str = [NSString stringForStdString:error.message()]; + NSError *err = [NSError errorWithDomain:kRTCPeerConnectionErrorDomain + code:static_cast<NSInteger>(error.type()) + userInfo:@{NSLocalizedDescriptionKey : str}]; + completionHandler(err); + } + }); +} +- (void)removeIceCandidates:(NSArray<RTC_OBJC_TYPE(RTCIceCandidate) *> *)iceCandidates { + std::vector<cricket::Candidate> candidates; + for (RTC_OBJC_TYPE(RTCIceCandidate) * iceCandidate in iceCandidates) { + std::unique_ptr<const webrtc::IceCandidateInterface> candidate( + iceCandidate.nativeCandidate); + if (candidate) { + candidates.push_back(candidate->candidate()); + // Need to fill the transport name from the sdp_mid. + candidates.back().set_transport_name(candidate->sdp_mid()); + } + } + if (!candidates.empty()) { + _peerConnection->RemoveIceCandidates(candidates); + } +} + +- (void)addStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream { + if (!_peerConnection->AddStream(stream.nativeMediaStream.get())) { + RTCLogError(@"Failed to add stream: %@", stream); + return; + } + [_localStreams addObject:stream]; +} + +- (void)removeStream:(RTC_OBJC_TYPE(RTCMediaStream) *)stream { + _peerConnection->RemoveStream(stream.nativeMediaStream.get()); + [_localStreams removeObject:stream]; +} + +- (nullable RTC_OBJC_TYPE(RTCRtpSender) *)addTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track + streamIds:(NSArray<NSString *> *)streamIds { + std::vector<std::string> nativeStreamIds; + for (NSString *streamId in streamIds) { + nativeStreamIds.push_back([streamId UTF8String]); + } + webrtc::RTCErrorOr<rtc::scoped_refptr<webrtc::RtpSenderInterface>> nativeSenderOrError = + _peerConnection->AddTrack(track.nativeTrack, nativeStreamIds); + if (!nativeSenderOrError.ok()) { + RTCLogError(@"Failed to add track %@: %s", track, nativeSenderOrError.error().message()); + return nil; + } + return [[RTC_OBJC_TYPE(RTCRtpSender) alloc] initWithFactory:self.factory + nativeRtpSender:nativeSenderOrError.MoveValue()]; +} + +- (BOOL)removeTrack:(RTC_OBJC_TYPE(RTCRtpSender) *)sender { + bool result = _peerConnection->RemoveTrackOrError(sender.nativeRtpSender).ok(); + if (!result) { + RTCLogError(@"Failed to remote track %@", sender); + } + return result; +} + +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *)addTransceiverWithTrack: + (RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track { + return [self addTransceiverWithTrack:track + init:[[RTC_OBJC_TYPE(RTCRtpTransceiverInit) alloc] init]]; +} + +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *) + addTransceiverWithTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track + init:(RTC_OBJC_TYPE(RTCRtpTransceiverInit) *)init { + webrtc::RTCErrorOr<rtc::scoped_refptr<webrtc::RtpTransceiverInterface>> nativeTransceiverOrError = + _peerConnection->AddTransceiver(track.nativeTrack, init.nativeInit); + if (!nativeTransceiverOrError.ok()) { + RTCLogError( + @"Failed to add transceiver %@: %s", track, nativeTransceiverOrError.error().message()); + return nil; + } + return [[RTC_OBJC_TYPE(RTCRtpTransceiver) alloc] + initWithFactory:self.factory + nativeRtpTransceiver:nativeTransceiverOrError.MoveValue()]; +} + +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *)addTransceiverOfType:(RTCRtpMediaType)mediaType { + return [self addTransceiverOfType:mediaType + init:[[RTC_OBJC_TYPE(RTCRtpTransceiverInit) alloc] init]]; +} + +- (nullable RTC_OBJC_TYPE(RTCRtpTransceiver) *) + addTransceiverOfType:(RTCRtpMediaType)mediaType + init:(RTC_OBJC_TYPE(RTCRtpTransceiverInit) *)init { + webrtc::RTCErrorOr<rtc::scoped_refptr<webrtc::RtpTransceiverInterface>> nativeTransceiverOrError = + _peerConnection->AddTransceiver( + [RTC_OBJC_TYPE(RTCRtpReceiver) nativeMediaTypeForMediaType:mediaType], init.nativeInit); + if (!nativeTransceiverOrError.ok()) { + RTCLogError(@"Failed to add transceiver %@: %s", + [RTC_OBJC_TYPE(RTCRtpReceiver) stringForMediaType:mediaType], + nativeTransceiverOrError.error().message()); + return nil; + } + return [[RTC_OBJC_TYPE(RTCRtpTransceiver) alloc] + initWithFactory:self.factory + nativeRtpTransceiver:nativeTransceiverOrError.MoveValue()]; +} + +- (void)restartIce { + _peerConnection->RestartIce(); +} + +- (void)offerForConstraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + completionHandler:(RTCCreateSessionDescriptionCompletionHandler)completionHandler { + RTC_DCHECK(completionHandler != nil); + rtc::scoped_refptr<webrtc::CreateSessionDescriptionObserverAdapter> observer = + rtc::make_ref_counted<webrtc::CreateSessionDescriptionObserverAdapter>(completionHandler); + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; + CopyConstraintsIntoOfferAnswerOptions(constraints.nativeConstraints.get(), &options); + + _peerConnection->CreateOffer(observer.get(), options); +} + +- (void)answerForConstraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + completionHandler:(RTCCreateSessionDescriptionCompletionHandler)completionHandler { + RTC_DCHECK(completionHandler != nil); + rtc::scoped_refptr<webrtc::CreateSessionDescriptionObserverAdapter> observer = + rtc::make_ref_counted<webrtc::CreateSessionDescriptionObserverAdapter>(completionHandler); + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; + CopyConstraintsIntoOfferAnswerOptions(constraints.nativeConstraints.get(), &options); + + _peerConnection->CreateAnswer(observer.get(), options); +} + +- (void)setLocalDescription:(RTC_OBJC_TYPE(RTCSessionDescription) *)sdp + completionHandler:(RTCSetSessionDescriptionCompletionHandler)completionHandler { + RTC_DCHECK(completionHandler != nil); + rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface> observer = + rtc::make_ref_counted<::SetSessionDescriptionObserver>(completionHandler); + _peerConnection->SetLocalDescription(sdp.nativeDescription, observer); +} + +- (void)setLocalDescriptionWithCompletionHandler: + (RTCSetSessionDescriptionCompletionHandler)completionHandler { + RTC_DCHECK(completionHandler != nil); + rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface> observer = + rtc::make_ref_counted<::SetSessionDescriptionObserver>(completionHandler); + _peerConnection->SetLocalDescription(observer); +} + +- (void)setRemoteDescription:(RTC_OBJC_TYPE(RTCSessionDescription) *)sdp + completionHandler:(RTCSetSessionDescriptionCompletionHandler)completionHandler { + RTC_DCHECK(completionHandler != nil); + rtc::scoped_refptr<webrtc::SetRemoteDescriptionObserverInterface> observer = + rtc::make_ref_counted<::SetSessionDescriptionObserver>(completionHandler); + _peerConnection->SetRemoteDescription(sdp.nativeDescription, observer); +} + +- (BOOL)setBweMinBitrateBps:(nullable NSNumber *)minBitrateBps + currentBitrateBps:(nullable NSNumber *)currentBitrateBps + maxBitrateBps:(nullable NSNumber *)maxBitrateBps { + webrtc::BitrateSettings params; + if (minBitrateBps != nil) { + params.min_bitrate_bps = absl::optional<int>(minBitrateBps.intValue); + } + if (currentBitrateBps != nil) { + params.start_bitrate_bps = absl::optional<int>(currentBitrateBps.intValue); + } + if (maxBitrateBps != nil) { + params.max_bitrate_bps = absl::optional<int>(maxBitrateBps.intValue); + } + return _peerConnection->SetBitrate(params).ok(); +} + +- (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath + maxSizeInBytes:(int64_t)maxSizeInBytes { + RTC_DCHECK(filePath.length); + RTC_DCHECK_GT(maxSizeInBytes, 0); + RTC_DCHECK(!_hasStartedRtcEventLog); + if (_hasStartedRtcEventLog) { + RTCLogError(@"Event logging already started."); + return NO; + } + FILE *f = fopen(filePath.UTF8String, "wb"); + if (!f) { + RTCLogError(@"Error opening file: %@. Error: %d", filePath, errno); + return NO; + } + // TODO(eladalon): It would be better to not allow negative values into PC. + const size_t max_size = (maxSizeInBytes < 0) ? webrtc::RtcEventLog::kUnlimitedOutput : + rtc::saturated_cast<size_t>(maxSizeInBytes); + + _hasStartedRtcEventLog = _peerConnection->StartRtcEventLog( + std::make_unique<webrtc::RtcEventLogOutputFile>(f, max_size)); + return _hasStartedRtcEventLog; +} + +- (void)stopRtcEventLog { + _peerConnection->StopRtcEventLog(); + _hasStartedRtcEventLog = NO; +} + +- (RTC_OBJC_TYPE(RTCRtpSender) *)senderWithKind:(NSString *)kind streamId:(NSString *)streamId { + std::string nativeKind = [NSString stdStringForString:kind]; + std::string nativeStreamId = [NSString stdStringForString:streamId]; + rtc::scoped_refptr<webrtc::RtpSenderInterface> nativeSender( + _peerConnection->CreateSender(nativeKind, nativeStreamId)); + return nativeSender ? [[RTC_OBJC_TYPE(RTCRtpSender) alloc] initWithFactory:self.factory + nativeRtpSender:nativeSender] : + nil; +} + +- (NSArray<RTC_OBJC_TYPE(RTCRtpSender) *> *)senders { + std::vector<rtc::scoped_refptr<webrtc::RtpSenderInterface>> nativeSenders( + _peerConnection->GetSenders()); + NSMutableArray *senders = [[NSMutableArray alloc] init]; + for (const auto &nativeSender : nativeSenders) { + RTC_OBJC_TYPE(RTCRtpSender) *sender = + [[RTC_OBJC_TYPE(RTCRtpSender) alloc] initWithFactory:self.factory + nativeRtpSender:nativeSender]; + [senders addObject:sender]; + } + return senders; +} + +- (NSArray<RTC_OBJC_TYPE(RTCRtpReceiver) *> *)receivers { + std::vector<rtc::scoped_refptr<webrtc::RtpReceiverInterface>> nativeReceivers( + _peerConnection->GetReceivers()); + NSMutableArray *receivers = [[NSMutableArray alloc] init]; + for (const auto &nativeReceiver : nativeReceivers) { + RTC_OBJC_TYPE(RTCRtpReceiver) *receiver = + [[RTC_OBJC_TYPE(RTCRtpReceiver) alloc] initWithFactory:self.factory + nativeRtpReceiver:nativeReceiver]; + [receivers addObject:receiver]; + } + return receivers; +} + +- (NSArray<RTC_OBJC_TYPE(RTCRtpTransceiver) *> *)transceivers { + std::vector<rtc::scoped_refptr<webrtc::RtpTransceiverInterface>> nativeTransceivers( + _peerConnection->GetTransceivers()); + NSMutableArray *transceivers = [[NSMutableArray alloc] init]; + for (const auto &nativeTransceiver : nativeTransceivers) { + RTC_OBJC_TYPE(RTCRtpTransceiver) *transceiver = + [[RTC_OBJC_TYPE(RTCRtpTransceiver) alloc] initWithFactory:self.factory + nativeRtpTransceiver:nativeTransceiver]; + [transceivers addObject:transceiver]; + } + return transceivers; +} + +#pragma mark - Private + ++ (webrtc::PeerConnectionInterface::SignalingState)nativeSignalingStateForState: + (RTCSignalingState)state { + switch (state) { + case RTCSignalingStateStable: + return webrtc::PeerConnectionInterface::kStable; + case RTCSignalingStateHaveLocalOffer: + return webrtc::PeerConnectionInterface::kHaveLocalOffer; + case RTCSignalingStateHaveLocalPrAnswer: + return webrtc::PeerConnectionInterface::kHaveLocalPrAnswer; + case RTCSignalingStateHaveRemoteOffer: + return webrtc::PeerConnectionInterface::kHaveRemoteOffer; + case RTCSignalingStateHaveRemotePrAnswer: + return webrtc::PeerConnectionInterface::kHaveRemotePrAnswer; + case RTCSignalingStateClosed: + return webrtc::PeerConnectionInterface::kClosed; + } +} + ++ (RTCSignalingState)signalingStateForNativeState: + (webrtc::PeerConnectionInterface::SignalingState)nativeState { + switch (nativeState) { + case webrtc::PeerConnectionInterface::kStable: + return RTCSignalingStateStable; + case webrtc::PeerConnectionInterface::kHaveLocalOffer: + return RTCSignalingStateHaveLocalOffer; + case webrtc::PeerConnectionInterface::kHaveLocalPrAnswer: + return RTCSignalingStateHaveLocalPrAnswer; + case webrtc::PeerConnectionInterface::kHaveRemoteOffer: + return RTCSignalingStateHaveRemoteOffer; + case webrtc::PeerConnectionInterface::kHaveRemotePrAnswer: + return RTCSignalingStateHaveRemotePrAnswer; + case webrtc::PeerConnectionInterface::kClosed: + return RTCSignalingStateClosed; + } +} + ++ (NSString *)stringForSignalingState:(RTCSignalingState)state { + switch (state) { + case RTCSignalingStateStable: + return @"STABLE"; + case RTCSignalingStateHaveLocalOffer: + return @"HAVE_LOCAL_OFFER"; + case RTCSignalingStateHaveLocalPrAnswer: + return @"HAVE_LOCAL_PRANSWER"; + case RTCSignalingStateHaveRemoteOffer: + return @"HAVE_REMOTE_OFFER"; + case RTCSignalingStateHaveRemotePrAnswer: + return @"HAVE_REMOTE_PRANSWER"; + case RTCSignalingStateClosed: + return @"CLOSED"; + } +} + ++ (webrtc::PeerConnectionInterface::PeerConnectionState)nativeConnectionStateForState: + (RTCPeerConnectionState)state { + switch (state) { + case RTCPeerConnectionStateNew: + return webrtc::PeerConnectionInterface::PeerConnectionState::kNew; + case RTCPeerConnectionStateConnecting: + return webrtc::PeerConnectionInterface::PeerConnectionState::kConnecting; + case RTCPeerConnectionStateConnected: + return webrtc::PeerConnectionInterface::PeerConnectionState::kConnected; + case RTCPeerConnectionStateFailed: + return webrtc::PeerConnectionInterface::PeerConnectionState::kFailed; + case RTCPeerConnectionStateDisconnected: + return webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected; + case RTCPeerConnectionStateClosed: + return webrtc::PeerConnectionInterface::PeerConnectionState::kClosed; + } +} + ++ (RTCPeerConnectionState)connectionStateForNativeState: + (webrtc::PeerConnectionInterface::PeerConnectionState)nativeState { + switch (nativeState) { + case webrtc::PeerConnectionInterface::PeerConnectionState::kNew: + return RTCPeerConnectionStateNew; + case webrtc::PeerConnectionInterface::PeerConnectionState::kConnecting: + return RTCPeerConnectionStateConnecting; + case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected: + return RTCPeerConnectionStateConnected; + case webrtc::PeerConnectionInterface::PeerConnectionState::kFailed: + return RTCPeerConnectionStateFailed; + case webrtc::PeerConnectionInterface::PeerConnectionState::kDisconnected: + return RTCPeerConnectionStateDisconnected; + case webrtc::PeerConnectionInterface::PeerConnectionState::kClosed: + return RTCPeerConnectionStateClosed; + } +} + ++ (NSString *)stringForConnectionState:(RTCPeerConnectionState)state { + switch (state) { + case RTCPeerConnectionStateNew: + return @"NEW"; + case RTCPeerConnectionStateConnecting: + return @"CONNECTING"; + case RTCPeerConnectionStateConnected: + return @"CONNECTED"; + case RTCPeerConnectionStateFailed: + return @"FAILED"; + case RTCPeerConnectionStateDisconnected: + return @"DISCONNECTED"; + case RTCPeerConnectionStateClosed: + return @"CLOSED"; + } +} + ++ (webrtc::PeerConnectionInterface::IceConnectionState) + nativeIceConnectionStateForState:(RTCIceConnectionState)state { + switch (state) { + case RTCIceConnectionStateNew: + return webrtc::PeerConnectionInterface::kIceConnectionNew; + case RTCIceConnectionStateChecking: + return webrtc::PeerConnectionInterface::kIceConnectionChecking; + case RTCIceConnectionStateConnected: + return webrtc::PeerConnectionInterface::kIceConnectionConnected; + case RTCIceConnectionStateCompleted: + return webrtc::PeerConnectionInterface::kIceConnectionCompleted; + case RTCIceConnectionStateFailed: + return webrtc::PeerConnectionInterface::kIceConnectionFailed; + case RTCIceConnectionStateDisconnected: + return webrtc::PeerConnectionInterface::kIceConnectionDisconnected; + case RTCIceConnectionStateClosed: + return webrtc::PeerConnectionInterface::kIceConnectionClosed; + case RTCIceConnectionStateCount: + return webrtc::PeerConnectionInterface::kIceConnectionMax; + } +} + ++ (RTCIceConnectionState)iceConnectionStateForNativeState: + (webrtc::PeerConnectionInterface::IceConnectionState)nativeState { + switch (nativeState) { + case webrtc::PeerConnectionInterface::kIceConnectionNew: + return RTCIceConnectionStateNew; + case webrtc::PeerConnectionInterface::kIceConnectionChecking: + return RTCIceConnectionStateChecking; + case webrtc::PeerConnectionInterface::kIceConnectionConnected: + return RTCIceConnectionStateConnected; + case webrtc::PeerConnectionInterface::kIceConnectionCompleted: + return RTCIceConnectionStateCompleted; + case webrtc::PeerConnectionInterface::kIceConnectionFailed: + return RTCIceConnectionStateFailed; + case webrtc::PeerConnectionInterface::kIceConnectionDisconnected: + return RTCIceConnectionStateDisconnected; + case webrtc::PeerConnectionInterface::kIceConnectionClosed: + return RTCIceConnectionStateClosed; + case webrtc::PeerConnectionInterface::kIceConnectionMax: + return RTCIceConnectionStateCount; + } +} + ++ (NSString *)stringForIceConnectionState:(RTCIceConnectionState)state { + switch (state) { + case RTCIceConnectionStateNew: + return @"NEW"; + case RTCIceConnectionStateChecking: + return @"CHECKING"; + case RTCIceConnectionStateConnected: + return @"CONNECTED"; + case RTCIceConnectionStateCompleted: + return @"COMPLETED"; + case RTCIceConnectionStateFailed: + return @"FAILED"; + case RTCIceConnectionStateDisconnected: + return @"DISCONNECTED"; + case RTCIceConnectionStateClosed: + return @"CLOSED"; + case RTCIceConnectionStateCount: + return @"COUNT"; + } +} + ++ (webrtc::PeerConnectionInterface::IceGatheringState) + nativeIceGatheringStateForState:(RTCIceGatheringState)state { + switch (state) { + case RTCIceGatheringStateNew: + return webrtc::PeerConnectionInterface::kIceGatheringNew; + case RTCIceGatheringStateGathering: + return webrtc::PeerConnectionInterface::kIceGatheringGathering; + case RTCIceGatheringStateComplete: + return webrtc::PeerConnectionInterface::kIceGatheringComplete; + } +} + ++ (RTCIceGatheringState)iceGatheringStateForNativeState: + (webrtc::PeerConnectionInterface::IceGatheringState)nativeState { + switch (nativeState) { + case webrtc::PeerConnectionInterface::kIceGatheringNew: + return RTCIceGatheringStateNew; + case webrtc::PeerConnectionInterface::kIceGatheringGathering: + return RTCIceGatheringStateGathering; + case webrtc::PeerConnectionInterface::kIceGatheringComplete: + return RTCIceGatheringStateComplete; + } +} + ++ (NSString *)stringForIceGatheringState:(RTCIceGatheringState)state { + switch (state) { + case RTCIceGatheringStateNew: + return @"NEW"; + case RTCIceGatheringStateGathering: + return @"GATHERING"; + case RTCIceGatheringStateComplete: + return @"COMPLETE"; + } +} + ++ (webrtc::PeerConnectionInterface::StatsOutputLevel) + nativeStatsOutputLevelForLevel:(RTCStatsOutputLevel)level { + switch (level) { + case RTCStatsOutputLevelStandard: + return webrtc::PeerConnectionInterface::kStatsOutputLevelStandard; + case RTCStatsOutputLevelDebug: + return webrtc::PeerConnectionInterface::kStatsOutputLevelDebug; + } +} + +- (rtc::scoped_refptr<webrtc::PeerConnectionInterface>)nativePeerConnection { + return _peerConnection; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h new file mode 100644 index 0000000000..f361b9f0ea --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -0,0 +1,85 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactory.h" + +#include "api/scoped_refptr.h" + +namespace webrtc { + +class AudioDeviceModule; +class AudioEncoderFactory; +class AudioDecoderFactory; +class NetworkControllerFactoryInterface; +class VideoEncoderFactory; +class VideoDecoderFactory; +class AudioProcessing; +struct PeerConnectionDependencies; + +} // namespace webrtc + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class extension exposes methods that work directly with injectable C++ components. + */ +@interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) +() + + - (instancetype)initNative NS_DESIGNATED_INITIALIZER; + +/* Initializer used when WebRTC is compiled with no media support */ +- (instancetype)initWithNoMedia; + +/* Initialize object with injectable native audio/video encoder/decoder factories */ +- (instancetype)initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory + audioDeviceModule: + (nullable webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule; + +- (instancetype) + initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory + audioDeviceModule:(nullable webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule + networkControllerFactory:(std::unique_ptr<webrtc::NetworkControllerFactoryInterface>) + networkControllerFactory; + +- (instancetype) + initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory + decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory; + +/** Initialize an RTCPeerConnection with a configuration, constraints, and + * dependencies. + */ +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithDependencies:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + dependencies:(std::unique_ptr<webrtc::PeerConnectionDependencies>)dependencies + delegate:(nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Private.h new file mode 100644 index 0000000000..822585ddc2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Private.h @@ -0,0 +1,36 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactory.h" + +#include "api/peer_connection_interface.h" +#include "api/scoped_refptr.h" +#include "rtc_base/thread.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) +() + + /** + * PeerConnectionFactoryInterface created and held by this + * RTCPeerConnectionFactory object. This is needed to pass to the underlying + * C++ APIs. + */ + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> nativeFactory; + +@property(nonatomic, readonly) rtc::Thread* signalingThread; +@property(nonatomic, readonly) rtc::Thread* workerThread; +@property(nonatomic, readonly) rtc::Thread* networkThread; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h new file mode 100644 index 0000000000..5575af98c9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -0,0 +1,113 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioSource); +@class RTC_OBJC_TYPE(RTCAudioTrack); +@class RTC_OBJC_TYPE(RTCConfiguration); +@class RTC_OBJC_TYPE(RTCMediaConstraints); +@class RTC_OBJC_TYPE(RTCMediaStream); +@class RTC_OBJC_TYPE(RTCPeerConnection); +@class RTC_OBJC_TYPE(RTCVideoSource); +@class RTC_OBJC_TYPE(RTCVideoTrack); +@class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions); +@protocol RTC_OBJC_TYPE +(RTCPeerConnectionDelegate); +@protocol RTC_OBJC_TYPE +(RTCVideoDecoderFactory); +@protocol RTC_OBJC_TYPE +(RTCVideoEncoderFactory); +@protocol RTC_OBJC_TYPE +(RTCSSLCertificateVerifier); +@protocol RTC_OBJC_TYPE +(RTCAudioDevice); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject + +/* Initialize object with default H264 video encoder/decoder factories and default ADM */ +- (instancetype)init; + +/* Initialize object with injectable video encoder/decoder factories and default ADM */ +- (instancetype) + initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory + decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory; + +/* Initialize object with injectable video encoder/decoder factories and injectable ADM */ +- (instancetype) + initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory + decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory + audioDevice:(nullable id<RTC_OBJC_TYPE(RTCAudioDevice)>)audioDevice; + +/** Initialize an RTCAudioSource with constraints. */ +- (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints: + (nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints; + +/** Initialize an RTCAudioTrack with an id. Convenience ctor to use an audio source + * with no constraints. + */ +- (RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrackWithTrackId:(NSString *)trackId; + +/** Initialize an RTCAudioTrack with a source and an id. */ +- (RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrackWithSource:(RTC_OBJC_TYPE(RTCAudioSource) *)source + trackId:(NSString *)trackId; + +/** Initialize a generic RTCVideoSource. The RTCVideoSource should be + * passed to a RTCVideoCapturer implementation, e.g. + * RTCCameraVideoCapturer, in order to produce frames. + */ +- (RTC_OBJC_TYPE(RTCVideoSource) *)videoSource; + +/** Initialize a generic RTCVideoSource with he posibility of marking + * it as usable for screen sharing. The RTCVideoSource should be + * passed to a RTCVideoCapturer implementation, e.g. + * RTCCameraVideoCapturer, in order to produce frames. + */ +- (RTC_OBJC_TYPE(RTCVideoSource) *)videoSourceForScreenCast:(BOOL)forScreenCast; + +/** Initialize an RTCVideoTrack with a source and an id. */ +- (RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrackWithSource:(RTC_OBJC_TYPE(RTCVideoSource) *)source + trackId:(NSString *)trackId; + +/** Initialize an RTCMediaStream with an id. */ +- (RTC_OBJC_TYPE(RTCMediaStream) *)mediaStreamWithStreamId:(NSString *)streamId; + +/** Initialize an RTCPeerConnection with a configuration, constraints, and + * delegate. + */ +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + delegate:(nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate; + +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + certificateVerifier: + (id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)>)certificateVerifier + delegate:(nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate; + +/** Set the options to be used for subsequently created RTCPeerConnections */ +- (void)setOptions:(nonnull RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions) *)options; + +/** Start an AecDump recording. This API call will likely change in the future. */ +- (BOOL)startAecDumpWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes; + +/* Stop an active AecDump recording */ +- (void)stopAecDump; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm new file mode 100644 index 0000000000..62b55543d4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -0,0 +1,346 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> + +#import "RTCPeerConnectionFactory+Native.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "RTCPeerConnectionFactoryOptions+Private.h" + +#import "RTCAudioSource+Private.h" +#import "RTCAudioTrack+Private.h" +#import "RTCMediaConstraints+Private.h" +#import "RTCMediaStream+Private.h" +#import "RTCPeerConnection+Private.h" +#import "RTCVideoSource+Private.h" +#import "RTCVideoTrack+Private.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoDecoderFactory.h" +#import "base/RTCVideoEncoderFactory.h" +#import "helpers/NSString+StdString.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/api/network_monitor_factory.h" +#include "sdk/objc/native/api/ssl_certificate_verifier.h" +#include "system_wrappers/include/field_trial.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/transport/field_trial_based_config.h" +#import "components/video_codec/RTCVideoDecoderFactoryH264.h" +#import "components/video_codec/RTCVideoEncoderFactoryH264.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" + +#include "sdk/objc/native/api/objc_audio_device_module.h" +#include "sdk/objc/native/api/video_decoder_factory.h" +#include "sdk/objc/native/api/video_encoder_factory.h" +#include "sdk/objc/native/src/objc_video_decoder_factory.h" +#include "sdk/objc/native/src/objc_video_encoder_factory.h" + +#if defined(WEBRTC_IOS) +#import "sdk/objc/native/api/audio_device_module.h" +#endif + +@implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { + std::unique_ptr<rtc::Thread> _networkThread; + std::unique_ptr<rtc::Thread> _workerThread; + std::unique_ptr<rtc::Thread> _signalingThread; + BOOL _hasStartedAecDump; +} + +@synthesize nativeFactory = _nativeFactory; + +- (rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModule { +#if defined(WEBRTC_IOS) + return webrtc::CreateAudioDeviceModule(); +#else + return nullptr; +#endif +} + +- (instancetype)init { + return [self + initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() + nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() + nativeVideoEncoderFactory:webrtc::ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE( + RTCVideoEncoderFactoryH264) alloc] init]) + nativeVideoDecoderFactory:webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE( + RTCVideoDecoderFactoryH264) alloc] init]) + audioDeviceModule:[self audioDeviceModule].get() + audioProcessingModule:nullptr]; +} + +- (instancetype) + initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory + decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory { + return [self initWithEncoderFactory:encoderFactory decoderFactory:decoderFactory audioDevice:nil]; +} + +- (instancetype) + initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory + decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory + audioDevice:(nullable id<RTC_OBJC_TYPE(RTCAudioDevice)>)audioDevice { +#ifdef HAVE_NO_MEDIA + return [self initWithNoMedia]; +#else + std::unique_ptr<webrtc::VideoEncoderFactory> native_encoder_factory; + std::unique_ptr<webrtc::VideoDecoderFactory> native_decoder_factory; + if (encoderFactory) { + native_encoder_factory = webrtc::ObjCToNativeVideoEncoderFactory(encoderFactory); + } + if (decoderFactory) { + native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory); + } + rtc::scoped_refptr<webrtc::AudioDeviceModule> audio_device_module; + if (audioDevice) { + audio_device_module = webrtc::CreateAudioDeviceModule(audioDevice); + } else { + audio_device_module = [self audioDeviceModule]; + } + return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() + nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() + nativeVideoEncoderFactory:std::move(native_encoder_factory) + nativeVideoDecoderFactory:std::move(native_decoder_factory) + audioDeviceModule:audio_device_module.get() + audioProcessingModule:nullptr]; +#endif +} + +- (instancetype)initNative { + if (self = [super init]) { + _networkThread = rtc::Thread::CreateWithSocketServer(); + _networkThread->SetName("network_thread", _networkThread.get()); + BOOL result = _networkThread->Start(); + RTC_DCHECK(result) << "Failed to start network thread."; + + _workerThread = rtc::Thread::Create(); + _workerThread->SetName("worker_thread", _workerThread.get()); + result = _workerThread->Start(); + RTC_DCHECK(result) << "Failed to start worker thread."; + + _signalingThread = rtc::Thread::Create(); + _signalingThread->SetName("signaling_thread", _signalingThread.get()); + result = _signalingThread->Start(); + RTC_DCHECK(result) << "Failed to start signaling thread."; + } + return self; +} + +- (instancetype)initWithNoMedia { + if (self = [self initNative]) { + webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = _networkThread.get(); + dependencies.worker_thread = _workerThread.get(); + dependencies.signaling_thread = _signalingThread.get(); + if (webrtc::field_trial::IsEnabled("WebRTC-Network-UseNWPathMonitor")) { + dependencies.network_monitor_factory = webrtc::CreateNetworkMonitorFactory(); + } + _nativeFactory = webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); + NSAssert(_nativeFactory, @"Failed to initialize PeerConnectionFactory!"); + } + return self; +} + +- (instancetype)initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory + audioDeviceModule:(webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule { + return [self initWithNativeAudioEncoderFactory:audioEncoderFactory + nativeAudioDecoderFactory:audioDecoderFactory + nativeVideoEncoderFactory:std::move(videoEncoderFactory) + nativeVideoDecoderFactory:std::move(videoDecoderFactory) + audioDeviceModule:audioDeviceModule + audioProcessingModule:audioProcessingModule + networkControllerFactory:nullptr]; +} +- (instancetype)initWithNativeAudioEncoderFactory: + (rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory + nativeAudioDecoderFactory: + (rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory + nativeVideoEncoderFactory: + (std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory + nativeVideoDecoderFactory: + (std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory + audioDeviceModule:(webrtc::AudioDeviceModule *)audioDeviceModule + audioProcessingModule: + (rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule + networkControllerFactory: + (std::unique_ptr<webrtc::NetworkControllerFactoryInterface>) + networkControllerFactory { + if (self = [self initNative]) { + webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = _networkThread.get(); + dependencies.worker_thread = _workerThread.get(); + dependencies.signaling_thread = _signalingThread.get(); + if (webrtc::field_trial::IsEnabled("WebRTC-Network-UseNWPathMonitor")) { + dependencies.network_monitor_factory = webrtc::CreateNetworkMonitorFactory(); + } + dependencies.trials = std::make_unique<webrtc::FieldTrialBasedConfig>(); + dependencies.task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(dependencies.trials.get()); + cricket::MediaEngineDependencies media_deps; + media_deps.adm = std::move(audioDeviceModule); + media_deps.task_queue_factory = dependencies.task_queue_factory.get(); + media_deps.audio_encoder_factory = std::move(audioEncoderFactory); + media_deps.audio_decoder_factory = std::move(audioDecoderFactory); + media_deps.video_encoder_factory = std::move(videoEncoderFactory); + media_deps.video_decoder_factory = std::move(videoDecoderFactory); + if (audioProcessingModule) { + media_deps.audio_processing = std::move(audioProcessingModule); + } else { + media_deps.audio_processing = webrtc::AudioProcessingBuilder().Create(); + } + media_deps.trials = dependencies.trials.get(); + dependencies.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); + dependencies.call_factory = webrtc::CreateCallFactory(); + dependencies.event_log_factory = + std::make_unique<webrtc::RtcEventLogFactory>(dependencies.task_queue_factory.get()); + dependencies.network_controller_factory = std::move(networkControllerFactory); + _nativeFactory = webrtc::CreateModularPeerConnectionFactory(std::move(dependencies)); + NSAssert(_nativeFactory, @"Failed to initialize PeerConnectionFactory!"); + } + return self; +} + +- (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints: + (nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints { + std::unique_ptr<webrtc::MediaConstraints> nativeConstraints; + if (constraints) { + nativeConstraints = constraints.nativeConstraints; + } + cricket::AudioOptions options; + CopyConstraintsIntoAudioOptions(nativeConstraints.get(), &options); + + rtc::scoped_refptr<webrtc::AudioSourceInterface> source = + _nativeFactory->CreateAudioSource(options); + return [[RTC_OBJC_TYPE(RTCAudioSource) alloc] initWithFactory:self nativeAudioSource:source]; +} + +- (RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrackWithTrackId:(NSString *)trackId { + RTC_OBJC_TYPE(RTCAudioSource) *audioSource = [self audioSourceWithConstraints:nil]; + return [self audioTrackWithSource:audioSource trackId:trackId]; +} + +- (RTC_OBJC_TYPE(RTCAudioTrack) *)audioTrackWithSource:(RTC_OBJC_TYPE(RTCAudioSource) *)source + trackId:(NSString *)trackId { + return [[RTC_OBJC_TYPE(RTCAudioTrack) alloc] initWithFactory:self source:source trackId:trackId]; +} + +- (RTC_OBJC_TYPE(RTCVideoSource) *)videoSource { + return [[RTC_OBJC_TYPE(RTCVideoSource) alloc] initWithFactory:self + signalingThread:_signalingThread.get() + workerThread:_workerThread.get()]; +} + +- (RTC_OBJC_TYPE(RTCVideoSource) *)videoSourceForScreenCast:(BOOL)forScreenCast { + return [[RTC_OBJC_TYPE(RTCVideoSource) alloc] initWithFactory:self + signalingThread:_signalingThread.get() + workerThread:_workerThread.get() + isScreenCast:forScreenCast]; +} + +- (RTC_OBJC_TYPE(RTCVideoTrack) *)videoTrackWithSource:(RTC_OBJC_TYPE(RTCVideoSource) *)source + trackId:(NSString *)trackId { + return [[RTC_OBJC_TYPE(RTCVideoTrack) alloc] initWithFactory:self source:source trackId:trackId]; +} + +- (RTC_OBJC_TYPE(RTCMediaStream) *)mediaStreamWithStreamId:(NSString *)streamId { + return [[RTC_OBJC_TYPE(RTCMediaStream) alloc] initWithFactory:self streamId:streamId]; +} + +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + delegate: + (nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate { + return [[RTC_OBJC_TYPE(RTCPeerConnection) alloc] initWithFactory:self + configuration:configuration + constraints:constraints + certificateVerifier:nil + delegate:delegate]; +} + +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithConfiguration:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + certificateVerifier: + (id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)>)certificateVerifier + delegate: + (nullable id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate { + return [[RTC_OBJC_TYPE(RTCPeerConnection) alloc] initWithFactory:self + configuration:configuration + constraints:constraints + certificateVerifier:certificateVerifier + delegate:delegate]; +} + +- (nullable RTC_OBJC_TYPE(RTCPeerConnection) *) + peerConnectionWithDependencies:(RTC_OBJC_TYPE(RTCConfiguration) *)configuration + constraints:(RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints + dependencies:(std::unique_ptr<webrtc::PeerConnectionDependencies>)dependencies + delegate:(id<RTC_OBJC_TYPE(RTCPeerConnectionDelegate)>)delegate { + return [[RTC_OBJC_TYPE(RTCPeerConnection) alloc] initWithDependencies:self + configuration:configuration + constraints:constraints + dependencies:std::move(dependencies) + delegate:delegate]; +} + +- (void)setOptions:(nonnull RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions) *)options { + RTC_DCHECK(options != nil); + _nativeFactory->SetOptions(options.nativeOptions); +} + +- (BOOL)startAecDumpWithFilePath:(NSString *)filePath + maxSizeInBytes:(int64_t)maxSizeInBytes { + RTC_DCHECK(filePath.length); + RTC_DCHECK_GT(maxSizeInBytes, 0); + + if (_hasStartedAecDump) { + RTCLogError(@"Aec dump already started."); + return NO; + } + FILE *f = fopen(filePath.UTF8String, "wb"); + if (!f) { + RTCLogError(@"Error opening file: %@. Error: %s", filePath, strerror(errno)); + return NO; + } + _hasStartedAecDump = _nativeFactory->StartAecDump(f, maxSizeInBytes); + return _hasStartedAecDump; +} + +- (void)stopAecDump { + _nativeFactory->StopAecDump(); + _hasStartedAecDump = NO; +} + +- (rtc::Thread *)signalingThread { + return _signalingThread.get(); +} + +- (rtc::Thread *)workerThread { + return _workerThread.get(); +} + +- (rtc::Thread *)networkThread { + return _networkThread.get(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h new file mode 100644 index 0000000000..070a0e74a5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h @@ -0,0 +1,21 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactoryBuilder.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCPeerConnectionFactoryBuilder (DefaultComponents) + ++ (RTCPeerConnectionFactoryBuilder *)defaultBuilder; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm new file mode 100644 index 0000000000..522e520e12 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.mm @@ -0,0 +1,49 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactory+Native.h" +#import "RTCPeerConnectionFactoryBuilder+DefaultComponents.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#import "components/video_codec/RTCVideoDecoderFactoryH264.h" +#import "components/video_codec/RTCVideoEncoderFactoryH264.h" +#include "sdk/objc/native/api/video_decoder_factory.h" +#include "sdk/objc/native/api/video_encoder_factory.h" + +#if defined(WEBRTC_IOS) +#import "sdk/objc/native/api/audio_device_module.h" +#endif + +@implementation RTCPeerConnectionFactoryBuilder (DefaultComponents) + ++ (RTCPeerConnectionFactoryBuilder *)defaultBuilder { + RTCPeerConnectionFactoryBuilder *builder = [[RTCPeerConnectionFactoryBuilder alloc] init]; + auto audioEncoderFactory = webrtc::CreateBuiltinAudioEncoderFactory(); + [builder setAudioEncoderFactory:audioEncoderFactory]; + + auto audioDecoderFactory = webrtc::CreateBuiltinAudioDecoderFactory(); + [builder setAudioDecoderFactory:audioDecoderFactory]; + + auto videoEncoderFactory = webrtc::ObjCToNativeVideoEncoderFactory( + [[RTC_OBJC_TYPE(RTCVideoEncoderFactoryH264) alloc] init]); + [builder setVideoEncoderFactory:std::move(videoEncoderFactory)]; + + auto videoDecoderFactory = webrtc::ObjCToNativeVideoDecoderFactory( + [[RTC_OBJC_TYPE(RTCVideoDecoderFactoryH264) alloc] init]); + [builder setVideoDecoderFactory:std::move(videoDecoderFactory)]; + +#if defined(WEBRTC_IOS) + [builder setAudioDeviceModule:webrtc::CreateAudioDeviceModule()]; +#endif + return builder; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h new file mode 100644 index 0000000000..f0b0de156a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.h @@ -0,0 +1,48 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactory.h" + +#include "api/scoped_refptr.h" + +namespace webrtc { + +class AudioDeviceModule; +class AudioEncoderFactory; +class AudioDecoderFactory; +class VideoEncoderFactory; +class VideoDecoderFactory; +class AudioProcessing; + +} // namespace webrtc + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCPeerConnectionFactoryBuilder : NSObject + ++ (RTCPeerConnectionFactoryBuilder *)builder; + +- (RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)createPeerConnectionFactory; + +- (void)setVideoEncoderFactory:(std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory; + +- (void)setVideoDecoderFactory:(std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory; + +- (void)setAudioEncoderFactory:(rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory; + +- (void)setAudioDecoderFactory:(rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory; + +- (void)setAudioDeviceModule:(rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModule; + +- (void)setAudioProcessingModule:(rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm new file mode 100644 index 0000000000..627909a0e3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm @@ -0,0 +1,72 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactoryBuilder.h" +#import "RTCPeerConnectionFactory+Native.h" + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" + +@implementation RTCPeerConnectionFactoryBuilder { + std::unique_ptr<webrtc::VideoEncoderFactory> _videoEncoderFactory; + std::unique_ptr<webrtc::VideoDecoderFactory> _videoDecoderFactory; + rtc::scoped_refptr<webrtc::AudioEncoderFactory> _audioEncoderFactory; + rtc::scoped_refptr<webrtc::AudioDecoderFactory> _audioDecoderFactory; + rtc::scoped_refptr<webrtc::AudioDeviceModule> _audioDeviceModule; + rtc::scoped_refptr<webrtc::AudioProcessing> _audioProcessingModule; +} + ++ (RTCPeerConnectionFactoryBuilder *)builder { + return [[RTCPeerConnectionFactoryBuilder alloc] init]; +} + +- (RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)createPeerConnectionFactory { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc]; + return [factory initWithNativeAudioEncoderFactory:_audioEncoderFactory + nativeAudioDecoderFactory:_audioDecoderFactory + nativeVideoEncoderFactory:std::move(_videoEncoderFactory) + nativeVideoDecoderFactory:std::move(_videoDecoderFactory) + audioDeviceModule:_audioDeviceModule.get() + audioProcessingModule:_audioProcessingModule]; +} + +- (void)setVideoEncoderFactory:(std::unique_ptr<webrtc::VideoEncoderFactory>)videoEncoderFactory { + _videoEncoderFactory = std::move(videoEncoderFactory); +} + +- (void)setVideoDecoderFactory:(std::unique_ptr<webrtc::VideoDecoderFactory>)videoDecoderFactory { + _videoDecoderFactory = std::move(videoDecoderFactory); +} + +- (void)setAudioEncoderFactory: + (rtc::scoped_refptr<webrtc::AudioEncoderFactory>)audioEncoderFactory { + _audioEncoderFactory = audioEncoderFactory; +} + +- (void)setAudioDecoderFactory: + (rtc::scoped_refptr<webrtc::AudioDecoderFactory>)audioDecoderFactory { + _audioDecoderFactory = audioDecoderFactory; +} + +- (void)setAudioDeviceModule:(rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModule { + _audioDeviceModule = audioDeviceModule; +} + +- (void)setAudioProcessingModule: + (rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule { + _audioProcessingModule = audioProcessingModule; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions+Private.h new file mode 100644 index 0000000000..8832b23695 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactoryOptions.h" + +#include "api/peer_connection_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCPeerConnectionFactoryOptions) +() + + /** Returns the equivalent native PeerConnectionFactoryInterface::Options + * structure. */ + @property(nonatomic, readonly) webrtc::PeerConnectionFactoryInterface::Options nativeOptions; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h new file mode 100644 index 0000000000..bfc54a5d7b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.h @@ -0,0 +1,38 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCPeerConnectionFactoryOptions) : NSObject + +@property(nonatomic, assign) BOOL disableEncryption; + +@property(nonatomic, assign) BOOL disableNetworkMonitor; + +@property(nonatomic, assign) BOOL ignoreLoopbackNetworkAdapter; + +@property(nonatomic, assign) BOOL ignoreVPNNetworkAdapter; + +@property(nonatomic, assign) BOOL ignoreCellularNetworkAdapter; + +@property(nonatomic, assign) BOOL ignoreWiFiNetworkAdapter; + +@property(nonatomic, assign) BOOL ignoreEthernetNetworkAdapter; + +- (instancetype)init NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.mm new file mode 100644 index 0000000000..5467bd5fc9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryOptions.mm @@ -0,0 +1,56 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCPeerConnectionFactoryOptions+Private.h" + +#include "rtc_base/network_constants.h" + +namespace { + +void setNetworkBit(webrtc::PeerConnectionFactoryInterface::Options* options, + rtc::AdapterType type, + bool ignore) { + if (ignore) { + options->network_ignore_mask |= type; + } else { + options->network_ignore_mask &= ~type; + } +} +} // namespace + +@implementation RTC_OBJC_TYPE (RTCPeerConnectionFactoryOptions) + +@synthesize disableEncryption = _disableEncryption; +@synthesize disableNetworkMonitor = _disableNetworkMonitor; +@synthesize ignoreLoopbackNetworkAdapter = _ignoreLoopbackNetworkAdapter; +@synthesize ignoreVPNNetworkAdapter = _ignoreVPNNetworkAdapter; +@synthesize ignoreCellularNetworkAdapter = _ignoreCellularNetworkAdapter; +@synthesize ignoreWiFiNetworkAdapter = _ignoreWiFiNetworkAdapter; +@synthesize ignoreEthernetNetworkAdapter = _ignoreEthernetNetworkAdapter; + +- (instancetype)init { + return [super init]; +} + +- (webrtc::PeerConnectionFactoryInterface::Options)nativeOptions { + webrtc::PeerConnectionFactoryInterface::Options options; + options.disable_encryption = self.disableEncryption; + options.disable_network_monitor = self.disableNetworkMonitor; + + setNetworkBit(&options, rtc::ADAPTER_TYPE_LOOPBACK, self.ignoreLoopbackNetworkAdapter); + setNetworkBit(&options, rtc::ADAPTER_TYPE_VPN, self.ignoreVPNNetworkAdapter); + setNetworkBit(&options, rtc::ADAPTER_TYPE_CELLULAR, self.ignoreCellularNetworkAdapter); + setNetworkBit(&options, rtc::ADAPTER_TYPE_WIFI, self.ignoreWiFiNetworkAdapter); + setNetworkBit(&options, rtc::ADAPTER_TYPE_ETHERNET, self.ignoreEthernetNetworkAdapter); + + return options; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters+Private.h new file mode 100644 index 0000000000..c4d196cf79 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtcpParameters.h" + +#include "api/rtp_parameters.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCRtcpParameters) +() + + /** Returns the equivalent native RtcpParameters structure. */ + @property(nonatomic, readonly) webrtc::RtcpParameters nativeParameters; + +/** Initialize the object with a native RtcpParameters structure. */ +- (instancetype)initWithNativeParameters:(const webrtc::RtcpParameters &)nativeParameters + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.h new file mode 100644 index 0000000000..2f7aad3aef --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.h @@ -0,0 +1,30 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtcpParameters) : NSObject + +/** The Canonical Name used by RTCP. */ +@property(nonatomic, readonly, copy) NSString *cname; + +/** Whether reduced size RTCP is configured or compound RTCP. */ +@property(nonatomic, assign) BOOL isReducedSize; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.mm new file mode 100644 index 0000000000..e92ee4b3e7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtcpParameters.mm @@ -0,0 +1,40 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtcpParameters+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCRtcpParameters) + +@synthesize cname = _cname; +@synthesize isReducedSize = _isReducedSize; + +- (instancetype)init { + webrtc::RtcpParameters nativeParameters; + return [self initWithNativeParameters:nativeParameters]; +} + +- (instancetype)initWithNativeParameters:(const webrtc::RtcpParameters &)nativeParameters { + if (self = [super init]) { + _cname = [NSString stringForStdString:nativeParameters.cname]; + _isReducedSize = nativeParameters.reduced_size; + } + return self; +} + +- (webrtc::RtcpParameters)nativeParameters { + webrtc::RtcpParameters parameters; + parameters.cname = [NSString stdStringForString:_cname]; + parameters.reduced_size = _isReducedSize; + return parameters; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters+Private.h new file mode 100644 index 0000000000..ff23cfd642 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpCodecParameters.h" + +#include "api/rtp_parameters.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCRtpCodecParameters) +() + + /** Returns the equivalent native RtpCodecParameters structure. */ + @property(nonatomic, readonly) webrtc::RtpCodecParameters nativeParameters; + +/** Initialize the object with a native RtpCodecParameters structure. */ +- (instancetype)initWithNativeParameters:(const webrtc::RtpCodecParameters &)nativeParameters + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h new file mode 100644 index 0000000000..6135223720 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.h @@ -0,0 +1,73 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_EXTERN const NSString *const kRTCRtxCodecName; +RTC_EXTERN const NSString *const kRTCRedCodecName; +RTC_EXTERN const NSString *const kRTCUlpfecCodecName; +RTC_EXTERN const NSString *const kRTCFlexfecCodecName; +RTC_EXTERN const NSString *const kRTCOpusCodecName; +RTC_EXTERN const NSString *const kRTCIsacCodecName; +RTC_EXTERN const NSString *const kRTCL16CodecName; +RTC_EXTERN const NSString *const kRTCG722CodecName; +RTC_EXTERN const NSString *const kRTCIlbcCodecName; +RTC_EXTERN const NSString *const kRTCPcmuCodecName; +RTC_EXTERN const NSString *const kRTCPcmaCodecName; +RTC_EXTERN const NSString *const kRTCDtmfCodecName; +RTC_EXTERN const NSString *const kRTCComfortNoiseCodecName; +RTC_EXTERN const NSString *const kRTCVp8CodecName; +RTC_EXTERN const NSString *const kRTCVp9CodecName; +RTC_EXTERN const NSString *const kRTCH264CodecName; + +/** Defined in https://www.w3.org/TR/webrtc/#idl-def-rtcrtpcodecparameters */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpCodecParameters) : NSObject + +/** The RTP payload type. */ +@property(nonatomic, assign) int payloadType; + +/** + * The codec MIME subtype. Valid types are listed in: + * http://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml#rtp-parameters-2 + * + * Several supported types are represented by the constants above. + */ +@property(nonatomic, readonly, nonnull) NSString *name; + +/** + * The media type of this codec. Equivalent to MIME top-level type. + * + * Valid values are kRTCMediaStreamTrackKindAudio and + * kRTCMediaStreamTrackKindVideo. + */ +@property(nonatomic, readonly, nonnull) NSString *kind; + +/** The codec clock rate expressed in Hertz. */ +@property(nonatomic, readonly, nullable) NSNumber *clockRate; + +/** + * The number of channels (mono=1, stereo=2). + * Set to null for video codecs. + **/ +@property(nonatomic, readonly, nullable) NSNumber *numChannels; + +/** The "format specific parameters" field from the "a=fmtp" line in the SDP */ +@property(nonatomic, readonly, nonnull) NSDictionary *parameters; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm new file mode 100644 index 0000000000..6201e57b93 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm @@ -0,0 +1,112 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpCodecParameters+Private.h" + +#import "RTCMediaStreamTrack.h" +#import "helpers/NSString+StdString.h" + +#include "media/base/media_constants.h" +#include "rtc_base/checks.h" + +const NSString * const kRTCRtxCodecName = @(cricket::kRtxCodecName); +const NSString * const kRTCRedCodecName = @(cricket::kRedCodecName); +const NSString * const kRTCUlpfecCodecName = @(cricket::kUlpfecCodecName); +const NSString * const kRTCFlexfecCodecName = @(cricket::kFlexfecCodecName); +const NSString * const kRTCOpusCodecName = @(cricket::kOpusCodecName); +const NSString * const kRTCL16CodecName = @(cricket::kL16CodecName); +const NSString * const kRTCG722CodecName = @(cricket::kG722CodecName); +const NSString * const kRTCIlbcCodecName = @(cricket::kIlbcCodecName); +const NSString * const kRTCPcmuCodecName = @(cricket::kPcmuCodecName); +const NSString * const kRTCPcmaCodecName = @(cricket::kPcmaCodecName); +const NSString * const kRTCDtmfCodecName = @(cricket::kDtmfCodecName); +const NSString * const kRTCComfortNoiseCodecName = + @(cricket::kComfortNoiseCodecName); +const NSString * const kRTCVp8CodecName = @(cricket::kVp8CodecName); +const NSString * const kRTCVp9CodecName = @(cricket::kVp9CodecName); +const NSString * const kRTCH264CodecName = @(cricket::kH264CodecName); + +@implementation RTC_OBJC_TYPE (RTCRtpCodecParameters) + +@synthesize payloadType = _payloadType; +@synthesize name = _name; +@synthesize kind = _kind; +@synthesize clockRate = _clockRate; +@synthesize numChannels = _numChannels; +@synthesize parameters = _parameters; + +- (instancetype)init { + webrtc::RtpCodecParameters nativeParameters; + return [self initWithNativeParameters:nativeParameters]; +} + +- (instancetype)initWithNativeParameters: + (const webrtc::RtpCodecParameters &)nativeParameters { + if (self = [super init]) { + _payloadType = nativeParameters.payload_type; + _name = [NSString stringForStdString:nativeParameters.name]; + switch (nativeParameters.kind) { + case cricket::MEDIA_TYPE_AUDIO: + _kind = kRTCMediaStreamTrackKindAudio; + break; + case cricket::MEDIA_TYPE_VIDEO: + _kind = kRTCMediaStreamTrackKindVideo; + break; + case cricket::MEDIA_TYPE_DATA: + RTC_DCHECK_NOTREACHED(); + break; + case cricket::MEDIA_TYPE_UNSUPPORTED: + RTC_DCHECK_NOTREACHED(); + break; + } + if (nativeParameters.clock_rate) { + _clockRate = [NSNumber numberWithInt:*nativeParameters.clock_rate]; + } + if (nativeParameters.num_channels) { + _numChannels = [NSNumber numberWithInt:*nativeParameters.num_channels]; + } + NSMutableDictionary *parameters = [NSMutableDictionary dictionary]; + for (const auto ¶meter : nativeParameters.parameters) { + [parameters setObject:[NSString stringForStdString:parameter.second] + forKey:[NSString stringForStdString:parameter.first]]; + } + _parameters = parameters; + } + return self; +} + +- (webrtc::RtpCodecParameters)nativeParameters { + webrtc::RtpCodecParameters parameters; + parameters.payload_type = _payloadType; + parameters.name = [NSString stdStringForString:_name]; + // NSString pointer comparison is safe here since "kind" is readonly and only + // populated above. + if (_kind == kRTCMediaStreamTrackKindAudio) { + parameters.kind = cricket::MEDIA_TYPE_AUDIO; + } else if (_kind == kRTCMediaStreamTrackKindVideo) { + parameters.kind = cricket::MEDIA_TYPE_VIDEO; + } else { + RTC_DCHECK_NOTREACHED(); + } + if (_clockRate != nil) { + parameters.clock_rate = absl::optional<int>(_clockRate.intValue); + } + if (_numChannels != nil) { + parameters.num_channels = absl::optional<int>(_numChannels.intValue); + } + for (NSString *paramKey in _parameters.allKeys) { + std::string key = [NSString stdStringForString:paramKey]; + std::string value = [NSString stdStringForString:_parameters[paramKey]]; + parameters.parameters[key] = value; + } + return parameters; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters+Private.h new file mode 100644 index 0000000000..d12ca624e3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpEncodingParameters.h" + +#include "api/rtp_parameters.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCRtpEncodingParameters) +() + + /** Returns the equivalent native RtpEncodingParameters structure. */ + @property(nonatomic, readonly) webrtc::RtpEncodingParameters nativeParameters; + +/** Initialize the object with a native RtpEncodingParameters structure. */ +- (instancetype)initWithNativeParameters:(const webrtc::RtpEncodingParameters &)nativeParameters + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h new file mode 100644 index 0000000000..07f6b7a39c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h @@ -0,0 +1,76 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Corresponds to webrtc::Priority. */ +typedef NS_ENUM(NSInteger, RTCPriority) { + RTCPriorityVeryLow, + RTCPriorityLow, + RTCPriorityMedium, + RTCPriorityHigh +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpEncodingParameters) : NSObject + +/** The idenfifier for the encoding layer. This is used in simulcast. */ +@property(nonatomic, copy, nullable) NSString *rid; + +/** Controls whether the encoding is currently transmitted. */ +@property(nonatomic, assign) BOOL isActive; + +/** The maximum bitrate to use for the encoding, or nil if there is no + * limit. + */ +@property(nonatomic, copy, nullable) NSNumber *maxBitrateBps; + +/** The minimum bitrate to use for the encoding, or nil if there is no + * limit. + */ +@property(nonatomic, copy, nullable) NSNumber *minBitrateBps; + +/** The maximum framerate to use for the encoding, or nil if there is no + * limit. + */ +@property(nonatomic, copy, nullable) NSNumber *maxFramerate; + +/** The requested number of temporal layers to use for the encoding, or nil + * if the default should be used. + */ +@property(nonatomic, copy, nullable) NSNumber *numTemporalLayers; + +/** Scale the width and height down by this factor for video. If nil, + * implementation default scaling factor will be used. + */ +@property(nonatomic, copy, nullable) NSNumber *scaleResolutionDownBy; + +/** The SSRC being used by this encoding. */ +@property(nonatomic, readonly, nullable) NSNumber *ssrc; + +/** The relative bitrate priority. */ +@property(nonatomic, assign) double bitratePriority; + +/** The relative DiffServ Code Point priority. */ +@property(nonatomic, assign) RTCPriority networkPriority; + +/** Allow dynamic frame length changes for audio: + https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime */ +@property(nonatomic, assign) BOOL adaptiveAudioPacketTime; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm new file mode 100644 index 0000000000..d6087dafb0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm @@ -0,0 +1,128 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpEncodingParameters+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCRtpEncodingParameters) + +@synthesize rid = _rid; +@synthesize isActive = _isActive; +@synthesize maxBitrateBps = _maxBitrateBps; +@synthesize minBitrateBps = _minBitrateBps; +@synthesize maxFramerate = _maxFramerate; +@synthesize numTemporalLayers = _numTemporalLayers; +@synthesize scaleResolutionDownBy = _scaleResolutionDownBy; +@synthesize ssrc = _ssrc; +@synthesize bitratePriority = _bitratePriority; +@synthesize networkPriority = _networkPriority; +@synthesize adaptiveAudioPacketTime = _adaptiveAudioPacketTime; + +- (instancetype)init { + webrtc::RtpEncodingParameters nativeParameters; + return [self initWithNativeParameters:nativeParameters]; +} + +- (instancetype)initWithNativeParameters: + (const webrtc::RtpEncodingParameters &)nativeParameters { + if (self = [super init]) { + if (!nativeParameters.rid.empty()) { + _rid = [NSString stringForStdString:nativeParameters.rid]; + } + _isActive = nativeParameters.active; + if (nativeParameters.max_bitrate_bps) { + _maxBitrateBps = + [NSNumber numberWithInt:*nativeParameters.max_bitrate_bps]; + } + if (nativeParameters.min_bitrate_bps) { + _minBitrateBps = + [NSNumber numberWithInt:*nativeParameters.min_bitrate_bps]; + } + if (nativeParameters.max_framerate) { + _maxFramerate = [NSNumber numberWithInt:*nativeParameters.max_framerate]; + } + if (nativeParameters.num_temporal_layers) { + _numTemporalLayers = [NSNumber numberWithInt:*nativeParameters.num_temporal_layers]; + } + if (nativeParameters.scale_resolution_down_by) { + _scaleResolutionDownBy = + [NSNumber numberWithDouble:*nativeParameters.scale_resolution_down_by]; + } + if (nativeParameters.ssrc) { + _ssrc = [NSNumber numberWithUnsignedLong:*nativeParameters.ssrc]; + } + _bitratePriority = nativeParameters.bitrate_priority; + _networkPriority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters) + priorityFromNativePriority:nativeParameters.network_priority]; + _adaptiveAudioPacketTime = nativeParameters.adaptive_ptime; + } + return self; +} + +- (webrtc::RtpEncodingParameters)nativeParameters { + webrtc::RtpEncodingParameters parameters; + if (_rid != nil) { + parameters.rid = [NSString stdStringForString:_rid]; + } + parameters.active = _isActive; + if (_maxBitrateBps != nil) { + parameters.max_bitrate_bps = absl::optional<int>(_maxBitrateBps.intValue); + } + if (_minBitrateBps != nil) { + parameters.min_bitrate_bps = absl::optional<int>(_minBitrateBps.intValue); + } + if (_maxFramerate != nil) { + parameters.max_framerate = absl::optional<int>(_maxFramerate.intValue); + } + if (_numTemporalLayers != nil) { + parameters.num_temporal_layers = absl::optional<int>(_numTemporalLayers.intValue); + } + if (_scaleResolutionDownBy != nil) { + parameters.scale_resolution_down_by = + absl::optional<double>(_scaleResolutionDownBy.doubleValue); + } + if (_ssrc != nil) { + parameters.ssrc = absl::optional<uint32_t>(_ssrc.unsignedLongValue); + } + parameters.bitrate_priority = _bitratePriority; + parameters.network_priority = + [RTC_OBJC_TYPE(RTCRtpEncodingParameters) nativePriorityFromPriority:_networkPriority]; + parameters.adaptive_ptime = _adaptiveAudioPacketTime; + return parameters; +} + ++ (webrtc::Priority)nativePriorityFromPriority:(RTCPriority)networkPriority { + switch (networkPriority) { + case RTCPriorityVeryLow: + return webrtc::Priority::kVeryLow; + case RTCPriorityLow: + return webrtc::Priority::kLow; + case RTCPriorityMedium: + return webrtc::Priority::kMedium; + case RTCPriorityHigh: + return webrtc::Priority::kHigh; + } +} + ++ (RTCPriority)priorityFromNativePriority:(webrtc::Priority)nativePriority { + switch (nativePriority) { + case webrtc::Priority::kVeryLow: + return RTCPriorityVeryLow; + case webrtc::Priority::kLow: + return RTCPriorityLow; + case webrtc::Priority::kMedium: + return RTCPriorityMedium; + case webrtc::Priority::kHigh: + return RTCPriorityHigh; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension+Private.h new file mode 100644 index 0000000000..0e0fbba5ac --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpHeaderExtension.h" + +#include "api/rtp_parameters.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCRtpHeaderExtension) +() + + /** Returns the equivalent native RtpExtension structure. */ + @property(nonatomic, readonly) webrtc::RtpExtension nativeParameters; + +/** Initialize the object with a native RtpExtension structure. */ +- (instancetype)initWithNativeParameters:(const webrtc::RtpExtension &)nativeParameters + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.h new file mode 100644 index 0000000000..4000bf5372 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.h @@ -0,0 +1,33 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpHeaderExtension) : NSObject + +/** The URI of the RTP header extension, as defined in RFC5285. */ +@property(nonatomic, readonly, copy) NSString *uri; + +/** The value put in the RTP packet to identify the header extension. */ +@property(nonatomic, readonly) int id; + +/** Whether the header extension is encrypted or not. */ +@property(nonatomic, readonly, getter=isEncrypted) BOOL encrypted; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.mm new file mode 100644 index 0000000000..68093e92ea --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpHeaderExtension.mm @@ -0,0 +1,43 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpHeaderExtension+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCRtpHeaderExtension) + +@synthesize uri = _uri; +@synthesize id = _id; +@synthesize encrypted = _encrypted; + +- (instancetype)init { + webrtc::RtpExtension nativeExtension; + return [self initWithNativeParameters:nativeExtension]; +} + +- (instancetype)initWithNativeParameters:(const webrtc::RtpExtension &)nativeParameters { + if (self = [super init]) { + _uri = [NSString stringForStdString:nativeParameters.uri]; + _id = nativeParameters.id; + _encrypted = nativeParameters.encrypt; + } + return self; +} + +- (webrtc::RtpExtension)nativeParameters { + webrtc::RtpExtension extension; + extension.uri = [NSString stdStringForString:_uri]; + extension.id = _id; + extension.encrypt = _encrypted; + return extension; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters+Private.h new file mode 100644 index 0000000000..139617f727 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpParameters.h" + +#include "api/rtp_parameters.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCRtpParameters) +() + + /** Returns the equivalent native RtpParameters structure. */ + @property(nonatomic, readonly) webrtc::RtpParameters nativeParameters; + +/** Initialize the object with a native RtpParameters structure. */ +- (instancetype)initWithNativeParameters:(const webrtc::RtpParameters &)nativeParameters + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.h new file mode 100644 index 0000000000..3d71c55ab9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.h @@ -0,0 +1,58 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCRtcpParameters.h" +#import "RTCRtpCodecParameters.h" +#import "RTCRtpEncodingParameters.h" +#import "RTCRtpHeaderExtension.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Corresponds to webrtc::DegradationPreference. */ +typedef NS_ENUM(NSInteger, RTCDegradationPreference) { + RTCDegradationPreferenceDisabled, + RTCDegradationPreferenceMaintainFramerate, + RTCDegradationPreferenceMaintainResolution, + RTCDegradationPreferenceBalanced +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpParameters) : NSObject + +/** A unique identifier for the last set of parameters applied. */ +@property(nonatomic, copy) NSString *transactionId; + +/** Parameters used for RTCP. */ +@property(nonatomic, readonly, copy) RTC_OBJC_TYPE(RTCRtcpParameters) * rtcp; + +/** An array containing parameters for RTP header extensions. */ +@property(nonatomic, readonly, copy) + NSArray<RTC_OBJC_TYPE(RTCRtpHeaderExtension) *> *headerExtensions; + +/** The currently active encodings in the order of preference. */ +@property(nonatomic, copy) NSArray<RTC_OBJC_TYPE(RTCRtpEncodingParameters) *> *encodings; + +/** The negotiated set of send codecs in order of preference. */ +@property(nonatomic, copy) NSArray<RTC_OBJC_TYPE(RTCRtpCodecParameters) *> *codecs; + +/** + * Degradation preference in case of CPU adaptation or constrained bandwidth. + * If nil, implementation default degradation preference will be used. + */ +@property(nonatomic, copy, nullable) NSNumber *degradationPreference; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.mm new file mode 100644 index 0000000000..2baf0ecd80 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpParameters.mm @@ -0,0 +1,121 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpParameters+Private.h" + +#import "RTCRtcpParameters+Private.h" +#import "RTCRtpCodecParameters+Private.h" +#import "RTCRtpEncodingParameters+Private.h" +#import "RTCRtpHeaderExtension+Private.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCRtpParameters) + +@synthesize transactionId = _transactionId; +@synthesize rtcp = _rtcp; +@synthesize headerExtensions = _headerExtensions; +@synthesize encodings = _encodings; +@synthesize codecs = _codecs; +@synthesize degradationPreference = _degradationPreference; + +- (instancetype)init { + webrtc::RtpParameters nativeParameters; + return [self initWithNativeParameters:nativeParameters]; +} + +- (instancetype)initWithNativeParameters: + (const webrtc::RtpParameters &)nativeParameters { + if (self = [super init]) { + _transactionId = [NSString stringForStdString:nativeParameters.transaction_id]; + _rtcp = + [[RTC_OBJC_TYPE(RTCRtcpParameters) alloc] initWithNativeParameters:nativeParameters.rtcp]; + + NSMutableArray *headerExtensions = [[NSMutableArray alloc] init]; + for (const auto &headerExtension : nativeParameters.header_extensions) { + [headerExtensions addObject:[[RTC_OBJC_TYPE(RTCRtpHeaderExtension) alloc] + initWithNativeParameters:headerExtension]]; + } + _headerExtensions = headerExtensions; + + NSMutableArray *encodings = [[NSMutableArray alloc] init]; + for (const auto &encoding : nativeParameters.encodings) { + [encodings addObject:[[RTC_OBJC_TYPE(RTCRtpEncodingParameters) alloc] + initWithNativeParameters:encoding]]; + } + _encodings = encodings; + + NSMutableArray *codecs = [[NSMutableArray alloc] init]; + for (const auto &codec : nativeParameters.codecs) { + [codecs + addObject:[[RTC_OBJC_TYPE(RTCRtpCodecParameters) alloc] initWithNativeParameters:codec]]; + } + _codecs = codecs; + + _degradationPreference = [RTC_OBJC_TYPE(RTCRtpParameters) + degradationPreferenceFromNativeDegradationPreference:nativeParameters + .degradation_preference]; + } + return self; +} + +- (webrtc::RtpParameters)nativeParameters { + webrtc::RtpParameters parameters; + parameters.transaction_id = [NSString stdStringForString:_transactionId]; + parameters.rtcp = [_rtcp nativeParameters]; + for (RTC_OBJC_TYPE(RTCRtpHeaderExtension) * headerExtension in _headerExtensions) { + parameters.header_extensions.push_back(headerExtension.nativeParameters); + } + for (RTC_OBJC_TYPE(RTCRtpEncodingParameters) * encoding in _encodings) { + parameters.encodings.push_back(encoding.nativeParameters); + } + for (RTC_OBJC_TYPE(RTCRtpCodecParameters) * codec in _codecs) { + parameters.codecs.push_back(codec.nativeParameters); + } + if (_degradationPreference) { + parameters.degradation_preference = [RTC_OBJC_TYPE(RTCRtpParameters) + nativeDegradationPreferenceFromDegradationPreference:(RTCDegradationPreference) + _degradationPreference.intValue]; + } + return parameters; +} + ++ (webrtc::DegradationPreference)nativeDegradationPreferenceFromDegradationPreference: + (RTCDegradationPreference)degradationPreference { + switch (degradationPreference) { + case RTCDegradationPreferenceDisabled: + return webrtc::DegradationPreference::DISABLED; + case RTCDegradationPreferenceMaintainFramerate: + return webrtc::DegradationPreference::MAINTAIN_FRAMERATE; + case RTCDegradationPreferenceMaintainResolution: + return webrtc::DegradationPreference::MAINTAIN_RESOLUTION; + case RTCDegradationPreferenceBalanced: + return webrtc::DegradationPreference::BALANCED; + } +} + ++ (NSNumber *)degradationPreferenceFromNativeDegradationPreference: + (absl::optional<webrtc::DegradationPreference>)nativeDegradationPreference { + if (!nativeDegradationPreference.has_value()) { + return nil; + } + + switch (*nativeDegradationPreference) { + case webrtc::DegradationPreference::DISABLED: + return @(RTCDegradationPreferenceDisabled); + case webrtc::DegradationPreference::MAINTAIN_FRAMERATE: + return @(RTCDegradationPreferenceMaintainFramerate); + case webrtc::DegradationPreference::MAINTAIN_RESOLUTION: + return @(RTCDegradationPreferenceMaintainResolution); + case webrtc::DegradationPreference::BALANCED: + return @(RTCDegradationPreferenceBalanced); + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Native.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Native.h new file mode 100644 index 0000000000..c15ce70079 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Native.h @@ -0,0 +1,32 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpReceiver.h" + +#include "api/crypto/frame_decryptor_interface.h" +#include "api/scoped_refptr.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class extension exposes methods that work directly with injectable C++ components. + */ +@interface RTC_OBJC_TYPE (RTCRtpReceiver) +() + + /** Sets a user defined frame decryptor that will decrypt the entire frame. + * This will decrypt the entire frame using the user provided decryption + * mechanism regardless of whether SRTP is enabled or not. + */ + - (void)setFrameDecryptor : (rtc::scoped_refptr<webrtc::FrameDecryptorInterface>)frameDecryptor; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Private.h new file mode 100644 index 0000000000..eccbcbc3a0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver+Private.h @@ -0,0 +1,52 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpReceiver.h" + +#include "api/rtp_receiver_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +namespace webrtc { + +class RtpReceiverDelegateAdapter : public RtpReceiverObserverInterface { + public: + RtpReceiverDelegateAdapter(RTC_OBJC_TYPE(RTCRtpReceiver) * receiver); + + void OnFirstPacketReceived(cricket::MediaType media_type) override; + + private: + __weak RTC_OBJC_TYPE(RTCRtpReceiver) * receiver_; +}; + +} // namespace webrtc + +@interface RTC_OBJC_TYPE (RTCRtpReceiver) +() + + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::RtpReceiverInterface> nativeRtpReceiver; + +/** Initialize an RTCRtpReceiver with a native RtpReceiverInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpReceiver:(rtc::scoped_refptr<webrtc::RtpReceiverInterface>)nativeRtpReceiver + NS_DESIGNATED_INITIALIZER; + ++ (RTCRtpMediaType)mediaTypeForNativeMediaType:(cricket::MediaType)nativeMediaType; + ++ (cricket::MediaType)nativeMediaTypeForMediaType:(RTCRtpMediaType)mediaType; + ++ (NSString *)stringForMediaType:(RTCRtpMediaType)mediaType; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.h new file mode 100644 index 0000000000..1e407fd71b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.h @@ -0,0 +1,86 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCMediaStreamTrack.h" +#import "RTCRtpParameters.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the media type of the RtpReceiver. */ +typedef NS_ENUM(NSInteger, RTCRtpMediaType) { + RTCRtpMediaTypeAudio, + RTCRtpMediaTypeVideo, + RTCRtpMediaTypeData, + RTCRtpMediaTypeUnsupported, +}; + +@class RTC_OBJC_TYPE(RTCRtpReceiver); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCRtpReceiverDelegate)<NSObject> + + /** Called when the first RTP packet is received. + * + * Note: Currently if there are multiple RtpReceivers of the same media type, + * they will all call OnFirstPacketReceived at once. + * + * For example, if we create three audio receivers, A/B/C, they will listen to + * the same signal from the underneath network layer. Whenever the first audio packet + * is received, the underneath signal will be fired. All the receivers A/B/C will be + * notified and the callback of the receiver's delegate will be called. + * + * The process is the same for video receivers. + */ + - (void)rtpReceiver + : (RTC_OBJC_TYPE(RTCRtpReceiver) *)rtpReceiver didReceiveFirstPacketForMediaType + : (RTCRtpMediaType)mediaType; + +@end + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCRtpReceiver)<NSObject> + + /** A unique identifier for this receiver. */ + @property(nonatomic, readonly) NSString *receiverId; + +/** The currently active RTCRtpParameters, as defined in + * https://www.w3.org/TR/webrtc/#idl-def-RTCRtpParameters. + * + * The WebRTC specification only defines RTCRtpParameters in terms of senders, + * but this API also applies them to receivers, similar to ORTC: + * http://ortc.org/wp-content/uploads/2016/03/ortc.html#rtcrtpparameters*. + */ +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCRtpParameters) * parameters; + +/** The RTCMediaStreamTrack associated with the receiver. + * Note: reading this property returns a new instance of + * RTCMediaStreamTrack. Use isEqual: instead of == to compare + * RTCMediaStreamTrack instances. + */ +@property(nonatomic, readonly, nullable) RTC_OBJC_TYPE(RTCMediaStreamTrack) * track; + +/** The delegate for this RtpReceiver. */ +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCRtpReceiverDelegate)> delegate; + +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpReceiver) : NSObject <RTC_OBJC_TYPE(RTCRtpReceiver)> + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.mm new file mode 100644 index 0000000000..60af86ac1b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpReceiver.mm @@ -0,0 +1,159 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpReceiver+Private.h" + +#import "RTCMediaStreamTrack+Private.h" +#import "RTCRtpParameters+Private.h" +#import "RTCRtpReceiver+Native.h" +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "api/media_stream_interface.h" + +namespace webrtc { + +RtpReceiverDelegateAdapter::RtpReceiverDelegateAdapter(RTC_OBJC_TYPE(RTCRtpReceiver) * receiver) { + RTC_CHECK(receiver); + receiver_ = receiver; +} + +void RtpReceiverDelegateAdapter::OnFirstPacketReceived( + cricket::MediaType media_type) { + RTCRtpMediaType packet_media_type = + [RTC_OBJC_TYPE(RTCRtpReceiver) mediaTypeForNativeMediaType:media_type]; + RTC_OBJC_TYPE(RTCRtpReceiver) *receiver = receiver_; + [receiver.delegate rtpReceiver:receiver didReceiveFirstPacketForMediaType:packet_media_type]; +} + +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCRtpReceiver) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::scoped_refptr<webrtc::RtpReceiverInterface> _nativeRtpReceiver; + std::unique_ptr<webrtc::RtpReceiverDelegateAdapter> _observer; +} + +@synthesize delegate = _delegate; + +- (NSString *)receiverId { + return [NSString stringForStdString:_nativeRtpReceiver->id()]; +} + +- (RTC_OBJC_TYPE(RTCRtpParameters) *)parameters { + return [[RTC_OBJC_TYPE(RTCRtpParameters) alloc] + initWithNativeParameters:_nativeRtpReceiver->GetParameters()]; +} + +- (nullable RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track { + rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> nativeTrack( + _nativeRtpReceiver->track()); + if (nativeTrack) { + return [RTC_OBJC_TYPE(RTCMediaStreamTrack) mediaTrackForNativeTrack:nativeTrack + factory:_factory]; + } + return nil; +} + +- (NSString *)description { + return [NSString + stringWithFormat:@"RTC_OBJC_TYPE(RTCRtpReceiver) {\n receiverId: %@\n}", self.receiverId]; +} + +- (void)dealloc { + if (_nativeRtpReceiver) { + _nativeRtpReceiver->SetObserver(nullptr); + } +} + +- (BOOL)isEqual:(id)object { + if (self == object) { + return YES; + } + if (object == nil) { + return NO; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + RTC_OBJC_TYPE(RTCRtpReceiver) *receiver = (RTC_OBJC_TYPE(RTCRtpReceiver) *)object; + return _nativeRtpReceiver == receiver.nativeRtpReceiver; +} + +- (NSUInteger)hash { + return (NSUInteger)_nativeRtpReceiver.get(); +} + +#pragma mark - Native + +- (void)setFrameDecryptor:(rtc::scoped_refptr<webrtc::FrameDecryptorInterface>)frameDecryptor { + _nativeRtpReceiver->SetFrameDecryptor(frameDecryptor); +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::RtpReceiverInterface>)nativeRtpReceiver { + return _nativeRtpReceiver; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpReceiver: + (rtc::scoped_refptr<webrtc::RtpReceiverInterface>)nativeRtpReceiver { + if (self = [super init]) { + _factory = factory; + _nativeRtpReceiver = nativeRtpReceiver; + RTCLogInfo(@"RTC_OBJC_TYPE(RTCRtpReceiver)(%p): created receiver: %@", self, self.description); + _observer.reset(new webrtc::RtpReceiverDelegateAdapter(self)); + _nativeRtpReceiver->SetObserver(_observer.get()); + } + return self; +} + ++ (RTCRtpMediaType)mediaTypeForNativeMediaType: + (cricket::MediaType)nativeMediaType { + switch (nativeMediaType) { + case cricket::MEDIA_TYPE_AUDIO: + return RTCRtpMediaTypeAudio; + case cricket::MEDIA_TYPE_VIDEO: + return RTCRtpMediaTypeVideo; + case cricket::MEDIA_TYPE_DATA: + return RTCRtpMediaTypeData; + case cricket::MEDIA_TYPE_UNSUPPORTED: + return RTCRtpMediaTypeUnsupported; + } +} + ++ (cricket::MediaType)nativeMediaTypeForMediaType:(RTCRtpMediaType)mediaType { + switch (mediaType) { + case RTCRtpMediaTypeAudio: + return cricket::MEDIA_TYPE_AUDIO; + case RTCRtpMediaTypeVideo: + return cricket::MEDIA_TYPE_VIDEO; + case RTCRtpMediaTypeData: + return cricket::MEDIA_TYPE_DATA; + case RTCRtpMediaTypeUnsupported: + return cricket::MEDIA_TYPE_UNSUPPORTED; + } +} + ++ (NSString *)stringForMediaType:(RTCRtpMediaType)mediaType { + switch (mediaType) { + case RTCRtpMediaTypeAudio: + return @"AUDIO"; + case RTCRtpMediaTypeVideo: + return @"VIDEO"; + case RTCRtpMediaTypeData: + return @"DATA"; + case RTCRtpMediaTypeUnsupported: + return @"UNSUPPORTED"; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Native.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Native.h new file mode 100644 index 0000000000..249d5c5e09 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Native.h @@ -0,0 +1,33 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpSender.h" + +#include "api/crypto/frame_encryptor_interface.h" +#include "api/scoped_refptr.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class extension exposes methods that work directly with injectable C++ components. + */ +@interface RTC_OBJC_TYPE (RTCRtpSender) +() + + /** Sets a defined frame encryptor that will encrypt the entire frame + * before it is sent across the network. This will encrypt the entire frame + * using the user provided encryption mechanism regardless of whether SRTP is + * enabled or not. + */ + - (void)setFrameEncryptor : (rtc::scoped_refptr<webrtc::FrameEncryptorInterface>)frameEncryptor; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Private.h new file mode 100644 index 0000000000..6fdb42bb22 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender+Private.h @@ -0,0 +1,31 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpSender.h" + +#include "api/rtp_sender_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +@interface RTC_OBJC_TYPE (RTCRtpSender) +() + + @property(nonatomic, readonly) rtc::scoped_refptr<webrtc::RtpSenderInterface> nativeRtpSender; + +/** Initialize an RTCRtpSender with a native RtpSenderInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpSender:(rtc::scoped_refptr<webrtc::RtpSenderInterface>)nativeRtpSender + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.h new file mode 100644 index 0000000000..41bb083d2e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.h @@ -0,0 +1,54 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCDtmfSender.h" +#import "RTCMacros.h" +#import "RTCMediaStreamTrack.h" +#import "RTCRtpParameters.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCRtpSender)<NSObject> + + /** A unique identifier for this sender. */ + @property(nonatomic, readonly) NSString *senderId; + +/** The currently active RTCRtpParameters, as defined in + * https://www.w3.org/TR/webrtc/#idl-def-RTCRtpParameters. + */ +@property(nonatomic, copy) RTC_OBJC_TYPE(RTCRtpParameters) * parameters; + +/** The RTCMediaStreamTrack associated with the sender. + * Note: reading this property returns a new instance of + * RTCMediaStreamTrack. Use isEqual: instead of == to compare + * RTCMediaStreamTrack instances. + */ +@property(nonatomic, copy, nullable) RTC_OBJC_TYPE(RTCMediaStreamTrack) * track; + +/** IDs of streams associated with the RTP sender */ +@property(nonatomic, copy) NSArray<NSString *> *streamIds; + +/** The RTCDtmfSender accociated with the RTP sender. */ +@property(nonatomic, readonly, nullable) id<RTC_OBJC_TYPE(RTCDtmfSender)> dtmfSender; + +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpSender) : NSObject <RTC_OBJC_TYPE(RTCRtpSender)> + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.mm new file mode 100644 index 0000000000..4fadb30f49 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpSender.mm @@ -0,0 +1,132 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpSender+Private.h" + +#import "RTCDtmfSender+Private.h" +#import "RTCMediaStreamTrack+Private.h" +#import "RTCRtpParameters+Private.h" +#import "RTCRtpSender+Native.h" +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "api/media_stream_interface.h" + +@implementation RTC_OBJC_TYPE (RTCRtpSender) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::scoped_refptr<webrtc::RtpSenderInterface> _nativeRtpSender; +} + +@synthesize dtmfSender = _dtmfSender; + +- (NSString *)senderId { + return [NSString stringForStdString:_nativeRtpSender->id()]; +} + +- (RTC_OBJC_TYPE(RTCRtpParameters) *)parameters { + return [[RTC_OBJC_TYPE(RTCRtpParameters) alloc] + initWithNativeParameters:_nativeRtpSender->GetParameters()]; +} + +- (void)setParameters:(RTC_OBJC_TYPE(RTCRtpParameters) *)parameters { + if (!_nativeRtpSender->SetParameters(parameters.nativeParameters).ok()) { + RTCLogError(@"RTC_OBJC_TYPE(RTCRtpSender)(%p): Failed to set parameters: %@", self, parameters); + } +} + +- (RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track { + rtc::scoped_refptr<webrtc::MediaStreamTrackInterface> nativeTrack( + _nativeRtpSender->track()); + if (nativeTrack) { + return [RTC_OBJC_TYPE(RTCMediaStreamTrack) mediaTrackForNativeTrack:nativeTrack + factory:_factory]; + } + return nil; +} + +- (void)setTrack:(RTC_OBJC_TYPE(RTCMediaStreamTrack) *)track { + if (!_nativeRtpSender->SetTrack(track.nativeTrack.get())) { + RTCLogError(@"RTC_OBJC_TYPE(RTCRtpSender)(%p): Failed to set track %@", self, track); + } +} + +- (NSArray<NSString *> *)streamIds { + std::vector<std::string> nativeStreamIds = _nativeRtpSender->stream_ids(); + NSMutableArray *streamIds = [NSMutableArray arrayWithCapacity:nativeStreamIds.size()]; + for (const auto &s : nativeStreamIds) { + [streamIds addObject:[NSString stringForStdString:s]]; + } + return streamIds; +} + +- (void)setStreamIds:(NSArray<NSString *> *)streamIds { + std::vector<std::string> nativeStreamIds; + for (NSString *streamId in streamIds) { + nativeStreamIds.push_back([streamId UTF8String]); + } + _nativeRtpSender->SetStreams(nativeStreamIds); +} + +- (NSString *)description { + return [NSString + stringWithFormat:@"RTC_OBJC_TYPE(RTCRtpSender) {\n senderId: %@\n}", self.senderId]; +} + +- (BOOL)isEqual:(id)object { + if (self == object) { + return YES; + } + if (object == nil) { + return NO; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + RTC_OBJC_TYPE(RTCRtpSender) *sender = (RTC_OBJC_TYPE(RTCRtpSender) *)object; + return _nativeRtpSender == sender.nativeRtpSender; +} + +- (NSUInteger)hash { + return (NSUInteger)_nativeRtpSender.get(); +} + +#pragma mark - Native + +- (void)setFrameEncryptor:(rtc::scoped_refptr<webrtc::FrameEncryptorInterface>)frameEncryptor { + _nativeRtpSender->SetFrameEncryptor(frameEncryptor); +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::RtpSenderInterface>)nativeRtpSender { + return _nativeRtpSender; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpSender:(rtc::scoped_refptr<webrtc::RtpSenderInterface>)nativeRtpSender { + NSParameterAssert(factory); + NSParameterAssert(nativeRtpSender); + if (self = [super init]) { + _factory = factory; + _nativeRtpSender = nativeRtpSender; + if (_nativeRtpSender->media_type() == cricket::MEDIA_TYPE_AUDIO) { + rtc::scoped_refptr<webrtc::DtmfSenderInterface> nativeDtmfSender( + _nativeRtpSender->GetDtmfSender()); + if (nativeDtmfSender) { + _dtmfSender = + [[RTC_OBJC_TYPE(RTCDtmfSender) alloc] initWithNativeDtmfSender:nativeDtmfSender]; + } + } + RTCLogInfo(@"RTC_OBJC_TYPE(RTCRtpSender)(%p): created sender: %@", self, self.description); + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver+Private.h new file mode 100644 index 0000000000..868cbd80fe --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver+Private.h @@ -0,0 +1,46 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpTransceiver.h" + +#include "api/rtp_transceiver_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); + +@interface RTC_OBJC_TYPE (RTCRtpTransceiverInit) +() + + @property(nonatomic, readonly) webrtc::RtpTransceiverInit nativeInit; + +@end + +@interface RTC_OBJC_TYPE (RTCRtpTransceiver) +() + + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::RtpTransceiverInterface> nativeRtpTransceiver; + +/** Initialize an RTCRtpTransceiver with a native RtpTransceiverInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpTransceiver: + (rtc::scoped_refptr<webrtc::RtpTransceiverInterface>)nativeRtpTransceiver + NS_DESIGNATED_INITIALIZER; + ++ (webrtc::RtpTransceiverDirection)nativeRtpTransceiverDirectionFromDirection: + (RTCRtpTransceiverDirection)direction; + ++ (RTCRtpTransceiverDirection)rtpTransceiverDirectionFromNativeDirection: + (webrtc::RtpTransceiverDirection)nativeDirection; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.h new file mode 100644 index 0000000000..fd59013639 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.h @@ -0,0 +1,137 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCRtpReceiver.h" +#import "RTCRtpSender.h" + +NS_ASSUME_NONNULL_BEGIN + +extern NSString *const kRTCRtpTransceiverErrorDomain; + +/** https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiverdirection */ +typedef NS_ENUM(NSInteger, RTCRtpTransceiverDirection) { + RTCRtpTransceiverDirectionSendRecv, + RTCRtpTransceiverDirectionSendOnly, + RTCRtpTransceiverDirectionRecvOnly, + RTCRtpTransceiverDirectionInactive, + RTCRtpTransceiverDirectionStopped +}; + +/** Structure for initializing an RTCRtpTransceiver in a call to + * RTCPeerConnection.addTransceiver. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiverinit + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpTransceiverInit) : NSObject + +/** Direction of the RTCRtpTransceiver. See RTCRtpTransceiver.direction. */ +@property(nonatomic) RTCRtpTransceiverDirection direction; + +/** The added RTCRtpTransceiver will be added to these streams. */ +@property(nonatomic) NSArray<NSString *> *streamIds; + +/** TODO(bugs.webrtc.org/7600): Not implemented. */ +@property(nonatomic) NSArray<RTC_OBJC_TYPE(RTCRtpEncodingParameters) *> *sendEncodings; + +@end + +@class RTC_OBJC_TYPE(RTCRtpTransceiver); + +/** The RTCRtpTransceiver maps to the RTCRtpTransceiver defined by the + * WebRTC specification. A transceiver represents a combination of an RTCRtpSender + * and an RTCRtpReceiver that share a common mid. As defined in JSEP, an + * RTCRtpTransceiver is said to be associated with a media description if its + * mid property is non-nil; otherwise, it is said to be disassociated. + * JSEP: https://tools.ietf.org/html/draft-ietf-rtcweb-jsep-24 + * + * Note that RTCRtpTransceivers are only supported when using + * RTCPeerConnection with Unified Plan SDP. + * + * WebRTC specification for RTCRtpTransceiver, the JavaScript analog: + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCRtpTransceiver)<NSObject> + + /** Media type of the transceiver. The sender and receiver will also have this + * type. + */ + @property(nonatomic, readonly) RTCRtpMediaType mediaType; + +/** The mid attribute is the mid negotiated and present in the local and + * remote descriptions. Before negotiation is complete, the mid value may be + * nil. After rollbacks, the value may change from a non-nil value to nil. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-mid + */ +@property(nonatomic, readonly) NSString *mid; + +/** The sender attribute exposes the RTCRtpSender corresponding to the RTP + * media that may be sent with the transceiver's mid. The sender is always + * present, regardless of the direction of media. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-sender + */ +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCRtpSender) * sender; + +/** The receiver attribute exposes the RTCRtpReceiver corresponding to the RTP + * media that may be received with the transceiver's mid. The receiver is + * always present, regardless of the direction of media. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-receiver + */ +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCRtpReceiver) * receiver; + +/** The isStopped attribute indicates that the sender of this transceiver will + * no longer send, and that the receiver will no longer receive. It is true if + * either stop has been called or if setting the local or remote description + * has caused the RTCRtpTransceiver to be stopped. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-stopped + */ +@property(nonatomic, readonly) BOOL isStopped; + +/** The direction attribute indicates the preferred direction of this + * transceiver, which will be used in calls to createOffer and createAnswer. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-direction + */ +@property(nonatomic, readonly) RTCRtpTransceiverDirection direction; + +/** The currentDirection attribute indicates the current direction negotiated + * for this transceiver. If this transceiver has never been represented in an + * offer/answer exchange, or if the transceiver is stopped, the value is not + * present and this method returns NO. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-currentdirection + */ +- (BOOL)currentDirection:(RTCRtpTransceiverDirection *)currentDirectionOut; + +/** The stop method irreversibly stops the RTCRtpTransceiver. The sender of + * this transceiver will no longer send, the receiver will no longer receive. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-stop + */ +- (void)stopInternal; + +/** An update of directionality does not take effect immediately. Instead, + * future calls to createOffer and createAnswer mark the corresponding media + * descriptions as sendrecv, sendonly, recvonly, or inactive. + * https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiver-direction + */ +- (void)setDirection:(RTCRtpTransceiverDirection)direction error:(NSError **)error; + +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCRtpTransceiver) : NSObject <RTC_OBJC_TYPE(RTCRtpTransceiver)> + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm new file mode 100644 index 0000000000..ae1cf79864 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCRtpTransceiver.mm @@ -0,0 +1,190 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCRtpTransceiver+Private.h" + +#import "RTCRtpEncodingParameters+Private.h" +#import "RTCRtpParameters+Private.h" +#import "RTCRtpReceiver+Private.h" +#import "RTCRtpSender+Private.h" +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +NSString *const kRTCRtpTransceiverErrorDomain = @"org.webrtc.RTCRtpTranceiver"; + +@implementation RTC_OBJC_TYPE (RTCRtpTransceiverInit) + +@synthesize direction = _direction; +@synthesize streamIds = _streamIds; +@synthesize sendEncodings = _sendEncodings; + +- (instancetype)init { + if (self = [super init]) { + _direction = RTCRtpTransceiverDirectionSendRecv; + } + return self; +} + +- (webrtc::RtpTransceiverInit)nativeInit { + webrtc::RtpTransceiverInit init; + init.direction = + [RTC_OBJC_TYPE(RTCRtpTransceiver) nativeRtpTransceiverDirectionFromDirection:_direction]; + for (NSString *streamId in _streamIds) { + init.stream_ids.push_back([streamId UTF8String]); + } + for (RTC_OBJC_TYPE(RTCRtpEncodingParameters) * sendEncoding in _sendEncodings) { + init.send_encodings.push_back(sendEncoding.nativeParameters); + } + return init; +} + +@end + +@implementation RTC_OBJC_TYPE (RTCRtpTransceiver) { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * _factory; + rtc::scoped_refptr<webrtc::RtpTransceiverInterface> _nativeRtpTransceiver; +} + +- (RTCRtpMediaType)mediaType { + return [RTC_OBJC_TYPE(RTCRtpReceiver) + mediaTypeForNativeMediaType:_nativeRtpTransceiver->media_type()]; +} + +- (NSString *)mid { + if (_nativeRtpTransceiver->mid()) { + return [NSString stringForStdString:*_nativeRtpTransceiver->mid()]; + } else { + return nil; + } +} + +@synthesize sender = _sender; +@synthesize receiver = _receiver; + +- (BOOL)isStopped { + return _nativeRtpTransceiver->stopped(); +} + +- (RTCRtpTransceiverDirection)direction { + return [RTC_OBJC_TYPE(RTCRtpTransceiver) + rtpTransceiverDirectionFromNativeDirection:_nativeRtpTransceiver->direction()]; +} + +- (void)setDirection:(RTCRtpTransceiverDirection)direction error:(NSError **)error { + webrtc::RTCError nativeError = _nativeRtpTransceiver->SetDirectionWithError( + [RTC_OBJC_TYPE(RTCRtpTransceiver) nativeRtpTransceiverDirectionFromDirection:direction]); + + if (!nativeError.ok() && error) { + *error = [NSError errorWithDomain:kRTCRtpTransceiverErrorDomain + code:static_cast<int>(nativeError.type()) + userInfo:@{ + @"message" : [NSString stringWithCString:nativeError.message() + encoding:NSUTF8StringEncoding] + }]; + } +} + +- (BOOL)currentDirection:(RTCRtpTransceiverDirection *)currentDirectionOut { + if (_nativeRtpTransceiver->current_direction()) { + *currentDirectionOut = [RTC_OBJC_TYPE(RTCRtpTransceiver) + rtpTransceiverDirectionFromNativeDirection:*_nativeRtpTransceiver->current_direction()]; + return YES; + } else { + return NO; + } +} + +- (void)stopInternal { + _nativeRtpTransceiver->StopInternal(); +} + +- (NSString *)description { + return [NSString + stringWithFormat:@"RTC_OBJC_TYPE(RTCRtpTransceiver) {\n sender: %@\n receiver: %@\n}", + _sender, + _receiver]; +} + +- (BOOL)isEqual:(id)object { + if (self == object) { + return YES; + } + if (object == nil) { + return NO; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + RTC_OBJC_TYPE(RTCRtpTransceiver) *transceiver = (RTC_OBJC_TYPE(RTCRtpTransceiver) *)object; + return _nativeRtpTransceiver == transceiver.nativeRtpTransceiver; +} + +- (NSUInteger)hash { + return (NSUInteger)_nativeRtpTransceiver.get(); +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::RtpTransceiverInterface>)nativeRtpTransceiver { + return _nativeRtpTransceiver; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeRtpTransceiver: + (rtc::scoped_refptr<webrtc::RtpTransceiverInterface>)nativeRtpTransceiver { + NSParameterAssert(factory); + NSParameterAssert(nativeRtpTransceiver); + if (self = [super init]) { + _factory = factory; + _nativeRtpTransceiver = nativeRtpTransceiver; + _sender = [[RTC_OBJC_TYPE(RTCRtpSender) alloc] initWithFactory:_factory + nativeRtpSender:nativeRtpTransceiver->sender()]; + _receiver = + [[RTC_OBJC_TYPE(RTCRtpReceiver) alloc] initWithFactory:_factory + nativeRtpReceiver:nativeRtpTransceiver->receiver()]; + RTCLogInfo( + @"RTC_OBJC_TYPE(RTCRtpTransceiver)(%p): created transceiver: %@", self, self.description); + } + return self; +} + ++ (webrtc::RtpTransceiverDirection)nativeRtpTransceiverDirectionFromDirection: + (RTCRtpTransceiverDirection)direction { + switch (direction) { + case RTCRtpTransceiverDirectionSendRecv: + return webrtc::RtpTransceiverDirection::kSendRecv; + case RTCRtpTransceiverDirectionSendOnly: + return webrtc::RtpTransceiverDirection::kSendOnly; + case RTCRtpTransceiverDirectionRecvOnly: + return webrtc::RtpTransceiverDirection::kRecvOnly; + case RTCRtpTransceiverDirectionInactive: + return webrtc::RtpTransceiverDirection::kInactive; + case RTCRtpTransceiverDirectionStopped: + return webrtc::RtpTransceiverDirection::kStopped; + } +} + ++ (RTCRtpTransceiverDirection)rtpTransceiverDirectionFromNativeDirection: + (webrtc::RtpTransceiverDirection)nativeDirection { + switch (nativeDirection) { + case webrtc::RtpTransceiverDirection::kSendRecv: + return RTCRtpTransceiverDirectionSendRecv; + case webrtc::RtpTransceiverDirection::kSendOnly: + return RTCRtpTransceiverDirectionSendOnly; + case webrtc::RtpTransceiverDirection::kRecvOnly: + return RTCRtpTransceiverDirectionRecvOnly; + case webrtc::RtpTransceiverDirection::kInactive: + return RTCRtpTransceiverDirectionInactive; + case webrtc::RtpTransceiverDirection::kStopped: + return RTCRtpTransceiverDirectionStopped; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.h new file mode 100644 index 0000000000..f68bc5e9e3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.h @@ -0,0 +1,20 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +/** + * Initialize and clean up the SSL library. Failure is fatal. These call the + * corresponding functions in webrtc/rtc_base/ssladapter.h. + */ +RTC_EXTERN BOOL RTCInitializeSSL(void); +RTC_EXTERN BOOL RTCCleanupSSL(void); diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.mm new file mode 100644 index 0000000000..430249577b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSSLAdapter.mm @@ -0,0 +1,26 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSSLAdapter.h" + +#include "rtc_base/checks.h" +#include "rtc_base/ssl_adapter.h" + +BOOL RTCInitializeSSL(void) { + BOOL initialized = rtc::InitializeSSL(); + RTC_DCHECK(initialized); + return initialized; +} + +BOOL RTCCleanupSSL(void) { + BOOL cleanedUp = rtc::CleanupSSL(); + RTC_DCHECK(cleanedUp); + return cleanedUp; +} diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription+Private.h new file mode 100644 index 0000000000..d01c04b0b5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription+Private.h @@ -0,0 +1,42 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSessionDescription.h" + +#include "api/jsep.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCSessionDescription) +() + + /** + * The native SessionDescriptionInterface representation of this + * RTCSessionDescription object. This is needed to pass to the underlying C++ + * APIs. + */ + @property(nonatomic, + readonly) std::unique_ptr<webrtc::SessionDescriptionInterface> nativeDescription; + +/** + * Initialize an RTCSessionDescription from a native + * SessionDescriptionInterface. No ownership is taken of the native session + * description. + */ +- (instancetype)initWithNativeDescription: + (const webrtc::SessionDescriptionInterface *)nativeDescription; + ++ (std::string)stdStringForType:(RTCSdpType)type; + ++ (RTCSdpType)typeForStdString:(const std::string &)string; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.h new file mode 100644 index 0000000000..8a9479d5cf --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.h @@ -0,0 +1,48 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +/** + * Represents the session description type. This exposes the same types that are + * in C++, which doesn't include the rollback type that is in the W3C spec. + */ +typedef NS_ENUM(NSInteger, RTCSdpType) { + RTCSdpTypeOffer, + RTCSdpTypePrAnswer, + RTCSdpTypeAnswer, + RTCSdpTypeRollback, +}; + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCSessionDescription) : NSObject + +/** The type of session description. */ +@property(nonatomic, readonly) RTCSdpType type; + +/** The SDP string representation of this session description. */ +@property(nonatomic, readonly) NSString *sdp; + +- (instancetype)init NS_UNAVAILABLE; + +/** Initialize a session description with a type and SDP string. */ +- (instancetype)initWithType:(RTCSdpType)type sdp:(NSString *)sdp NS_DESIGNATED_INITIALIZER; + ++ (NSString *)stringForType:(RTCSdpType)type; + ++ (RTCSdpType)typeForString:(NSString *)string; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.mm new file mode 100644 index 0000000000..539c90b14c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCSessionDescription.mm @@ -0,0 +1,103 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCSessionDescription+Private.h" + +#import "base/RTCLogging.h" +#import "helpers/NSString+StdString.h" + +#include "rtc_base/checks.h" + +@implementation RTC_OBJC_TYPE (RTCSessionDescription) + +@synthesize type = _type; +@synthesize sdp = _sdp; + ++ (NSString *)stringForType:(RTCSdpType)type { + std::string string = [[self class] stdStringForType:type]; + return [NSString stringForStdString:string]; +} + ++ (RTCSdpType)typeForString:(NSString *)string { + std::string typeString = string.stdString; + return [[self class] typeForStdString:typeString]; +} + +- (instancetype)initWithType:(RTCSdpType)type sdp:(NSString *)sdp { + if (self = [super init]) { + _type = type; + _sdp = [sdp copy]; + } + return self; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCSessionDescription):\n%@\n%@", + [[self class] stringForType:_type], + _sdp]; +} + +#pragma mark - Private + +- (std::unique_ptr<webrtc::SessionDescriptionInterface>)nativeDescription { + webrtc::SdpParseError error; + + std::unique_ptr<webrtc::SessionDescriptionInterface> description(webrtc::CreateSessionDescription( + [[self class] stdStringForType:_type], _sdp.stdString, &error)); + + if (!description) { + RTCLogError(@"Failed to create session description: %s\nline: %s", + error.description.c_str(), + error.line.c_str()); + } + + return description; +} + +- (instancetype)initWithNativeDescription: + (const webrtc::SessionDescriptionInterface *)nativeDescription { + NSParameterAssert(nativeDescription); + std::string sdp; + nativeDescription->ToString(&sdp); + RTCSdpType type = [[self class] typeForStdString:nativeDescription->type()]; + + return [self initWithType:type + sdp:[NSString stringForStdString:sdp]]; +} + ++ (std::string)stdStringForType:(RTCSdpType)type { + switch (type) { + case RTCSdpTypeOffer: + return webrtc::SessionDescriptionInterface::kOffer; + case RTCSdpTypePrAnswer: + return webrtc::SessionDescriptionInterface::kPrAnswer; + case RTCSdpTypeAnswer: + return webrtc::SessionDescriptionInterface::kAnswer; + case RTCSdpTypeRollback: + return webrtc::SessionDescriptionInterface::kRollback; + } +} + ++ (RTCSdpType)typeForStdString:(const std::string &)string { + if (string == webrtc::SessionDescriptionInterface::kOffer) { + return RTCSdpTypeOffer; + } else if (string == webrtc::SessionDescriptionInterface::kPrAnswer) { + return RTCSdpTypePrAnswer; + } else if (string == webrtc::SessionDescriptionInterface::kAnswer) { + return RTCSdpTypeAnswer; + } else if (string == webrtc::SessionDescriptionInterface::kRollback) { + return RTCSdpTypeRollback; + } else { + RTC_DCHECK_NOTREACHED(); + return RTCSdpTypeOffer; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport+Private.h new file mode 100644 index 0000000000..e91302a207 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport+Private.h @@ -0,0 +1,20 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCStatisticsReport.h" + +#include "api/stats/rtc_stats_report.h" + +@interface RTC_OBJC_TYPE (RTCStatisticsReport) +(Private) + + - (instancetype)initWithReport : (const webrtc::RTCStatsReport &)report; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.h new file mode 100644 index 0000000000..06dbf48d88 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.h @@ -0,0 +1,55 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +@class RTC_OBJC_TYPE(RTCStatistics); + +NS_ASSUME_NONNULL_BEGIN + +/** A statistics report. Encapsulates a number of RTCStatistics objects. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCStatisticsReport) : NSObject + +/** The timestamp of the report in microseconds since 1970-01-01T00:00:00Z. */ +@property(nonatomic, readonly) CFTimeInterval timestamp_us; + +/** RTCStatistics objects by id. */ +@property(nonatomic, readonly) NSDictionary<NSString *, RTC_OBJC_TYPE(RTCStatistics) *> *statistics; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +/** A part of a report (a subreport) covering a certain area. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCStatistics) : NSObject + +/** The id of this subreport, e.g. "RTCMediaStreamTrack_receiver_2". */ +@property(nonatomic, readonly) NSString *id; + +/** The timestamp of the subreport in microseconds since 1970-01-01T00:00:00Z. */ +@property(nonatomic, readonly) CFTimeInterval timestamp_us; + +/** The type of the subreport, e.g. "track", "codec". */ +@property(nonatomic, readonly) NSString *type; + +/** The keys and values of the subreport, e.g. "totalFramesDuration = 5.551". + The values are either NSNumbers or NSStrings or NSArrays encapsulating NSNumbers + or NSStrings, or NSDictionary of NSString keys to NSNumber values. */ +@property(nonatomic, readonly) NSDictionary<NSString *, NSObject *> *values; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.mm new file mode 100644 index 0000000000..bfe2424553 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCStatisticsReport.mm @@ -0,0 +1,193 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCStatisticsReport+Private.h" + +#include "helpers/NSString+StdString.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +/** Converts a single value to a suitable NSNumber, NSString or NSArray containing NSNumbers + or NSStrings, or NSDictionary of NSString keys to NSNumber values.*/ +NSObject *ValueFromStatsMember(const RTCStatsMemberInterface *member) { + if (member->is_defined()) { + switch (member->type()) { + case RTCStatsMemberInterface::kBool: + return [NSNumber numberWithBool:*member->cast_to<RTCStatsMember<bool>>()]; + case RTCStatsMemberInterface::kInt32: + return [NSNumber numberWithInt:*member->cast_to<RTCStatsMember<int32_t>>()]; + case RTCStatsMemberInterface::kUint32: + return [NSNumber numberWithUnsignedInt:*member->cast_to<RTCStatsMember<uint32_t>>()]; + case RTCStatsMemberInterface::kInt64: + return [NSNumber numberWithLong:*member->cast_to<RTCStatsMember<int64_t>>()]; + case RTCStatsMemberInterface::kUint64: + return [NSNumber numberWithUnsignedLong:*member->cast_to<RTCStatsMember<uint64_t>>()]; + case RTCStatsMemberInterface::kDouble: + return [NSNumber numberWithDouble:*member->cast_to<RTCStatsMember<double>>()]; + case RTCStatsMemberInterface::kString: + return [NSString stringForStdString:*member->cast_to<RTCStatsMember<std::string>>()]; + case RTCStatsMemberInterface::kSequenceBool: { + std::vector<bool> sequence = *member->cast_to<RTCStatsMember<std::vector<bool>>>(); + NSMutableArray *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (auto item : sequence) { + [array addObject:[NSNumber numberWithBool:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceInt32: { + std::vector<int32_t> sequence = *member->cast_to<RTCStatsMember<std::vector<int32_t>>>(); + NSMutableArray<NSNumber *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSNumber numberWithInt:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceUint32: { + std::vector<uint32_t> sequence = *member->cast_to<RTCStatsMember<std::vector<uint32_t>>>(); + NSMutableArray<NSNumber *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSNumber numberWithUnsignedInt:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceInt64: { + std::vector<int64_t> sequence = *member->cast_to<RTCStatsMember<std::vector<int64_t>>>(); + NSMutableArray<NSNumber *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSNumber numberWithLong:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceUint64: { + std::vector<uint64_t> sequence = *member->cast_to<RTCStatsMember<std::vector<uint64_t>>>(); + NSMutableArray<NSNumber *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSNumber numberWithUnsignedLong:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceDouble: { + std::vector<double> sequence = *member->cast_to<RTCStatsMember<std::vector<double>>>(); + NSMutableArray<NSNumber *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSNumber numberWithDouble:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kSequenceString: { + std::vector<std::string> sequence = + *member->cast_to<RTCStatsMember<std::vector<std::string>>>(); + NSMutableArray<NSString *> *array = [NSMutableArray arrayWithCapacity:sequence.size()]; + for (const auto &item : sequence) { + [array addObject:[NSString stringForStdString:item]]; + } + return [array copy]; + } + case RTCStatsMemberInterface::kMapStringUint64: { + std::map<std::string, uint64_t> map = + *member->cast_to<RTCStatsMember<std::map<std::string, uint64_t>>>(); + NSMutableDictionary<NSString *, NSNumber *> *dictionary = + [NSMutableDictionary dictionaryWithCapacity:map.size()]; + for (const auto &item : map) { + dictionary[[NSString stringForStdString:item.first]] = @(item.second); + } + return [dictionary copy]; + } + case RTCStatsMemberInterface::kMapStringDouble: { + std::map<std::string, double> map = + *member->cast_to<RTCStatsMember<std::map<std::string, double>>>(); + NSMutableDictionary<NSString *, NSNumber *> *dictionary = + [NSMutableDictionary dictionaryWithCapacity:map.size()]; + for (const auto &item : map) { + dictionary[[NSString stringForStdString:item.first]] = @(item.second); + } + return [dictionary copy]; + } + default: + RTC_DCHECK_NOTREACHED(); + } + } + + return nil; +} +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCStatistics) + +@synthesize id = _id; +@synthesize timestamp_us = _timestamp_us; +@synthesize type = _type; +@synthesize values = _values; + +- (instancetype)initWithStatistics:(const webrtc::RTCStats &)statistics { + if (self = [super init]) { + _id = [NSString stringForStdString:statistics.id()]; + _timestamp_us = statistics.timestamp().us(); + _type = [NSString stringWithCString:statistics.type() encoding:NSUTF8StringEncoding]; + + NSMutableDictionary<NSString *, NSObject *> *values = [NSMutableDictionary dictionary]; + for (const webrtc::RTCStatsMemberInterface *member : statistics.Members()) { + NSObject *value = ValueFromStatsMember(member); + if (value) { + NSString *name = [NSString stringWithCString:member->name() encoding:NSUTF8StringEncoding]; + RTC_DCHECK(name.length > 0); + RTC_DCHECK(!values[name]); + values[name] = value; + } + } + _values = [values copy]; + } + + return self; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"id = %@, type = %@, timestamp = %.0f, values = %@", + self.id, + self.type, + self.timestamp_us, + self.values]; +} + +@end + +@implementation RTC_OBJC_TYPE (RTCStatisticsReport) + +@synthesize timestamp_us = _timestamp_us; +@synthesize statistics = _statistics; + +- (NSString *)description { + return [NSString + stringWithFormat:@"timestamp = %.0f, statistics = %@", self.timestamp_us, self.statistics]; +} + +@end + +@implementation RTC_OBJC_TYPE (RTCStatisticsReport) (Private) + +- (instancetype)initWithReport : (const webrtc::RTCStatsReport &)report { + if (self = [super init]) { + _timestamp_us = report.timestamp().us(); + + NSMutableDictionary *statisticsById = + [NSMutableDictionary dictionaryWithCapacity:report.size()]; + for (const auto &stat : report) { + RTC_OBJC_TYPE(RTCStatistics) *statistics = + [[RTC_OBJC_TYPE(RTCStatistics) alloc] initWithStatistics:stat]; + statisticsById[statistics.id] = statistics; + } + _statistics = [statisticsById copy]; + } + + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.h new file mode 100644 index 0000000000..5c66e5a63a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.h @@ -0,0 +1,21 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +RTC_EXTERN void RTCSetupInternalTracer(void); +/** Starts capture to specified file. Must be a valid writable path. + * Returns YES if capture starts. + */ +RTC_EXTERN BOOL RTCStartInternalCapture(NSString* filePath); +RTC_EXTERN void RTCStopInternalCapture(void); +RTC_EXTERN void RTCShutdownInternalTracer(void); diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.mm new file mode 100644 index 0000000000..72f9f4da13 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCTracing.mm @@ -0,0 +1,29 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCTracing.h" + +#include "rtc_base/event_tracer.h" + +void RTCSetupInternalTracer(void) { + rtc::tracing::SetupInternalTracer(); +} + +BOOL RTCStartInternalCapture(NSString *filePath) { + return rtc::tracing::StartInternalCapture(filePath.UTF8String); +} + +void RTCStopInternalCapture(void) { + rtc::tracing::StopInternalCapture(); +} + +void RTCShutdownInternalTracer(void) { + rtc::tracing::ShutdownInternalTracer(); +} diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h new file mode 100644 index 0000000000..5eff996c4f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "base/RTCVideoCodecInfo.h" + +#include "api/video_codecs/sdp_video_format.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interface for converting to/from internal C++ formats. */ +@interface RTC_OBJC_TYPE (RTCVideoCodecInfo) +(Private) + + - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format; +- (webrtc::SdpVideoFormat)nativeSdpVideoFormat; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm new file mode 100644 index 0000000000..2eb8d366d2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm @@ -0,0 +1,38 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoCodecInfo+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) +(Private) + + - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format { + NSMutableDictionary *params = [NSMutableDictionary dictionary]; + for (auto it = format.parameters.begin(); it != format.parameters.end(); ++it) { + [params setObject:[NSString stringForStdString:it->second] + forKey:[NSString stringForStdString:it->first]]; + } + return [self initWithName:[NSString stringForStdString:format.name] parameters:params]; +} + +- (webrtc::SdpVideoFormat)nativeSdpVideoFormat { + std::map<std::string, std::string> parameters; + for (NSString *paramKey in self.parameters.allKeys) { + std::string key = [NSString stdStringForString:paramKey]; + std::string value = [NSString stdStringForString:self.parameters[paramKey]]; + parameters[key] = value; + } + + return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h new file mode 100644 index 0000000000..8323b18dc1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "base/RTCVideoEncoderSettings.h" + +#include "modules/video_coding/include/video_codec_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interfaces for converting to/from internal C++ formats. */ +@interface RTC_OBJC_TYPE (RTCVideoEncoderSettings) +(Private) + + - (instancetype)initWithNativeVideoCodec : (const webrtc::VideoCodec *__nullable)videoCodec; +- (webrtc::VideoCodec)nativeVideoCodec; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.mm new file mode 100644 index 0000000000..dec3a61090 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.mm @@ -0,0 +1,52 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoEncoderSettings+Private.h" + +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderSettings) +(Private) + + - (instancetype)initWithNativeVideoCodec : (const webrtc::VideoCodec *)videoCodec { + if (self = [super init]) { + if (videoCodec) { + const char *codecName = CodecTypeToPayloadString(videoCodec->codecType); + self.name = [NSString stringWithUTF8String:codecName]; + + self.width = videoCodec->width; + self.height = videoCodec->height; + self.startBitrate = videoCodec->startBitrate; + self.maxBitrate = videoCodec->maxBitrate; + self.minBitrate = videoCodec->minBitrate; + self.maxFramerate = videoCodec->maxFramerate; + self.qpMax = videoCodec->qpMax; + self.mode = (RTCVideoCodecMode)videoCodec->mode; + } + } + + return self; +} + +- (webrtc::VideoCodec)nativeVideoCodec { + webrtc::VideoCodec videoCodec; + videoCodec.width = self.width; + videoCodec.height = self.height; + videoCodec.startBitrate = self.startBitrate; + videoCodec.maxBitrate = self.maxBitrate; + videoCodec.minBitrate = self.minBitrate; + videoCodec.maxBitrate = self.maxBitrate; + videoCodec.qpMax = self.qpMax; + videoCodec.mode = (webrtc::VideoCodecMode)self.mode; + + return videoCodec; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource+Private.h new file mode 100644 index 0000000000..8e475dd21e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource+Private.h @@ -0,0 +1,51 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoSource.h" + +#import "RTCMediaSource+Private.h" + +#include "api/media_stream_interface.h" +#include "rtc_base/thread.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCVideoSource) +() + + /** + * The VideoTrackSourceInterface object passed to this RTCVideoSource during + * construction. + */ + @property(nonatomic, + readonly) rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeVideoSource; + +/** Initialize an RTCVideoSource from a native VideoTrackSourceInterface. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeVideoSource: + (rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)nativeVideoSource + NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type NS_UNAVAILABLE; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + signalingThread:(rtc::Thread *)signalingThread + workerThread:(rtc::Thread *)workerThread; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + signalingThread:(rtc::Thread *)signalingThread + workerThread:(rtc::Thread *)workerThread + isScreenCast:(BOOL)isScreenCast; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.h new file mode 100644 index 0000000000..cdef8b89a1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.h @@ -0,0 +1,37 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCMediaSource.h" +#import "RTCVideoCapturer.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT + +@interface RTC_OBJC_TYPE (RTCVideoSource) : RTC_OBJC_TYPE(RTCMediaSource) <RTC_OBJC_TYPE(RTCVideoCapturerDelegate)> + +- (instancetype)init NS_UNAVAILABLE; + +/** + * Calling this function will cause frames to be scaled down to the + * requested resolution. Also, frames will be cropped to match the + * requested aspect ratio, and frames will be dropped to match the + * requested fps. The requested aspect ratio is orientation agnostic and + * will be adjusted to maintain the input orientation, so it doesn't + * matter if e.g. 1280x720 or 720x1280 is requested. + */ +- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.mm new file mode 100644 index 0000000000..486ca93771 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoSource.mm @@ -0,0 +1,92 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoSource+Private.h" + +#include "pc/video_track_source_proxy.h" +#include "rtc_base/checks.h" +#include "sdk/objc/native/src/objc_video_track_source.h" + +static webrtc::ObjCVideoTrackSource *getObjCVideoSource( + const rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> nativeSource) { + webrtc::VideoTrackSourceProxy *proxy_source = + static_cast<webrtc::VideoTrackSourceProxy *>(nativeSource.get()); + return static_cast<webrtc::ObjCVideoTrackSource *>(proxy_source->internal()); +} + +// TODO(magjed): Refactor this class and target ObjCVideoTrackSource only once +// RTCAVFoundationVideoSource is gone. See http://crbug/webrtc/7177 for more +// info. +@implementation RTC_OBJC_TYPE (RTCVideoSource) { + rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _nativeVideoSource; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeVideoSource: + (rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)nativeVideoSource { + RTC_DCHECK(factory); + RTC_DCHECK(nativeVideoSource); + if (self = [super initWithFactory:factory + nativeMediaSource:nativeVideoSource + type:RTCMediaSourceTypeVideo]) { + _nativeVideoSource = nativeVideoSource; + } + return self; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeMediaSource:(rtc::scoped_refptr<webrtc::MediaSourceInterface>)nativeMediaSource + type:(RTCMediaSourceType)type { + RTC_DCHECK_NOTREACHED(); + return nil; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + signalingThread:(rtc::Thread *)signalingThread + workerThread:(rtc::Thread *)workerThread { + return [self initWithFactory:factory + signalingThread:signalingThread + workerThread:workerThread + isScreenCast:NO]; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + signalingThread:(rtc::Thread *)signalingThread + workerThread:(rtc::Thread *)workerThread + isScreenCast:(BOOL)isScreenCast { + rtc::scoped_refptr<webrtc::ObjCVideoTrackSource> objCVideoTrackSource = + rtc::make_ref_counted<webrtc::ObjCVideoTrackSource>(isScreenCast); + + return [self initWithFactory:factory + nativeVideoSource:webrtc::VideoTrackSourceProxy::Create( + signalingThread, workerThread, objCVideoTrackSource)]; +} + +- (NSString *)description { + NSString *stateString = [[self class] stringForState:self.state]; + return [NSString stringWithFormat:@"RTC_OBJC_TYPE(RTCVideoSource)( %p ): %@", self, stateString]; +} + +- (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer + didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + getObjCVideoSource(_nativeVideoSource)->OnCapturedFrame(frame); +} + +- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps { + getObjCVideoSource(_nativeVideoSource)->OnOutputFormatRequest(width, height, fps); +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)nativeVideoSource { + return _nativeVideoSource; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack+Private.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack+Private.h new file mode 100644 index 0000000000..f1a8d7e4ed --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack+Private.h @@ -0,0 +1,30 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoTrack.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCVideoTrack) +() + + /** VideoTrackInterface created or passed in at construction. */ + @property(nonatomic, readonly) rtc::scoped_refptr<webrtc::VideoTrackInterface> nativeVideoTrack; + +/** Initialize an RTCVideoTrack with its source and an id. */ +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + source:(RTC_OBJC_TYPE(RTCVideoSource) *)source + trackId:(NSString *)trackId; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.h b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.h new file mode 100644 index 0000000000..5382b7169f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -0,0 +1,38 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMediaStreamTrack.h" + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@protocol RTC_OBJC_TYPE +(RTCVideoRenderer); +@class RTC_OBJC_TYPE(RTCPeerConnectionFactory); +@class RTC_OBJC_TYPE(RTCVideoSource); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoTrack) : RTC_OBJC_TYPE(RTCMediaStreamTrack) + +/** The video source for this video track. */ +@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; + +- (instancetype)init NS_UNAVAILABLE; + +/** Register a renderer that will render all frames received on this track. */ +- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)renderer; + +/** Deregister a renderer. */ +- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)renderer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.mm new file mode 100644 index 0000000000..d3296f6279 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -0,0 +1,125 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoTrack+Private.h" + +#import "RTCMediaStreamTrack+Private.h" +#import "RTCPeerConnectionFactory+Private.h" +#import "RTCVideoSource+Private.h" +#import "api/RTCVideoRendererAdapter+Private.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCVideoTrack) { + rtc::Thread *_workerThread; + NSMutableArray *_adapters /* accessed on _workerThread */; +} + +@synthesize source = _source; + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + source:(RTC_OBJC_TYPE(RTCVideoSource) *)source + trackId:(NSString *)trackId { + NSParameterAssert(factory); + NSParameterAssert(source); + NSParameterAssert(trackId.length); + std::string nativeId = [NSString stdStringForString:trackId]; + rtc::scoped_refptr<webrtc::VideoTrackInterface> track = + factory.nativeFactory->CreateVideoTrack(source.nativeVideoSource, nativeId); + if (self = [self initWithFactory:factory nativeTrack:track type:RTCMediaStreamTrackTypeVideo]) { + _source = source; + } + return self; +} + +- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory + nativeTrack: + (rtc::scoped_refptr<webrtc::MediaStreamTrackInterface>)nativeMediaTrack + type:(RTCMediaStreamTrackType)type { + NSParameterAssert(factory); + NSParameterAssert(nativeMediaTrack); + NSParameterAssert(type == RTCMediaStreamTrackTypeVideo); + if (self = [super initWithFactory:factory nativeTrack:nativeMediaTrack type:type]) { + _adapters = [NSMutableArray array]; + _workerThread = factory.workerThread; + } + return self; +} + +- (void)dealloc { + for (RTCVideoRendererAdapter *adapter in _adapters) { + self.nativeVideoTrack->RemoveSink(adapter.nativeVideoRenderer); + } +} + +- (RTC_OBJC_TYPE(RTCVideoSource) *)source { + if (!_source) { + rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> source( + self.nativeVideoTrack->GetSource()); + if (source) { + _source = [[RTC_OBJC_TYPE(RTCVideoSource) alloc] initWithFactory:self.factory + nativeVideoSource:source]; + } + } + return _source; +} + +- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)renderer { + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); + return; + } + + // Make sure we don't have this renderer yet. + for (RTCVideoRendererAdapter *adapter in _adapters) { + if (adapter.videoRenderer == renderer) { + RTC_LOG(LS_INFO) << "|renderer| is already attached to this track"; + return; + } + } + // Create a wrapper that provides a native pointer for us. + RTCVideoRendererAdapter* adapter = + [[RTCVideoRendererAdapter alloc] initWithNativeRenderer:renderer]; + [_adapters addObject:adapter]; + self.nativeVideoTrack->AddOrUpdateSink(adapter.nativeVideoRenderer, + rtc::VideoSinkWants()); +} + +- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCVideoRenderer)>)renderer { + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); + return; + } + __block NSUInteger indexToRemove = NSNotFound; + [_adapters enumerateObjectsUsingBlock:^(RTCVideoRendererAdapter *adapter, + NSUInteger idx, + BOOL *stop) { + if (adapter.videoRenderer == renderer) { + indexToRemove = idx; + *stop = YES; + } + }]; + if (indexToRemove == NSNotFound) { + RTC_LOG(LS_INFO) << "removeRenderer called with a renderer that has not been previously added"; + return; + } + RTCVideoRendererAdapter *adapterToRemove = + [_adapters objectAtIndex:indexToRemove]; + self.nativeVideoTrack->RemoveSink(adapterToRemove.nativeVideoRenderer); + [_adapters removeObjectAtIndex:indexToRemove]; +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::VideoTrackInterface>)nativeVideoTrack { + return rtc::scoped_refptr<webrtc::VideoTrackInterface>( + static_cast<webrtc::VideoTrackInterface *>(self.nativeTrack.get())); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.h new file mode 100644 index 0000000000..8b17a75aef --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.h @@ -0,0 +1,17 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +RTC_EXTERN NSString* const kRTCVideoCodecVp8Name; +RTC_EXTERN NSString* const kRTCVideoCodecVp9Name; +RTC_EXTERN NSString* const kRTCVideoCodecAv1Name; diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.mm new file mode 100644 index 0000000000..1ab236a2c2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoCodecConstants.mm @@ -0,0 +1,18 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "RTCVideoCodecConstants.h" + +#include "media/base/media_constants.h" + +NSString *const kRTCVideoCodecVp8Name = @(cricket::kVp8CodecName); +NSString *const kRTCVideoCodecVp9Name = @(cricket::kVp9CodecName); +NSString *const kRTCVideoCodecAv1Name = @(cricket::kAv1CodecName); diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.h new file mode 100644 index 0000000000..3f6a689564 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.h @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoDecoderAV1) : NSObject + +/* This returns a AV1 decoder that can be returned from a RTCVideoDecoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoDecoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)av1Decoder; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.mm new file mode 100644 index 0000000000..81f5f93eec --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderAV1.mm @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoderAV1.h" +#import "RTCWrappedNativeVideoDecoder.h" + +#include "modules/video_coding/codecs/av1/dav1d_decoder.h" + +@implementation RTC_OBJC_TYPE (RTCVideoDecoderAV1) + ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)av1Decoder { + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoDecoder) alloc] + initWithNativeDecoder:std::unique_ptr<webrtc::VideoDecoder>(webrtc::CreateDav1dDecoder())]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.h new file mode 100644 index 0000000000..a118b25ed7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoDecoderVP8) : NSObject + +/* This returns a VP8 decoder that can be returned from a RTCVideoDecoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoDecoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)vp8Decoder; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.mm new file mode 100644 index 0000000000..c150cf6d3a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP8.mm @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoderVP8.h" +#import "RTCWrappedNativeVideoDecoder.h" + +#include "modules/video_coding/codecs/vp8/include/vp8.h" + +@implementation RTC_OBJC_TYPE (RTCVideoDecoderVP8) + ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)vp8Decoder { + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoDecoder) alloc] + initWithNativeDecoder:std::unique_ptr<webrtc::VideoDecoder>(webrtc::VP8Decoder::Create())]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.h new file mode 100644 index 0000000000..de7e62012b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.h @@ -0,0 +1,27 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoDecoderVP9) : NSObject + +/* This returns a VP9 decoder that can be returned from a RTCVideoDecoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoDecoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)vp9Decoder; + ++ (bool)isSupported; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.mm new file mode 100644 index 0000000000..05446d436d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoDecoderVP9.mm @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoderVP9.h" +#import "RTCWrappedNativeVideoDecoder.h" + +#include "modules/video_coding/codecs/vp9/include/vp9.h" + +@implementation RTC_OBJC_TYPE (RTCVideoDecoderVP9) + ++ (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)vp9Decoder { + std::unique_ptr<webrtc::VideoDecoder> nativeDecoder(webrtc::VP9Decoder::Create()); + if (nativeDecoder == nullptr) { + return nil; + } + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoDecoder) alloc] + initWithNativeDecoder:std::move(nativeDecoder)]; +} + ++ (bool)isSupported { +#if defined(RTC_ENABLE_VP9) + return true; +#else + return false; +#endif +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h new file mode 100644 index 0000000000..8aa55e4bfa --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h @@ -0,0 +1,27 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderAV1) : NSObject + +/* This returns a AV1 encoder that can be returned from a RTCVideoEncoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoEncoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)av1Encoder; + ++ (bool)isSupported; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm new file mode 100644 index 0000000000..d2fe65293b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoderAV1.h" +#import "RTCWrappedNativeVideoEncoder.h" +#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderAV1) + ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)av1Encoder { + std::unique_ptr<webrtc::VideoEncoder> nativeEncoder(webrtc::CreateLibaomAv1Encoder()); + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] + initWithNativeEncoder:std::move(nativeEncoder)]; +} + ++ (bool)isSupported { + return true; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.h new file mode 100644 index 0000000000..e136a5bda8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderVP8) : NSObject + +/* This returns a VP8 encoder that can be returned from a RTCVideoEncoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoEncoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)vp8Encoder; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.mm new file mode 100644 index 0000000000..d72f705813 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP8.mm @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoderVP8.h" +#import "RTCWrappedNativeVideoEncoder.h" + +#include "modules/video_coding/codecs/vp8/include/vp8.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderVP8) + ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)vp8Encoder { + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] + initWithNativeEncoder:std::unique_ptr<webrtc::VideoEncoder>(webrtc::VP8Encoder::Create())]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h new file mode 100644 index 0000000000..f7dac6117d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h @@ -0,0 +1,27 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderVP9) : NSObject + +/* This returns a VP9 encoder that can be returned from a RTCVideoEncoderFactory injected into + * RTCPeerConnectionFactory. Even though it implements the RTCVideoEncoder protocol, it can not be + * used independently from the RTCPeerConnectionFactory. + */ ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)vp9Encoder; + ++ (bool)isSupported; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm new file mode 100644 index 0000000000..18a9353f7e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoderVP9.h" +#import "RTCWrappedNativeVideoEncoder.h" + +#include "modules/video_coding/codecs/vp9/include/vp9.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderVP9) + ++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)vp9Encoder { + std::unique_ptr<webrtc::VideoEncoder> nativeEncoder(webrtc::VP9Encoder::Create()); + if (nativeEncoder == nullptr) { + return nil; + } + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] + initWithNativeEncoder:std::move(nativeEncoder)]; +} + ++ (bool)isSupported { +#if defined(RTC_ENABLE_VP9) + return true; +#else + return false; +#endif +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h new file mode 100644 index 0000000000..3a9b39e959 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "base/RTCMacros.h" +#import "base/RTCVideoDecoder.h" + +#include "api/video_codecs/video_decoder.h" +#include "media/base/codec.h" + +@interface RTC_OBJC_TYPE (RTCWrappedNativeVideoDecoder) : NSObject <RTC_OBJC_TYPE (RTCVideoDecoder)> + +- (instancetype)initWithNativeDecoder:(std::unique_ptr<webrtc::VideoDecoder>)decoder; + +/* This moves the ownership of the wrapped decoder to the caller. */ +- (std::unique_ptr<webrtc::VideoDecoder>)releaseWrappedDecoder; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.mm new file mode 100644 index 0000000000..29d2265e20 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.mm @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCWrappedNativeVideoDecoder.h" +#import "base/RTCMacros.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCWrappedNativeVideoDecoder) { + std::unique_ptr<webrtc::VideoDecoder> _wrappedDecoder; +} + +- (instancetype)initWithNativeDecoder:(std::unique_ptr<webrtc::VideoDecoder>)decoder { + if (self = [super init]) { + _wrappedDecoder = std::move(decoder); + } + + return self; +} + +- (std::unique_ptr<webrtc::VideoDecoder>)releaseWrappedDecoder { + return std::move(_wrappedDecoder); +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoDecoder) + +- (void)setCallback:(RTCVideoDecoderCallback)callback { + RTC_DCHECK_NOTREACHED(); +} + +- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (NSInteger)releaseDecoder { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +// TODO(bugs.webrtc.org/15444): Remove obsolete missingFrames param. +- (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)encodedImage + missingFrames:(BOOL)missingFrames + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info + renderTimeMs:(int64_t)renderTimeMs { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (NSString *)implementationName { + RTC_DCHECK_NOTREACHED(); + return nil; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h new file mode 100644 index 0000000000..8df9ceec35 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "base/RTCMacros.h" +#import "base/RTCVideoEncoder.h" + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/codec.h" + +@interface RTC_OBJC_TYPE (RTCWrappedNativeVideoEncoder) : NSObject <RTC_OBJC_TYPE (RTCVideoEncoder)> + +- (instancetype)initWithNativeEncoder:(std::unique_ptr<webrtc::VideoEncoder>)encoder; + +/* This moves the ownership of the wrapped encoder to the caller. */ +- (std::unique_ptr<webrtc::VideoEncoder>)releaseWrappedEncoder; + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.mm b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.mm new file mode 100644 index 0000000000..4160572814 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.mm @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCWrappedNativeVideoEncoder.h" +#import "base/RTCMacros.h" +#import "helpers/NSString+StdString.h" + +@implementation RTC_OBJC_TYPE (RTCWrappedNativeVideoEncoder) { + std::unique_ptr<webrtc::VideoEncoder> _wrappedEncoder; +} + +- (instancetype)initWithNativeEncoder:(std::unique_ptr<webrtc::VideoEncoder>)encoder { + if (self = [super init]) { + _wrappedEncoder = std::move(encoder); + } + + return self; +} + +- (std::unique_ptr<webrtc::VideoEncoder>)releaseWrappedEncoder { + return std::move(_wrappedEncoder); +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoEncoder) + +- (void)setCallback:(RTCVideoEncoderCallback)callback { + RTC_DCHECK_NOTREACHED(); +} + +- (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)settings + numberOfCores:(int)numberOfCores { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (NSInteger)releaseEncoder { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info + frameTypes:(NSArray<NSNumber *> *)frameTypes { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { + RTC_DCHECK_NOTREACHED(); + return 0; +} + +- (NSString *)implementationName { + RTC_DCHECK_NOTREACHED(); + return nil; +} + +- (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings { + RTC_DCHECK_NOTREACHED(); + return nil; +} + +- (NSInteger)resolutionAlignment { + RTC_DCHECK_NOTREACHED(); + return 1; +} + +- (BOOL)applyAlignmentToAllSimulcastLayers { + RTC_DCHECK_NOTREACHED(); + return NO; +} + +- (BOOL)supportsNativeHandle { + RTC_DCHECK_NOTREACHED(); + return NO; +} +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h new file mode 100644 index 0000000000..20dc807991 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeI420Buffer.h" + +#include "api/video/i420_buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCI420Buffer) +() { + @protected + rtc::scoped_refptr<webrtc::I420BufferInterface> _i420Buffer; +} + +/** Initialize an RTCI420Buffer with its backing I420BufferInterface. */ +- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr<webrtc::I420BufferInterface>)i420Buffer; +- (rtc::scoped_refptr<webrtc::I420BufferInterface>)nativeI420Buffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.h b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.h new file mode 100644 index 0000000000..3afe2090a2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.h @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCI420Buffer.h" +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCI420Buffer implements the RTCI420Buffer protocol */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCI420Buffer) : NSObject<RTC_OBJC_TYPE(RTCI420Buffer)> +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm new file mode 100644 index 0000000000..7aafd98f43 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm @@ -0,0 +1,154 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeI420Buffer+Private.h" + +#include "api/video/i420_buffer.h" + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +#import <UIKit/UIKit.h> +#include "third_party/libyuv/include/libyuv.h" +#endif + +@implementation RTC_OBJC_TYPE (RTCI420Buffer) + +- (instancetype)initWithWidth:(int)width height:(int)height { + if (self = [super init]) { + _i420Buffer = webrtc::I420Buffer::Create(width, height); + } + + return self; +} + +- (instancetype)initWithWidth:(int)width + height:(int)height + dataY:(const uint8_t *)dataY + dataU:(const uint8_t *)dataU + dataV:(const uint8_t *)dataV { + if (self = [super init]) { + _i420Buffer = webrtc::I420Buffer::Copy( + width, height, dataY, width, dataU, (width + 1) / 2, dataV, (width + 1) / 2); + } + return self; +} + +- (instancetype)initWithWidth:(int)width + height:(int)height + strideY:(int)strideY + strideU:(int)strideU + strideV:(int)strideV { + if (self = [super init]) { + _i420Buffer = webrtc::I420Buffer::Create(width, height, strideY, strideU, strideV); + } + + return self; +} + +- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr<webrtc::I420BufferInterface>)i420Buffer { + if (self = [super init]) { + _i420Buffer = i420Buffer; + } + + return self; +} + +- (int)width { + return _i420Buffer->width(); +} + +- (int)height { + return _i420Buffer->height(); +} + +- (int)strideY { + return _i420Buffer->StrideY(); +} + +- (int)strideU { + return _i420Buffer->StrideU(); +} + +- (int)strideV { + return _i420Buffer->StrideV(); +} + +- (int)chromaWidth { + return _i420Buffer->ChromaWidth(); +} + +- (int)chromaHeight { + return _i420Buffer->ChromaHeight(); +} + +- (const uint8_t *)dataY { + return _i420Buffer->DataY(); +} + +- (const uint8_t *)dataU { + return _i420Buffer->DataU(); +} + +- (const uint8_t *)dataV { + return _i420Buffer->DataV(); +} + +- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight { + rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer = + _i420Buffer->CropAndScale(offsetX, offsetY, cropWidth, cropHeight, scaleWidth, scaleHeight); + RTC_DCHECK_EQ(scaled_buffer->type(), webrtc::VideoFrameBuffer::Type::kI420); + // Calling ToI420() doesn't do any conversions. + rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = scaled_buffer->ToI420(); + RTC_OBJC_TYPE(RTCI420Buffer) *result = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer]; + return result; +} + +- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 { + return self; +} + +#pragma mark - Private + +- (rtc::scoped_refptr<webrtc::I420BufferInterface>)nativeI420Buffer { + return _i420Buffer; +} + +#pragma mark - Debugging + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +- (id)debugQuickLookObject { + UIGraphicsBeginImageContext(CGSizeMake(_i420Buffer->width(), _i420Buffer->height())); + CGContextRef c = UIGraphicsGetCurrentContext(); + uint8_t *ctxData = (uint8_t *)CGBitmapContextGetData(c); + + libyuv::I420ToARGB(_i420Buffer->DataY(), + _i420Buffer->StrideY(), + _i420Buffer->DataU(), + _i420Buffer->StrideU(), + _i420Buffer->DataV(), + _i420Buffer->StrideV(), + ctxData, + CGBitmapContextGetBytesPerRow(c), + CGBitmapContextGetWidth(c), + CGBitmapContextGetHeight(c)); + + UIImage *image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + + return image; +} +#endif + +@end diff --git a/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h new file mode 100644 index 0000000000..053a10a304 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCMacros.h" +#import "RTCMutableI420Buffer.h" +#import "RTCNativeI420Buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Mutable version of RTCI420Buffer */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMutableI420Buffer) : RTC_OBJC_TYPE(RTCI420Buffer)<RTC_OBJC_TYPE(RTCMutableI420Buffer)> +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.mm b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.mm new file mode 100644 index 0000000000..1e669bcb9c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.mm @@ -0,0 +1,31 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeMutableI420Buffer.h" + +#import "RTCNativeI420Buffer+Private.h" + +#include "api/video/i420_buffer.h" + +@implementation RTC_OBJC_TYPE (RTCMutableI420Buffer) + +- (uint8_t *)mutableDataY { + return static_cast<webrtc::I420Buffer *>(_i420Buffer.get())->MutableDataY(); +} + +- (uint8_t *)mutableDataU { + return static_cast<webrtc::I420Buffer *>(_i420Buffer.get())->MutableDataU(); +} + +- (uint8_t *)mutableDataV { + return static_cast<webrtc::I420Buffer *>(_i420Buffer.get())->MutableDataV(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCCodecSpecificInfo.h b/third_party/libwebrtc/sdk/objc/base/RTCCodecSpecificInfo.h new file mode 100644 index 0000000000..5e7800e524 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCCodecSpecificInfo.h @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Implement this protocol to pass codec specific info from the encoder. + * Corresponds to webrtc::CodecSpecificInfo. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCCodecSpecificInfo)<NSObject> @end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.h b/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.h new file mode 100644 index 0000000000..28529e5906 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.h @@ -0,0 +1,52 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents an encoded frame's type. */ +typedef NS_ENUM(NSUInteger, RTCFrameType) { + RTCFrameTypeEmptyFrame = 0, + RTCFrameTypeAudioFrameSpeech = 1, + RTCFrameTypeAudioFrameCN = 2, + RTCFrameTypeVideoFrameKey = 3, + RTCFrameTypeVideoFrameDelta = 4, +}; + +typedef NS_ENUM(NSUInteger, RTCVideoContentType) { + RTCVideoContentTypeUnspecified, + RTCVideoContentTypeScreenshare, +}; + +/** Represents an encoded frame. Corresponds to webrtc::EncodedImage. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCEncodedImage) : NSObject + +@property(nonatomic, strong) NSData *buffer; +@property(nonatomic, assign) int32_t encodedWidth; +@property(nonatomic, assign) int32_t encodedHeight; +@property(nonatomic, assign) uint32_t timeStamp; +@property(nonatomic, assign) int64_t captureTimeMs; +@property(nonatomic, assign) int64_t ntpTimeMs; +@property(nonatomic, assign) uint8_t flags; +@property(nonatomic, assign) int64_t encodeStartMs; +@property(nonatomic, assign) int64_t encodeFinishMs; +@property(nonatomic, assign) RTCFrameType frameType; +@property(nonatomic, assign) RTCVideoRotation rotation; +@property(nonatomic, strong) NSNumber *qp; +@property(nonatomic, assign) RTCVideoContentType contentType; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.m b/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.m new file mode 100644 index 0000000000..ad8441aabd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCEncodedImage.m @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCEncodedImage.h" + +@implementation RTC_OBJC_TYPE (RTCEncodedImage) + +@synthesize buffer = _buffer; +@synthesize encodedWidth = _encodedWidth; +@synthesize encodedHeight = _encodedHeight; +@synthesize timeStamp = _timeStamp; +@synthesize captureTimeMs = _captureTimeMs; +@synthesize ntpTimeMs = _ntpTimeMs; +@synthesize flags = _flags; +@synthesize encodeStartMs = _encodeStartMs; +@synthesize encodeFinishMs = _encodeFinishMs; +@synthesize frameType = _frameType; +@synthesize rotation = _rotation; +@synthesize qp = _qp; +@synthesize contentType = _contentType; + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCI420Buffer.h b/third_party/libwebrtc/sdk/objc/base/RTCI420Buffer.h new file mode 100644 index 0000000000..b97f05a5ba --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCI420Buffer.h @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCYUVPlanarBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Protocol for RTCYUVPlanarBuffers containing I420 data */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCI420Buffer)<RTC_OBJC_TYPE(RTCYUVPlanarBuffer)> @end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCLogging.h b/third_party/libwebrtc/sdk/objc/base/RTCLogging.h new file mode 100644 index 0000000000..0fa6a91b69 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCLogging.h @@ -0,0 +1,66 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +// Subset of rtc::LoggingSeverity. +typedef NS_ENUM(NSInteger, RTCLoggingSeverity) { + RTCLoggingSeverityVerbose, + RTCLoggingSeverityInfo, + RTCLoggingSeverityWarning, + RTCLoggingSeverityError, + RTCLoggingSeverityNone, +}; + +// Wrapper for C++ RTC_LOG(sev) macros. +// Logs the log string to the webrtc logstream for the given severity. +RTC_EXTERN void RTCLogEx(RTCLoggingSeverity severity, NSString* log_string); + +// Wrapper for rtc::LogMessage::LogToDebug. +// Sets the minimum severity to be logged to console. +RTC_EXTERN void RTCSetMinDebugLogLevel(RTCLoggingSeverity severity); + +// Returns the filename with the path prefix removed. +RTC_EXTERN NSString* RTCFileName(const char* filePath); + +// Some convenience macros. + +#define RTCLogString(format, ...) \ + [NSString stringWithFormat:@"(%@:%d %s): " format, \ + RTCFileName(__FILE__), \ + __LINE__, \ + __FUNCTION__, \ + ##__VA_ARGS__] + +#define RTCLogFormat(severity, format, ...) \ + do { \ + NSString* log_string = RTCLogString(format, ##__VA_ARGS__); \ + RTCLogEx(severity, log_string); \ + } while (false) + +#define RTCLogVerbose(format, ...) RTCLogFormat(RTCLoggingSeverityVerbose, format, ##__VA_ARGS__) + +#define RTCLogInfo(format, ...) RTCLogFormat(RTCLoggingSeverityInfo, format, ##__VA_ARGS__) + +#define RTCLogWarning(format, ...) RTCLogFormat(RTCLoggingSeverityWarning, format, ##__VA_ARGS__) + +#define RTCLogError(format, ...) RTCLogFormat(RTCLoggingSeverityError, format, ##__VA_ARGS__) + +#if !defined(NDEBUG) +#define RTCLogDebug(format, ...) RTCLogInfo(format, ##__VA_ARGS__) +#else +#define RTCLogDebug(format, ...) \ + do { \ + } while (false) +#endif + +#define RTCLog(format, ...) RTCLogInfo(format, ##__VA_ARGS__) diff --git a/third_party/libwebrtc/sdk/objc/base/RTCLogging.mm b/third_party/libwebrtc/sdk/objc/base/RTCLogging.mm new file mode 100644 index 0000000000..e8dae02efb --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCLogging.mm @@ -0,0 +1,48 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCLogging.h" + +#include "rtc_base/logging.h" + +rtc::LoggingSeverity RTCGetNativeLoggingSeverity(RTCLoggingSeverity severity) { + switch (severity) { + case RTCLoggingSeverityVerbose: + return rtc::LS_VERBOSE; + case RTCLoggingSeverityInfo: + return rtc::LS_INFO; + case RTCLoggingSeverityWarning: + return rtc::LS_WARNING; + case RTCLoggingSeverityError: + return rtc::LS_ERROR; + case RTCLoggingSeverityNone: + return rtc::LS_NONE; + } +} + +void RTCLogEx(RTCLoggingSeverity severity, NSString* log_string) { + if (log_string.length) { + const char* utf8_string = log_string.UTF8String; + RTC_LOG_V(RTCGetNativeLoggingSeverity(severity)) << utf8_string; + } +} + +void RTCSetMinDebugLogLevel(RTCLoggingSeverity severity) { + rtc::LogMessage::LogToDebug(RTCGetNativeLoggingSeverity(severity)); +} + +NSString* RTCFileName(const char* file_path) { + NSString* ns_file_path = + [[NSString alloc] initWithBytesNoCopy:const_cast<char*>(file_path) + length:strlen(file_path) + encoding:NSUTF8StringEncoding + freeWhenDone:NO]; + return ns_file_path.lastPathComponent; +} diff --git a/third_party/libwebrtc/sdk/objc/base/RTCMacros.h b/third_party/libwebrtc/sdk/objc/base/RTCMacros.h new file mode 100644 index 0000000000..114ced0ea6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCMacros.h @@ -0,0 +1,63 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_BASE_RTCMACROS_H_ +#define SDK_OBJC_BASE_RTCMACROS_H_ + +#ifdef WEBRTC_ENABLE_OBJC_SYMBOL_EXPORT + +#if defined(WEBRTC_LIBRARY_IMPL) +#define RTC_OBJC_EXPORT __attribute__((visibility("default"))) +#endif + +#endif // WEBRTC_ENABLE_OBJC_SYMBOL_EXPORT + +#ifndef RTC_OBJC_EXPORT +#define RTC_OBJC_EXPORT +#endif + +// Internal macros used to correctly concatenate symbols. +#define RTC_SYMBOL_CONCAT_HELPER(a, b) a##b +#define RTC_SYMBOL_CONCAT(a, b) RTC_SYMBOL_CONCAT_HELPER(a, b) + +// RTC_OBJC_TYPE_PREFIX +// +// Macro used to prepend a prefix to the API types that are exported with +// RTC_OBJC_EXPORT. +// +// Clients can patch the definition of this macro locally and build +// WebRTC.framework with their own prefix in case symbol clashing is a +// problem. +// +// This macro must be defined uniformily across all the translation units. +#ifndef RTC_OBJC_TYPE_PREFIX +#define RTC_OBJC_TYPE_PREFIX +#endif + +// RCT_OBJC_TYPE +// +// Macro used internally to declare API types. Declaring an API type without +// using this macro will not include the declared type in the set of types +// that will be affected by the configurable RTC_OBJC_TYPE_PREFIX. +#define RTC_OBJC_TYPE(type_name) RTC_SYMBOL_CONCAT(RTC_OBJC_TYPE_PREFIX, type_name) + +#if defined(__cplusplus) +#define RTC_EXTERN extern "C" RTC_OBJC_EXPORT +#else +#define RTC_EXTERN extern RTC_OBJC_EXPORT +#endif + +#ifdef __OBJC__ +#define RTC_FWD_DECL_OBJC_CLASS(classname) @class classname +#else +#define RTC_FWD_DECL_OBJC_CLASS(classname) typedef struct objc_object classname +#endif + +#endif // SDK_OBJC_BASE_RTCMACROS_H_ diff --git a/third_party/libwebrtc/sdk/objc/base/RTCMutableI420Buffer.h b/third_party/libwebrtc/sdk/objc/base/RTCMutableI420Buffer.h new file mode 100644 index 0000000000..cde721980b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCMutableI420Buffer.h @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCI420Buffer.h" +#import "RTCMutableYUVPlanarBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Extension of the I420 buffer with mutable data access */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCMutableI420Buffer)<RTC_OBJC_TYPE(RTCI420Buffer), RTC_OBJC_TYPE(RTCMutableYUVPlanarBuffer)> @end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCMutableYUVPlanarBuffer.h b/third_party/libwebrtc/sdk/objc/base/RTCMutableYUVPlanarBuffer.h new file mode 100644 index 0000000000..bd14e3bca3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCMutableYUVPlanarBuffer.h @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCYUVPlanarBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Extension of the YUV planar data buffer with mutable data access */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCMutableYUVPlanarBuffer)<RTC_OBJC_TYPE(RTCYUVPlanarBuffer)> + + @property(nonatomic, readonly) uint8_t *mutableDataY; +@property(nonatomic, readonly) uint8_t *mutableDataU; +@property(nonatomic, readonly) uint8_t *mutableDataV; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCSSLCertificateVerifier.h b/third_party/libwebrtc/sdk/objc/base/RTCSSLCertificateVerifier.h new file mode 100644 index 0000000000..53da0cceff --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCSSLCertificateVerifier.h @@ -0,0 +1,25 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCSSLCertificateVerifier)<NSObject> + + /** The certificate to verify */ + - (BOOL)verify : (NSData *)derCertificate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.h new file mode 100644 index 0000000000..a1ffdcf38e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.h @@ -0,0 +1,35 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoFrame.h" + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCVideoCapturer); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoCapturerDelegate)<NSObject> - + (void)capturer : (RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer didCaptureVideoFrame + : (RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +@end + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoCapturer) : NSObject + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)> delegate; + +- (instancetype)initWithDelegate:(id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.m b/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.m new file mode 100644 index 0000000000..ca31a731f0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoCapturer.m @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoCapturer.h" + +@implementation RTC_OBJC_TYPE (RTCVideoCapturer) + +@synthesize delegate = _delegate; + +- (instancetype)initWithDelegate:(id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate { + if (self = [super init]) { + _delegate = delegate; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.h new file mode 100644 index 0000000000..fa28958f25 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.h @@ -0,0 +1,36 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Holds information to identify a codec. Corresponds to webrtc::SdpVideoFormat. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoCodecInfo) : NSObject <NSCoding> + +- (instancetype)init NS_UNAVAILABLE; + +- (instancetype)initWithName:(NSString *)name; + +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters + NS_DESIGNATED_INITIALIZER; + +- (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; + +@property(nonatomic, readonly) NSString *name; +@property(nonatomic, readonly) NSDictionary<NSString *, NSString *> *parameters; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.m b/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.m new file mode 100644 index 0000000000..ce26ae1de3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoCodecInfo.m @@ -0,0 +1,65 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoCodecInfo.h" + +@implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) + +@synthesize name = _name; +@synthesize parameters = _parameters; + +- (instancetype)initWithName:(NSString *)name { + return [self initWithName:name parameters:nil]; +} + +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary<NSString *, NSString *> *)parameters { + if (self = [super init]) { + _name = name; + _parameters = (parameters ? parameters : @{}); + } + + return self; +} + +- (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + if (!info || + ![self.name isEqualToString:info.name] || + ![self.parameters isEqualToDictionary:info.parameters]) { + return NO; + } + return YES; +} + +- (BOOL)isEqual:(id)object { + if (self == object) + return YES; + if (![object isKindOfClass:[self class]]) + return NO; + return [self isEqualToCodecInfo:object]; +} + +- (NSUInteger)hash { + return [self.name hash] ^ [self.parameters hash]; +} + +#pragma mark - NSCoding + +- (instancetype)initWithCoder:(NSCoder *)decoder { + return [self initWithName:[decoder decodeObjectForKey:@"name"] + parameters:[decoder decodeObjectForKey:@"parameters"]]; +} + +- (void)encodeWithCoder:(NSCoder *)encoder { + [encoder encodeObject:_name forKey:@"name"]; + [encoder encodeObject:_parameters forKey:@"parameters"]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoder.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoder.h new file mode 100644 index 0000000000..2565afa724 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoder.h @@ -0,0 +1,41 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCodecSpecificInfo.h" +#import "RTCEncodedImage.h" +#import "RTCMacros.h" +#import "RTCVideoEncoderSettings.h" +#import "RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Callback block for decoder. */ +typedef void (^RTCVideoDecoderCallback)(RTC_OBJC_TYPE(RTCVideoFrame) * frame); + +/** Protocol for decoder implementations. */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoDecoder)<NSObject> + + - (void)setCallback : (RTCVideoDecoderCallback)callback; +- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores; +- (NSInteger)releaseDecoder; +// TODO(bugs.webrtc.org/15444): Remove obsolete missingFrames param. +- (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)encodedImage + missingFrames:(BOOL)missingFrames + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info + renderTimeMs:(int64_t)renderTimeMs; +- (NSString *)implementationName; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoderFactory.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoderFactory.h new file mode 100644 index 0000000000..8d90138521 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoDecoderFactory.h @@ -0,0 +1,32 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoCodecInfo.h" +#import "RTCVideoDecoder.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCVideoDecoderFactory is an Objective-C version of webrtc::VideoDecoderFactory. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoDecoderFactory)<NSObject> + + - (nullable id<RTC_OBJC_TYPE(RTCVideoDecoder)>)createDecoder + : (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *) + supportedCodecs; // TODO(andersc): "supportedFormats" instead? + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoder.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoder.h new file mode 100644 index 0000000000..27e6927ae2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoder.h @@ -0,0 +1,59 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCodecSpecificInfo.h" +#import "RTCEncodedImage.h" +#import "RTCMacros.h" +#import "RTCVideoEncoderQpThresholds.h" +#import "RTCVideoEncoderSettings.h" +#import "RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Callback block for encoder. */ +typedef BOOL (^RTCVideoEncoderCallback)(RTC_OBJC_TYPE(RTCEncodedImage) * frame, + id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)> info); + +/** Protocol for encoder implementations. */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoEncoder)<NSObject> + + - (void)setCallback : (nullable RTCVideoEncoderCallback)callback; +- (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)settings + numberOfCores:(int)numberOfCores; +- (NSInteger)releaseEncoder; +- (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info + frameTypes:(NSArray<NSNumber *> *)frameTypes; +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate; +- (NSString *)implementationName; + +/** Returns QP scaling settings for encoder. The quality scaler adjusts the resolution in order to + * keep the QP from the encoded images within the given range. Returning nil from this function + * disables quality scaling. */ +- (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings; + +/** Resolutions should be aligned to this value. */ +@property(nonatomic, readonly) NSInteger resolutionAlignment; + +/** If enabled, resolution alignment is applied to all simulcast layers simultaneously so that when + scaled, all resolutions comply with 'resolutionAlignment'. */ +@property(nonatomic, readonly) BOOL applyAlignmentToAllSimulcastLayers; + +/** If YES, the receiver is expected to resample/scale the source texture to the expected output + size. */ +@property(nonatomic, readonly) BOOL supportsNativeHandle; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderFactory.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderFactory.h new file mode 100644 index 0000000000..a73cd77990 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderFactory.h @@ -0,0 +1,52 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoCodecInfo.h" +#import "RTCVideoEncoder.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCVideoEncoderFactory is an Objective-C version of + webrtc::VideoEncoderFactory::VideoEncoderSelector. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoEncoderSelector)<NSObject> + + - (void)registerCurrentEncoderInfo : (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; +- (nullable RTC_OBJC_TYPE(RTCVideoCodecInfo) *)encoderForBitrate:(NSInteger)bitrate; +- (nullable RTC_OBJC_TYPE(RTCVideoCodecInfo) *)encoderForBrokenEncoder; + +@optional +- (nullable RTC_OBJC_TYPE(RTCVideoCodecInfo) *)encoderForResolutionChangeBySize:(CGSize)size; + +@end + +/** RTCVideoEncoderFactory is an Objective-C version of webrtc::VideoEncoderFactory. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoEncoderFactory)<NSObject> + + - (nullable id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder + : (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *) + supportedCodecs; // TODO(andersc): "supportedFormats" instead? + +@optional +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)implementations; +- (nullable id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)>)encoderSelector; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.h new file mode 100644 index 0000000000..1a6e9e88ab --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.h @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +/** QP thresholds for encoder. Corresponds to webrtc::VideoEncoder::QpThresholds. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderQpThresholds) : NSObject + +- (instancetype)initWithThresholdsLow:(NSInteger)low high:(NSInteger)high; + +@property(nonatomic, readonly) NSInteger low; +@property(nonatomic, readonly) NSInteger high; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.m b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.m new file mode 100644 index 0000000000..fb7012f44f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderQpThresholds.m @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoEncoderQpThresholds.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderQpThresholds) + +@synthesize low = _low; +@synthesize high = _high; + +- (instancetype)initWithThresholdsLow:(NSInteger)low high:(NSInteger)high { + if (self = [super init]) { + _low = low; + _high = high; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.h new file mode 100644 index 0000000000..ae792eab71 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.h @@ -0,0 +1,42 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(NSUInteger, RTCVideoCodecMode) { + RTCVideoCodecModeRealtimeVideo, + RTCVideoCodecModeScreensharing, +}; + +/** Settings for encoder. Corresponds to webrtc::VideoCodec. */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderSettings) : NSObject + +@property(nonatomic, strong) NSString *name; + +@property(nonatomic, assign) unsigned short width; +@property(nonatomic, assign) unsigned short height; + +@property(nonatomic, assign) unsigned int startBitrate; // kilobits/sec. +@property(nonatomic, assign) unsigned int maxBitrate; +@property(nonatomic, assign) unsigned int minBitrate; + +@property(nonatomic, assign) uint32_t maxFramerate; + +@property(nonatomic, assign) unsigned int qpMax; +@property(nonatomic, assign) RTCVideoCodecMode mode; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.m b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.m new file mode 100644 index 0000000000..f66cd2cf77 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoEncoderSettings.m @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoEncoderSettings.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderSettings) + +@synthesize name = _name; +@synthesize width = _width; +@synthesize height = _height; +@synthesize startBitrate = _startBitrate; +@synthesize maxBitrate = _maxBitrate; +@synthesize minBitrate = _minBitrate; +@synthesize maxFramerate = _maxFramerate; +@synthesize qpMax = _qpMax; +@synthesize mode = _mode; + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.h new file mode 100644 index 0000000000..edf074b682 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.h @@ -0,0 +1,86 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(NSInteger, RTCVideoRotation) { + RTCVideoRotation_0 = 0, + RTCVideoRotation_90 = 90, + RTCVideoRotation_180 = 180, + RTCVideoRotation_270 = 270, +}; + +@protocol RTC_OBJC_TYPE +(RTCVideoFrameBuffer); + +// RTCVideoFrame is an ObjectiveC version of webrtc::VideoFrame. +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoFrame) : NSObject + +/** Width without rotation applied. */ +@property(nonatomic, readonly) int width; + +/** Height without rotation applied. */ +@property(nonatomic, readonly) int height; +@property(nonatomic, readonly) RTCVideoRotation rotation; + +/** Timestamp in nanoseconds. */ +@property(nonatomic, readonly) int64_t timeStampNs; + +/** Timestamp 90 kHz. */ +@property(nonatomic, assign) int32_t timeStamp; + +@property(nonatomic, readonly) id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> buffer; + +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)new NS_UNAVAILABLE; + +/** Initialize an RTCVideoFrame from a pixel buffer, rotation, and timestamp. + * Deprecated - initialize with a RTCCVPixelBuffer instead + */ +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs + DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); + +/** Initialize an RTCVideoFrame from a pixel buffer combined with cropping and + * scaling. Cropping will be applied first on the pixel buffer, followed by + * scaling to the final resolution of scaledWidth x scaledHeight. + */ +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + scaledWidth:(int)scaledWidth + scaledHeight:(int)scaledHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs + DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); + +/** Initialize an RTCVideoFrame from a frame buffer, rotation, and timestamp. + */ +- (instancetype)initWithBuffer:(id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)frameBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs; + +/** Return a frame that is guaranteed to be I420, i.e. it is possible to access + * the YUV data on it. + */ +- (RTC_OBJC_TYPE(RTCVideoFrame) *)newI420VideoFrame; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.mm b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.mm new file mode 100644 index 0000000000..e162238d73 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrame.mm @@ -0,0 +1,78 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoFrame.h" + +#import "RTCI420Buffer.h" +#import "RTCVideoFrameBuffer.h" + +@implementation RTC_OBJC_TYPE (RTCVideoFrame) { + RTCVideoRotation _rotation; + int64_t _timeStampNs; +} + +@synthesize buffer = _buffer; +@synthesize timeStamp; + +- (int)width { + return _buffer.width; +} + +- (int)height { + return _buffer.height; +} + +- (RTCVideoRotation)rotation { + return _rotation; +} + +- (int64_t)timeStampNs { + return _timeStampNs; +} + +- (RTC_OBJC_TYPE(RTCVideoFrame) *)newI420VideoFrame { + return [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:[_buffer toI420] + rotation:_rotation + timeStampNs:_timeStampNs]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { + // Deprecated. + return nil; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + scaledWidth:(int)scaledWidth + scaledHeight:(int)scaledHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { + // Deprecated. + return nil; +} + +- (instancetype)initWithBuffer:(id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)buffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { + if (self = [super init]) { + _buffer = buffer; + _rotation = rotation; + _timeStampNs = timeStampNs; + } + + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoFrameBuffer.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrameBuffer.h new file mode 100644 index 0000000000..a36807951b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoFrameBuffer.h @@ -0,0 +1,40 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@protocol RTC_OBJC_TYPE +(RTCI420Buffer); + +// RTCVideoFrameBuffer is an ObjectiveC version of webrtc::VideoFrameBuffer. +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoFrameBuffer)<NSObject> + + @property(nonatomic, readonly) int width; +@property(nonatomic, readonly) int height; + +- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420; + +@optional +- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCVideoRenderer.h b/third_party/libwebrtc/sdk/objc/base/RTCVideoRenderer.h new file mode 100644 index 0000000000..0f763295ad --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCVideoRenderer.h @@ -0,0 +1,43 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#if TARGET_OS_IPHONE +#import <UIKit/UIKit.h> +#endif + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCVideoFrame); + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoRenderer)<NSObject> + + /** The size of the frame. */ + - (void)setSize : (CGSize)size; + +/** The frame to be displayed. */ +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +@end + +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoViewDelegate) + + - (void)videoView : (id<RTC_OBJC_TYPE(RTCVideoRenderer)>)videoView didChangeVideoSize + : (CGSize)size; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/base/RTCYUVPlanarBuffer.h b/third_party/libwebrtc/sdk/objc/base/RTCYUVPlanarBuffer.h new file mode 100644 index 0000000000..be01b915f5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/base/RTCYUVPlanarBuffer.h @@ -0,0 +1,46 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrameBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Protocol for RTCVideoFrameBuffers containing YUV planar data. */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCYUVPlanarBuffer)<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> + + @property(nonatomic, readonly) int chromaWidth; +@property(nonatomic, readonly) int chromaHeight; +@property(nonatomic, readonly) const uint8_t *dataY; +@property(nonatomic, readonly) const uint8_t *dataU; +@property(nonatomic, readonly) const uint8_t *dataV; +@property(nonatomic, readonly) int strideY; +@property(nonatomic, readonly) int strideU; +@property(nonatomic, readonly) int strideV; + +- (instancetype)initWithWidth:(int)width + height:(int)height + dataY:(const uint8_t *)dataY + dataU:(const uint8_t *)dataU + dataV:(const uint8_t *)dataV; +- (instancetype)initWithWidth:(int)width height:(int)height; +- (instancetype)initWithWidth:(int)width + height:(int)height + strideY:(int)strideY + strideU:(int)strideU + strideV:(int)strideV; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioDevice.h b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioDevice.h new file mode 100644 index 0000000000..f445825ff0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioDevice.h @@ -0,0 +1,308 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AudioUnit/AudioUnit.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceGetPlayoutDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + AudioBufferList *_Nonnull outputData); + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + AudioBufferList *_Nonnull inputData, + void *_Nullable renderContext); + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceDeliverRecordedDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + const AudioBufferList *_Nullable inputData, + void *_Nullable renderContext, + NS_NOESCAPE RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) _Nullable renderBlock); + +/** + * Delegate object provided by native ADM during RTCAudioDevice initialization. + * Provides blocks to poll playback audio samples from native ADM and to feed + * recorded audio samples into native ADM. + */ +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioDeviceDelegate)<NSObject> + /** + * Implementation of RTCAudioSource should call this block to feed recorded PCM (16-bit integer) + * into native ADM. Stereo data is expected to be interleaved starting with the left channel. + * Either `inputData` with pre-filled audio data must be provided during block + * call or `renderBlock` must be provided which must fill provided audio buffer with recorded + * samples. + * + * NOTE: Implementation of RTCAudioDevice is expected to call the block on the same thread until + * `notifyAudioInterrupted` is called. When `notifyAudioInterrupted` is called implementation + * can call the block on a different thread. + */ + @property(readonly, nonnull) + RTC_OBJC_TYPE(RTCAudioDeviceDeliverRecordedDataBlock) deliverRecordedData; + +/** + * Provides input sample rate preference as it preferred by native ADM. + */ +@property(readonly) double preferredInputSampleRate; + +/** + * Provides input IO buffer duration preference as it preferred by native ADM. + */ +@property(readonly) NSTimeInterval preferredInputIOBufferDuration; + +/** + * Provides output sample rate preference as it preferred by native ADM. + */ +@property(readonly) double preferredOutputSampleRate; + +/** + * Provides output IO buffer duration preference as it preferred by native ADM. + */ +@property(readonly) NSTimeInterval preferredOutputIOBufferDuration; + +/** + * Implementation of RTCAudioDevice should call this block to request PCM (16-bit integer) + * from native ADM to play. Stereo data is interleaved starting with the left channel. + * + * NOTE: Implementation of RTCAudioDevice is expected to invoke of this block on the + * same thread until `notifyAudioInterrupted` is called. When `notifyAudioInterrupted` is called + * implementation can call the block from a different thread. + */ +@property(readonly, nonnull) RTC_OBJC_TYPE(RTCAudioDeviceGetPlayoutDataBlock) getPlayoutData; + +/** + * Notifies native ADM that some of the audio input parameters of RTCAudioDevice like + * samle rate and/or IO buffer duration and/or IO latency had possibly changed. + * Native ADM will adjust its audio input buffer to match current parameters of audio device. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioInputParametersChange; + +/** + * Notifies native ADM that some of the audio output parameters of RTCAudioDevice like + * samle rate and/or IO buffer duration and/or IO latency had possibly changed. + * Native ADM will adjust its audio output buffer to match current parameters of audio device. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioOutputParametersChange; + +/** + * Notifies native ADM that audio input is interrupted and further audio playout + * and recording might happen on a different thread. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioInputInterrupted; + +/** + * Notifies native ADM that audio output is interrupted and further audio playout + * and recording might happen on a different thread. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioOutputInterrupted; + +/** + * Asynchronously execute block of code within the context of + * thread which owns native ADM. + * + * NOTE: Intended to be used to invoke `notifyAudioInputParametersChange`, + * `notifyAudioOutputParametersChange`, `notifyAudioInputInterrupted`, + * `notifyAudioOutputInterrupted` on native ADM thread. + * Also could be used by `RTCAudioDevice` implementation to tie + * mutations of underlying audio objects (AVAudioEngine, AudioUnit, etc) + * to the native ADM thread. Could be useful to handle events like audio route change, which + * could lead to audio parameters change. + */ +- (void)dispatchAsync:(dispatch_block_t)block; + +/** + * Synchronously execute block of code within the context of + * thread which owns native ADM. Allows reentrancy. + * + * NOTE: Intended to be used to invoke `notifyAudioInputParametersChange`, + * `notifyAudioOutputParametersChange`, `notifyAudioInputInterrupted`, + * `notifyAudioOutputInterrupted` on native ADM thread and make sure + * aforementioned is completed before `dispatchSync` returns. Could be useful + * when implementation of `RTCAudioDevice` tie mutation to underlying audio objects (AVAudioEngine, + * AudioUnit, etc) to own thread to satisfy requirement that native ADM audio parameters + * must be kept in sync with current audio parameters before audio is actually played or recorded. + */ +- (void)dispatchSync:(dispatch_block_t)block; + +@end + +/** + * Protocol to abstract platform specific ways to implement playback and recording. + * + * NOTE: All the members of protocol are called by native ADM from the same thread + * between calls to `initializeWithDelegate` and `terminate`. + * NOTE: Implementation is fully responsible for configuring application's AVAudioSession. + * An example implementation of RTCAudioDevice: https://github.com/mstyura/RTCAudioDevice + * TODO(yura.yaroshevich): Implement custom RTCAudioDevice for AppRTCMobile demo app. + */ +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioDevice)<NSObject> + + /** + * Indicates current sample rate of audio recording. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate + * notifyAudioParametersChange]`. + */ + @property(readonly) double deviceInputSampleRate; + +/** + * Indicates current size of record buffer. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSTimeInterval inputIOBufferDuration; + +/** + * Indicates current number of recorded audio channels. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSInteger inputNumberOfChannels; + +/** + * Indicates current input latency + */ +@property(readonly) NSTimeInterval inputLatency; + +/** + * Indicates current sample rate of audio playback. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) double deviceOutputSampleRate; + +/** + * Indicates current size of playback buffer. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSTimeInterval outputIOBufferDuration; + +/** + * Indicates current number of playback audio channels. Changes to this property + * must be notified back to WebRTC via `[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSInteger outputNumberOfChannels; + +/** + * Indicates current output latency + */ +@property(readonly) NSTimeInterval outputLatency; + +/** + * Indicates if invocation of `initializeWithDelegate` required before usage of RTCAudioDevice. + * YES indicates that `initializeWithDelegate` was called earlier without subsequent call to + * `terminate`. NO indicates that either `initializeWithDelegate` not called or `terminate` called. + */ +@property(readonly) BOOL isInitialized; + +/** + * Initializes RTCAudioDevice with RTCAudioDeviceDelegate. + * Implementation must return YES if RTCAudioDevice initialized successfully and NO otherwise. + */ +- (BOOL)initializeWithDelegate:(id<RTC_OBJC_TYPE(RTCAudioDeviceDelegate)>)delegate; + +/** + * De-initializes RTCAudioDevice. Implementation should forget about `delegate` provided in + * `initializeWithDelegate`. + */ +- (BOOL)terminateDevice; + +/** + * Property to indicate if `initializePlayout` call required before invocation of `startPlayout`. + * YES indicates that `initializePlayout` was successfully invoked earlier or not necessary, + * NO indicates that `initializePlayout` invocation required. + */ +@property(readonly) BOOL isPlayoutInitialized; + +/** + * Prepares RTCAudioDevice to play audio. + * Called by native ADM before invocation of `startPlayout`. + * Implementation is expected to return YES in case of successful playout initialization and NO + * otherwise. + */ +- (BOOL)initializePlayout; + +/** + * Property to indicate if RTCAudioDevice should be playing according to + * earlier calls of `startPlayout` and `stopPlayout`. + */ +@property(readonly) BOOL isPlaying; + +/** + * Method is called when native ADM wants to play audio. + * Implementation is expected to return YES if playback start request + * successfully handled and NO otherwise. + */ +- (BOOL)startPlayout; + +/** + * Method is called when native ADM no longer needs to play audio. + * Implementation is expected to return YES if playback stop request + * successfully handled and NO otherwise. + */ +- (BOOL)stopPlayout; + +/** + * Property to indicate if `initializeRecording` call required before usage of `startRecording`. + * YES indicates that `initializeRecording` was successfully invoked earlier or not necessary, + * NO indicates that `initializeRecording` invocation required. + */ +@property(readonly) BOOL isRecordingInitialized; + +/** + * Prepares RTCAudioDevice to record audio. + * Called by native ADM before invocation of `startRecording`. + * Implementation may use this method to prepare resources required to record audio. + * Implementation is expected to return YES in case of successful record initialization and NO + * otherwise. + */ +- (BOOL)initializeRecording; + +/** + * Property to indicate if RTCAudioDevice should record audio according to + * earlier calls to `startRecording` and `stopRecording`. + */ +@property(readonly) BOOL isRecording; + +/** + * Method is called when native ADM wants to record audio. + * Implementation is expected to return YES if recording start request + * successfully handled and NO otherwise. + */ +- (BOOL)startRecording; + +/** + * Method is called when native ADM no longer needs to record audio. + * Implementation is expected to return YES if recording stop request + * successfully handled and NO otherwise. + */ +- (BOOL)stopRecording; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Configuration.mm b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Configuration.mm new file mode 100644 index 0000000000..4bf0a542e1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Configuration.mm @@ -0,0 +1,150 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSession+Private.h" +#import "RTCAudioSessionConfiguration.h" + +#import "base/RTCLogging.h" + +@implementation RTC_OBJC_TYPE (RTCAudioSession) +(Configuration) + + - (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error + : (NSError **)outError { + return [self setConfiguration:configuration + active:NO + shouldSetActive:NO + error:outError]; +} + +- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration + active:(BOOL)active + error:(NSError **)outError { + return [self setConfiguration:configuration + active:active + shouldSetActive:YES + error:outError]; +} + +#pragma mark - Private + +- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration + active:(BOOL)active + shouldSetActive:(BOOL)shouldSetActive + error:(NSError **)outError { + NSParameterAssert(configuration); + if (outError) { + *outError = nil; + } + + // Provide an error even if there isn't one so we can log it. We will not + // return immediately on error in this function and instead try to set + // everything we can. + NSError *error = nil; + + if (self.category != configuration.category || self.mode != configuration.mode || + self.categoryOptions != configuration.categoryOptions) { + NSError *configuringError = nil; + if (![self setCategory:configuration.category + mode:configuration.mode + options:configuration.categoryOptions + error:&configuringError]) { + RTCLogError(@"Failed to set category and mode: %@", configuringError.localizedDescription); + error = configuringError; + } else { + RTCLog(@"Set category to: %@, mode: %@", configuration.category, configuration.mode); + } + } + + if (self.preferredSampleRate != configuration.sampleRate) { + NSError *sampleRateError = nil; + if (![self setPreferredSampleRate:configuration.sampleRate + error:&sampleRateError]) { + RTCLogError(@"Failed to set preferred sample rate: %@", + sampleRateError.localizedDescription); + if (!self.ignoresPreferredAttributeConfigurationErrors) { + error = sampleRateError; + } + } else { + RTCLog(@"Set preferred sample rate to: %.2f", + configuration.sampleRate); + } + } + + if (self.preferredIOBufferDuration != configuration.ioBufferDuration) { + NSError *bufferDurationError = nil; + if (![self setPreferredIOBufferDuration:configuration.ioBufferDuration + error:&bufferDurationError]) { + RTCLogError(@"Failed to set preferred IO buffer duration: %@", + bufferDurationError.localizedDescription); + if (!self.ignoresPreferredAttributeConfigurationErrors) { + error = bufferDurationError; + } + } else { + RTCLog(@"Set preferred IO buffer duration to: %f", + configuration.ioBufferDuration); + } + } + + if (shouldSetActive) { + NSError *activeError = nil; + if (![self setActive:active error:&activeError]) { + RTCLogError(@"Failed to setActive to %d: %@", + active, activeError.localizedDescription); + error = activeError; + } + } + + if (self.isActive && + // TODO(tkchin): Figure out which category/mode numChannels is valid for. + [self.mode isEqualToString:AVAudioSessionModeVoiceChat]) { + // Try to set the preferred number of hardware audio channels. These calls + // must be done after setting the audio session’s category and mode and + // activating the session. + NSInteger inputNumberOfChannels = configuration.inputNumberOfChannels; + if (self.inputNumberOfChannels != inputNumberOfChannels) { + NSError *inputChannelsError = nil; + if (![self setPreferredInputNumberOfChannels:inputNumberOfChannels + error:&inputChannelsError]) { + RTCLogError(@"Failed to set preferred input number of channels: %@", + inputChannelsError.localizedDescription); + if (!self.ignoresPreferredAttributeConfigurationErrors) { + error = inputChannelsError; + } + } else { + RTCLog(@"Set input number of channels to: %ld", + (long)inputNumberOfChannels); + } + } + NSInteger outputNumberOfChannels = configuration.outputNumberOfChannels; + if (self.outputNumberOfChannels != outputNumberOfChannels) { + NSError *outputChannelsError = nil; + if (![self setPreferredOutputNumberOfChannels:outputNumberOfChannels + error:&outputChannelsError]) { + RTCLogError(@"Failed to set preferred output number of channels: %@", + outputChannelsError.localizedDescription); + if (!self.ignoresPreferredAttributeConfigurationErrors) { + error = outputChannelsError; + } + } else { + RTCLog(@"Set output number of channels to: %ld", + (long)outputNumberOfChannels); + } + } + } + + if (outError) { + *outError = error; + } + + return error == nil; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Private.h b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Private.h new file mode 100644 index 0000000000..2be1b9fb3d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession+Private.h @@ -0,0 +1,95 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSession.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration); + +@interface RTC_OBJC_TYPE (RTCAudioSession) +() + + /** Number of times setActive:YES has succeeded without a balanced call to + * setActive:NO. + */ + @property(nonatomic, readonly) int activationCount; + +/** The number of times `beginWebRTCSession` was called without a balanced call + * to `endWebRTCSession`. + */ +@property(nonatomic, readonly) int webRTCSessionCount; + +/** Convenience BOOL that checks useManualAudio and isAudioEnebled. */ +@property(readonly) BOOL canPlayOrRecord; + +/** Tracks whether we have been sent an interruption event that hasn't been matched by either an + * interrupted end event or a foreground event. + */ +@property(nonatomic, assign) BOOL isInterrupted; + +/** Adds the delegate to the list of delegates, and places it at the front of + * the list. This delegate will be notified before other delegates of + * audio events. + */ +- (void)pushDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate; + +/** Signals RTCAudioSession that a WebRTC session is about to begin and + * audio configuration is needed. Will configure the audio session for WebRTC + * if not already configured and if configuration is not delayed. + * Successful calls must be balanced by a call to endWebRTCSession. + */ +- (BOOL)beginWebRTCSession:(NSError **)outError; + +/** Signals RTCAudioSession that a WebRTC session is about to end and audio + * unconfiguration is needed. Will unconfigure the audio session for WebRTC + * if this is the last unmatched call and if configuration is not delayed. + */ +- (BOOL)endWebRTCSession:(NSError **)outError; + +/** Configure the audio session for WebRTC. This call will fail if the session + * is already configured. On other failures, we will attempt to restore the + * previously used audio session configuration. + * `lockForConfiguration` must be called first. + * Successful calls to configureWebRTCSession must be matched by calls to + * `unconfigureWebRTCSession`. + */ +- (BOOL)configureWebRTCSession:(NSError **)outError; + +/** Unconfigures the session for WebRTC. This will attempt to restore the + * audio session to the settings used before `configureWebRTCSession` was + * called. + * `lockForConfiguration` must be called first. + */ +- (BOOL)unconfigureWebRTCSession:(NSError **)outError; + +/** Returns a configuration error with the given description. */ +- (NSError *)configurationErrorWithDescription:(NSString *)description; + +/** Notifies the receiver that a playout glitch was detected. */ +- (void)notifyDidDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches; + +/** Notifies the receiver that there was an error when starting an audio unit. */ +- (void)notifyAudioUnitStartFailedWithError:(OSStatus)error; + +// Properties and methods for tests. +- (void)notifyDidBeginInterruption; +- (void)notifyDidEndInterruptionWithShouldResumeSession:(BOOL)shouldResumeSession; +- (void)notifyDidChangeRouteWithReason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute; +- (void)notifyMediaServicesWereLost; +- (void)notifyMediaServicesWereReset; +- (void)notifyDidChangeCanPlayOrRecord:(BOOL)canPlayOrRecord; +- (void)notifyDidStartPlayOrRecord; +- (void)notifyDidStopPlayOrRecord; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.h b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.h new file mode 100644 index 0000000000..2730664858 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.h @@ -0,0 +1,269 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +extern NSString *const kRTCAudioSessionErrorDomain; +/** Method that requires lock was called without lock. */ +extern NSInteger const kRTCAudioSessionErrorLockRequired; +/** Unknown configuration error occurred. */ +extern NSInteger const kRTCAudioSessionErrorConfiguration; + +@class RTC_OBJC_TYPE(RTCAudioSession); +@class RTC_OBJC_TYPE(RTCAudioSessionConfiguration); + +// Surfaces AVAudioSession events. WebRTC will listen directly for notifications +// from AVAudioSession and handle them before calling these delegate methods, +// at which point applications can perform additional processing if required. +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCAudioSessionDelegate)<NSObject> + + @optional +/** Called on a system notification thread when AVAudioSession starts an + * interruption event. + */ +- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session; + +/** Called on a system notification thread when AVAudioSession ends an + * interruption event. + */ +- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session + shouldResumeSession:(BOOL)shouldResumeSession; + +/** Called on a system notification thread when AVAudioSession changes the + * route. + */ +- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session + reason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute; + +/** Called on a system notification thread when AVAudioSession media server + * terminates. + */ +- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session; + +/** Called on a system notification thread when AVAudioSession media server + * restarts. + */ +- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session; + +// TODO(tkchin): Maybe handle SilenceSecondaryAudioHintNotification. + +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session + didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord; + +/** Called on a WebRTC thread when the audio device is notified to begin + * playback or recording. + */ +- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session; + +/** Called on a WebRTC thread when the audio device is notified to stop + * playback or recording. + */ +- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session; + +/** Called when the AVAudioSession output volume value changes. */ +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + didChangeOutputVolume:(float)outputVolume; + +/** Called when the audio device detects a playout glitch. The argument is the + * number of glitches detected so far in the current audio playout session. + */ +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + didDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches; + +/** Called when the audio session is about to change the active state. + */ +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession willSetActive:(BOOL)active; + +/** Called after the audio session sucessfully changed the active state. + */ +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession didSetActive:(BOOL)active; + +/** Called after the audio session failed to change the active state. + */ +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + failedToSetActive:(BOOL)active + error:(NSError *)error; + +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + audioUnitStartFailedWithError:(NSError *)error; + +@end + +/** This is a protocol used to inform RTCAudioSession when the audio session + * activation state has changed outside of RTCAudioSession. The current known use + * case of this is when CallKit activates the audio session for the application + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCAudioSessionActivationDelegate)<NSObject> + + /** Called when the audio session is activated outside of the app by iOS. */ + - (void)audioSessionDidActivate : (AVAudioSession *)session; + +/** Called when the audio session is deactivated outside of the app by iOS. */ +- (void)audioSessionDidDeactivate:(AVAudioSession *)session; + +@end + +/** Proxy class for AVAudioSession that adds a locking mechanism similar to + * AVCaptureDevice. This is used to that interleaving configurations between + * WebRTC and the application layer are avoided. + * + * RTCAudioSession also coordinates activation so that the audio session is + * activated only once. See `setActive:error:`. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioSession) : NSObject <RTC_OBJC_TYPE(RTCAudioSessionActivationDelegate)> + +/** Convenience property to access the AVAudioSession singleton. Callers should + * not call setters on AVAudioSession directly, but other method invocations + * are fine. + */ +@property(nonatomic, readonly) AVAudioSession *session; + +/** Our best guess at whether the session is active based on results of calls to + * AVAudioSession. + */ +@property(nonatomic, readonly) BOOL isActive; + +/** If YES, WebRTC will not initialize the audio unit automatically when an + * audio track is ready for playout or recording. Instead, applications should + * call setIsAudioEnabled. If NO, WebRTC will initialize the audio unit + * as soon as an audio track is ready for playout or recording. + */ +@property(nonatomic, assign) BOOL useManualAudio; + +/** This property is only effective if useManualAudio is YES. + * Represents permission for WebRTC to initialize the VoIP audio unit. + * When set to NO, if the VoIP audio unit used by WebRTC is active, it will be + * stopped and uninitialized. This will stop incoming and outgoing audio. + * When set to YES, WebRTC will initialize and start the audio unit when it is + * needed (e.g. due to establishing an audio connection). + * This property was introduced to work around an issue where if an AVPlayer is + * playing audio while the VoIP audio unit is initialized, its audio would be + * either cut off completely or played at a reduced volume. By preventing + * the audio unit from being initialized until after the audio has completed, + * we are able to prevent the abrupt cutoff. + */ +@property(nonatomic, assign) BOOL isAudioEnabled; + +// Proxy properties. +@property(readonly) NSString *category; +@property(readonly) AVAudioSessionCategoryOptions categoryOptions; +@property(readonly) NSString *mode; +@property(readonly) BOOL secondaryAudioShouldBeSilencedHint; +@property(readonly) AVAudioSessionRouteDescription *currentRoute; +@property(readonly) NSInteger maximumInputNumberOfChannels; +@property(readonly) NSInteger maximumOutputNumberOfChannels; +@property(readonly) float inputGain; +@property(readonly) BOOL inputGainSettable; +@property(readonly) BOOL inputAvailable; +@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *inputDataSources; +@property(readonly, nullable) AVAudioSessionDataSourceDescription *inputDataSource; +@property(readonly, nullable) NSArray<AVAudioSessionDataSourceDescription *> *outputDataSources; +@property(readonly, nullable) AVAudioSessionDataSourceDescription *outputDataSource; +@property(readonly) double sampleRate; +@property(readonly) double preferredSampleRate; +@property(readonly) NSInteger inputNumberOfChannels; +@property(readonly) NSInteger outputNumberOfChannels; +@property(readonly) float outputVolume; +@property(readonly) NSTimeInterval inputLatency; +@property(readonly) NSTimeInterval outputLatency; +@property(readonly) NSTimeInterval IOBufferDuration; +@property(readonly) NSTimeInterval preferredIOBufferDuration; + +/** + When YES, calls to -setConfiguration:error: and -setConfiguration:active:error: ignore errors in + configuring the audio session's "preferred" attributes (e.g. preferredInputNumberOfChannels). + Typically, configurations to preferred attributes are optimizations, and ignoring this type of + configuration error allows code flow to continue along the happy path when these optimization are + not available. The default value of this property is NO. + */ +@property(nonatomic) BOOL ignoresPreferredAttributeConfigurationErrors; + +/** Default constructor. */ ++ (instancetype)sharedInstance; +- (instancetype)init NS_UNAVAILABLE; + +/** Adds a delegate, which is held weakly. */ +- (void)addDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate; +/** Removes an added delegate. */ +- (void)removeDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate; + +/** Request exclusive access to the audio session for configuration. This call + * will block if the lock is held by another object. + */ +- (void)lockForConfiguration; +/** Relinquishes exclusive access to the audio session. */ +- (void)unlockForConfiguration; + +/** If `active`, activates the audio session if it isn't already active. + * Successful calls must be balanced with a setActive:NO when activation is no + * longer required. If not `active`, deactivates the audio session if one is + * active and this is the last balanced call. When deactivating, the + * AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation option is passed to + * AVAudioSession. + */ +- (BOOL)setActive:(BOOL)active error:(NSError **)outError; + +// The following methods are proxies for the associated methods on +// AVAudioSession. `lockForConfiguration` must be called before using them +// otherwise they will fail with kRTCAudioSessionErrorLockRequired. + +- (BOOL)setCategory:(AVAudioSessionCategory)category + mode:(AVAudioSessionMode)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError; +- (BOOL)setCategory:(AVAudioSessionCategory)category + withOptions:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError; +- (BOOL)setMode:(AVAudioSessionMode)mode error:(NSError **)outError; +- (BOOL)setInputGain:(float)gain error:(NSError **)outError; +- (BOOL)setPreferredSampleRate:(double)sampleRate error:(NSError **)outError; +- (BOOL)setPreferredIOBufferDuration:(NSTimeInterval)duration error:(NSError **)outError; +- (BOOL)setPreferredInputNumberOfChannels:(NSInteger)count error:(NSError **)outError; +- (BOOL)setPreferredOutputNumberOfChannels:(NSInteger)count error:(NSError **)outError; +- (BOOL)overrideOutputAudioPort:(AVAudioSessionPortOverride)portOverride error:(NSError **)outError; +- (BOOL)setPreferredInput:(AVAudioSessionPortDescription *)inPort error:(NSError **)outError; +- (BOOL)setInputDataSource:(AVAudioSessionDataSourceDescription *)dataSource + error:(NSError **)outError; +- (BOOL)setOutputDataSource:(AVAudioSessionDataSourceDescription *)dataSource + error:(NSError **)outError; +@end + +@interface RTC_OBJC_TYPE (RTCAudioSession) +(Configuration) + + /** Applies the configuration to the current session. Attempts to set all + * properties even if previous ones fail. Only the last error will be + * returned. + * `lockForConfiguration` must be called first. + */ + - (BOOL)setConfiguration : (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration error + : (NSError **)outError; + +/** Convenience method that calls both setConfiguration and setActive. + * `lockForConfiguration` must be called first. + */ +- (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration + active:(BOOL)active + error:(NSError **)outError; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.mm b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.mm new file mode 100644 index 0000000000..f6b91d5409 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSession.mm @@ -0,0 +1,1010 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSession+Private.h" + +#import <UIKit/UIKit.h> + +#include <atomic> +#include <vector> + +#include "absl/base/attributes.h" +#include "rtc_base/checks.h" +#include "rtc_base/synchronization/mutex.h" + +#import "RTCAudioSessionConfiguration.h" +#import "base/RTCLogging.h" + +#if !defined(ABSL_HAVE_THREAD_LOCAL) +#error ABSL_HAVE_THREAD_LOCAL should be defined for MacOS / iOS Targets. +#endif + +NSString *const kRTCAudioSessionErrorDomain = @"org.webrtc.RTC_OBJC_TYPE(RTCAudioSession)"; +NSInteger const kRTCAudioSessionErrorLockRequired = -1; +NSInteger const kRTCAudioSessionErrorConfiguration = -2; +NSString * const kRTCAudioSessionOutputVolumeSelector = @"outputVolume"; + +namespace { +// Since webrtc::Mutex is not a reentrant lock and cannot check if the mutex is locked, +// we need a separate variable to check that the mutex is locked in the RTCAudioSession. +ABSL_CONST_INIT thread_local bool mutex_locked = false; +} // namespace + +@interface RTC_OBJC_TYPE (RTCAudioSession) +() @property(nonatomic, + readonly) std::vector<__weak id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)> > delegates; +@end + +// This class needs to be thread-safe because it is accessed from many threads. +// TODO(tkchin): Consider more granular locking. We're not expecting a lot of +// lock contention so coarse locks should be fine for now. +@implementation RTC_OBJC_TYPE (RTCAudioSession) { + webrtc::Mutex _mutex; + AVAudioSession *_session; + std::atomic<int> _activationCount; + std::atomic<int> _webRTCSessionCount; + BOOL _isActive; + BOOL _useManualAudio; + BOOL _isAudioEnabled; + BOOL _canPlayOrRecord; + BOOL _isInterrupted; +} + +@synthesize session = _session; +@synthesize delegates = _delegates; +@synthesize ignoresPreferredAttributeConfigurationErrors = + _ignoresPreferredAttributeConfigurationErrors; + ++ (instancetype)sharedInstance { + static dispatch_once_t onceToken; + static RTC_OBJC_TYPE(RTCAudioSession) *sharedInstance = nil; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + +- (instancetype)init { + return [self initWithAudioSession:[AVAudioSession sharedInstance]]; +} + +/** This initializer provides a way for unit tests to inject a fake/mock audio session. */ +- (instancetype)initWithAudioSession:(id)audioSession { + if (self = [super init]) { + _session = audioSession; + + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + [center addObserver:self + selector:@selector(handleInterruptionNotification:) + name:AVAudioSessionInterruptionNotification + object:nil]; + [center addObserver:self + selector:@selector(handleRouteChangeNotification:) + name:AVAudioSessionRouteChangeNotification + object:nil]; + [center addObserver:self + selector:@selector(handleMediaServicesWereLost:) + name:AVAudioSessionMediaServicesWereLostNotification + object:nil]; + [center addObserver:self + selector:@selector(handleMediaServicesWereReset:) + name:AVAudioSessionMediaServicesWereResetNotification + object:nil]; + // Posted on the main thread when the primary audio from other applications + // starts and stops. Foreground applications may use this notification as a + // hint to enable or disable audio that is secondary. + [center addObserver:self + selector:@selector(handleSilenceSecondaryAudioHintNotification:) + name:AVAudioSessionSilenceSecondaryAudioHintNotification + object:nil]; + // Also track foreground event in order to deal with interruption ended situation. + [center addObserver:self + selector:@selector(handleApplicationDidBecomeActive:) + name:UIApplicationDidBecomeActiveNotification + object:nil]; + [_session addObserver:self + forKeyPath:kRTCAudioSessionOutputVolumeSelector + options:NSKeyValueObservingOptionNew | NSKeyValueObservingOptionOld + context:(__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class]; + + RTCLog(@"RTC_OBJC_TYPE(RTCAudioSession) (%p): init.", self); + } + return self; +} + +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; + [_session removeObserver:self + forKeyPath:kRTCAudioSessionOutputVolumeSelector + context:(__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class]; + RTCLog(@"RTC_OBJC_TYPE(RTCAudioSession) (%p): dealloc.", self); +} + +- (NSString *)description { + NSString *format = @"RTC_OBJC_TYPE(RTCAudioSession): {\n" + " category: %@\n" + " categoryOptions: %ld\n" + " mode: %@\n" + " isActive: %d\n" + " sampleRate: %.2f\n" + " IOBufferDuration: %f\n" + " outputNumberOfChannels: %ld\n" + " inputNumberOfChannels: %ld\n" + " outputLatency: %f\n" + " inputLatency: %f\n" + " outputVolume: %f\n" + "}"; + NSString *description = [NSString stringWithFormat:format, + self.category, (long)self.categoryOptions, self.mode, + self.isActive, self.sampleRate, self.IOBufferDuration, + self.outputNumberOfChannels, self.inputNumberOfChannels, + self.outputLatency, self.inputLatency, self.outputVolume]; + return description; +} + +- (void)setIsActive:(BOOL)isActive { + @synchronized(self) { + _isActive = isActive; + } +} + +- (BOOL)isActive { + @synchronized(self) { + return _isActive; + } +} + +- (void)setUseManualAudio:(BOOL)useManualAudio { + @synchronized(self) { + if (_useManualAudio == useManualAudio) { + return; + } + _useManualAudio = useManualAudio; + } + [self updateCanPlayOrRecord]; +} + +- (BOOL)useManualAudio { + @synchronized(self) { + return _useManualAudio; + } +} + +- (void)setIsAudioEnabled:(BOOL)isAudioEnabled { + @synchronized(self) { + if (_isAudioEnabled == isAudioEnabled) { + return; + } + _isAudioEnabled = isAudioEnabled; + } + [self updateCanPlayOrRecord]; +} + +- (BOOL)isAudioEnabled { + @synchronized(self) { + return _isAudioEnabled; + } +} + +- (void)setIgnoresPreferredAttributeConfigurationErrors: + (BOOL)ignoresPreferredAttributeConfigurationErrors { + @synchronized(self) { + if (_ignoresPreferredAttributeConfigurationErrors == + ignoresPreferredAttributeConfigurationErrors) { + return; + } + _ignoresPreferredAttributeConfigurationErrors = ignoresPreferredAttributeConfigurationErrors; + } +} + +- (BOOL)ignoresPreferredAttributeConfigurationErrors { + @synchronized(self) { + return _ignoresPreferredAttributeConfigurationErrors; + } +} + +// TODO(tkchin): Check for duplicates. +- (void)addDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate { + RTCLog(@"Adding delegate: (%p)", delegate); + if (!delegate) { + return; + } + @synchronized(self) { + _delegates.push_back(delegate); + [self removeZeroedDelegates]; + } +} + +- (void)removeDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate { + RTCLog(@"Removing delegate: (%p)", delegate); + if (!delegate) { + return; + } + @synchronized(self) { + _delegates.erase(std::remove(_delegates.begin(), + _delegates.end(), + delegate), + _delegates.end()); + [self removeZeroedDelegates]; + } +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wthread-safety-analysis" + +- (void)lockForConfiguration { + RTC_CHECK(!mutex_locked); + _mutex.Lock(); + mutex_locked = true; +} + +- (void)unlockForConfiguration { + mutex_locked = false; + _mutex.Unlock(); +} + +#pragma clang diagnostic pop + +#pragma mark - AVAudioSession proxy methods + +- (NSString *)category { + return self.session.category; +} + +- (AVAudioSessionCategoryOptions)categoryOptions { + return self.session.categoryOptions; +} + +- (NSString *)mode { + return self.session.mode; +} + +- (BOOL)secondaryAudioShouldBeSilencedHint { + return self.session.secondaryAudioShouldBeSilencedHint; +} + +- (AVAudioSessionRouteDescription *)currentRoute { + return self.session.currentRoute; +} + +- (NSInteger)maximumInputNumberOfChannels { + return self.session.maximumInputNumberOfChannels; +} + +- (NSInteger)maximumOutputNumberOfChannels { + return self.session.maximumOutputNumberOfChannels; +} + +- (float)inputGain { + return self.session.inputGain; +} + +- (BOOL)inputGainSettable { + return self.session.inputGainSettable; +} + +- (BOOL)inputAvailable { + return self.session.inputAvailable; +} + +- (NSArray<AVAudioSessionDataSourceDescription *> *)inputDataSources { + return self.session.inputDataSources; +} + +- (AVAudioSessionDataSourceDescription *)inputDataSource { + return self.session.inputDataSource; +} + +- (NSArray<AVAudioSessionDataSourceDescription *> *)outputDataSources { + return self.session.outputDataSources; +} + +- (AVAudioSessionDataSourceDescription *)outputDataSource { + return self.session.outputDataSource; +} + +- (double)sampleRate { + return self.session.sampleRate; +} + +- (double)preferredSampleRate { + return self.session.preferredSampleRate; +} + +- (NSInteger)inputNumberOfChannels { + return self.session.inputNumberOfChannels; +} + +- (NSInteger)outputNumberOfChannels { + return self.session.outputNumberOfChannels; +} + +- (float)outputVolume { + return self.session.outputVolume; +} + +- (NSTimeInterval)inputLatency { + return self.session.inputLatency; +} + +- (NSTimeInterval)outputLatency { + return self.session.outputLatency; +} + +- (NSTimeInterval)IOBufferDuration { + return self.session.IOBufferDuration; +} + +- (NSTimeInterval)preferredIOBufferDuration { + return self.session.preferredIOBufferDuration; +} + +- (BOOL)setActive:(BOOL)active + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + int activationCount = _activationCount.load(); + if (!active && activationCount == 0) { + RTCLogWarning(@"Attempting to deactivate without prior activation."); + } + [self notifyWillSetActive:active]; + BOOL success = YES; + BOOL isActive = self.isActive; + // Keep a local error so we can log it. + NSError *error = nil; + BOOL shouldSetActive = + (active && !isActive) || (!active && isActive && activationCount == 1); + // Attempt to activate if we're not active. + // Attempt to deactivate if we're active and it's the last unbalanced call. + if (shouldSetActive) { + AVAudioSession *session = self.session; + // AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation is used to ensure + // that other audio sessions that were interrupted by our session can return + // to their active state. It is recommended for VoIP apps to use this + // option. + AVAudioSessionSetActiveOptions options = + active ? 0 : AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation; + success = [session setActive:active + withOptions:options + error:&error]; + if (outError) { + *outError = error; + } + } + if (success) { + if (active) { + if (shouldSetActive) { + self.isActive = active; + if (self.isInterrupted) { + self.isInterrupted = NO; + [self notifyDidEndInterruptionWithShouldResumeSession:YES]; + } + } + [self incrementActivationCount]; + [self notifyDidSetActive:active]; + } + } else { + RTCLogError(@"Failed to setActive:%d. Error: %@", + active, error.localizedDescription); + [self notifyFailedToSetActive:active error:error]; + } + // Set isActive and decrement activation count on deactivation + // whether or not it succeeded. + if (!active) { + self.isActive = active; + [self notifyDidSetActive:active]; + [self decrementActivationCount]; + } + RTCLog(@"Number of current activations: %d", _activationCount.load()); + return success; +} + +- (BOOL)setCategory:(AVAudioSessionCategory)category + mode:(AVAudioSessionMode)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setCategory:category mode:mode options:options error:outError]; +} + +- (BOOL)setCategory:(AVAudioSessionCategory)category + withOptions:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setCategory:category withOptions:options error:outError]; +} + +- (BOOL)setMode:(AVAudioSessionMode)mode error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setMode:mode error:outError]; +} + +- (BOOL)setInputGain:(float)gain error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setInputGain:gain error:outError]; +} + +- (BOOL)setPreferredSampleRate:(double)sampleRate error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setPreferredSampleRate:sampleRate error:outError]; +} + +- (BOOL)setPreferredIOBufferDuration:(NSTimeInterval)duration + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setPreferredIOBufferDuration:duration error:outError]; +} + +- (BOOL)setPreferredInputNumberOfChannels:(NSInteger)count + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setPreferredInputNumberOfChannels:count error:outError]; +} +- (BOOL)setPreferredOutputNumberOfChannels:(NSInteger)count + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setPreferredOutputNumberOfChannels:count error:outError]; +} + +- (BOOL)overrideOutputAudioPort:(AVAudioSessionPortOverride)portOverride + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session overrideOutputAudioPort:portOverride error:outError]; +} + +- (BOOL)setPreferredInput:(AVAudioSessionPortDescription *)inPort + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setPreferredInput:inPort error:outError]; +} + +- (BOOL)setInputDataSource:(AVAudioSessionDataSourceDescription *)dataSource + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setInputDataSource:dataSource error:outError]; +} + +- (BOOL)setOutputDataSource:(AVAudioSessionDataSourceDescription *)dataSource + error:(NSError **)outError { + if (![self checkLock:outError]) { + return NO; + } + return [self.session setOutputDataSource:dataSource error:outError]; +} + +#pragma mark - Notifications + +- (void)handleInterruptionNotification:(NSNotification *)notification { + NSNumber* typeNumber = + notification.userInfo[AVAudioSessionInterruptionTypeKey]; + AVAudioSessionInterruptionType type = + (AVAudioSessionInterruptionType)typeNumber.unsignedIntegerValue; + switch (type) { + case AVAudioSessionInterruptionTypeBegan: + RTCLog(@"Audio session interruption began."); + self.isActive = NO; + self.isInterrupted = YES; + [self notifyDidBeginInterruption]; + break; + case AVAudioSessionInterruptionTypeEnded: { + RTCLog(@"Audio session interruption ended."); + self.isInterrupted = NO; + [self updateAudioSessionAfterEvent]; + NSNumber *optionsNumber = + notification.userInfo[AVAudioSessionInterruptionOptionKey]; + AVAudioSessionInterruptionOptions options = + optionsNumber.unsignedIntegerValue; + BOOL shouldResume = + options & AVAudioSessionInterruptionOptionShouldResume; + [self notifyDidEndInterruptionWithShouldResumeSession:shouldResume]; + break; + } + } +} + +- (void)handleRouteChangeNotification:(NSNotification *)notification { + // Get reason for current route change. + NSNumber* reasonNumber = + notification.userInfo[AVAudioSessionRouteChangeReasonKey]; + AVAudioSessionRouteChangeReason reason = + (AVAudioSessionRouteChangeReason)reasonNumber.unsignedIntegerValue; + RTCLog(@"Audio route changed:"); + switch (reason) { + case AVAudioSessionRouteChangeReasonUnknown: + RTCLog(@"Audio route changed: ReasonUnknown"); + break; + case AVAudioSessionRouteChangeReasonNewDeviceAvailable: + RTCLog(@"Audio route changed: NewDeviceAvailable"); + break; + case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: + RTCLog(@"Audio route changed: OldDeviceUnavailable"); + break; + case AVAudioSessionRouteChangeReasonCategoryChange: + RTCLog(@"Audio route changed: CategoryChange to :%@", + self.session.category); + break; + case AVAudioSessionRouteChangeReasonOverride: + RTCLog(@"Audio route changed: Override"); + break; + case AVAudioSessionRouteChangeReasonWakeFromSleep: + RTCLog(@"Audio route changed: WakeFromSleep"); + break; + case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory: + RTCLog(@"Audio route changed: NoSuitableRouteForCategory"); + break; + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: + RTCLog(@"Audio route changed: RouteConfigurationChange"); + break; + } + AVAudioSessionRouteDescription* previousRoute = + notification.userInfo[AVAudioSessionRouteChangePreviousRouteKey]; + // Log previous route configuration. + RTCLog(@"Previous route: %@\nCurrent route:%@", + previousRoute, self.session.currentRoute); + [self notifyDidChangeRouteWithReason:reason previousRoute:previousRoute]; +} + +- (void)handleMediaServicesWereLost:(NSNotification *)notification { + RTCLog(@"Media services were lost."); + [self updateAudioSessionAfterEvent]; + [self notifyMediaServicesWereLost]; +} + +- (void)handleMediaServicesWereReset:(NSNotification *)notification { + RTCLog(@"Media services were reset."); + [self updateAudioSessionAfterEvent]; + [self notifyMediaServicesWereReset]; +} + +- (void)handleSilenceSecondaryAudioHintNotification:(NSNotification *)notification { + // TODO(henrika): just adding logs here for now until we know if we are ever + // see this notification and might be affected by it or if further actions + // are required. + NSNumber *typeNumber = + notification.userInfo[AVAudioSessionSilenceSecondaryAudioHintTypeKey]; + AVAudioSessionSilenceSecondaryAudioHintType type = + (AVAudioSessionSilenceSecondaryAudioHintType)typeNumber.unsignedIntegerValue; + switch (type) { + case AVAudioSessionSilenceSecondaryAudioHintTypeBegin: + RTCLog(@"Another application's primary audio has started."); + break; + case AVAudioSessionSilenceSecondaryAudioHintTypeEnd: + RTCLog(@"Another application's primary audio has stopped."); + break; + } +} + +- (void)handleApplicationDidBecomeActive:(NSNotification *)notification { + BOOL isInterrupted = self.isInterrupted; + RTCLog(@"Application became active after an interruption. Treating as interruption " + "end. isInterrupted changed from %d to 0.", + isInterrupted); + if (isInterrupted) { + self.isInterrupted = NO; + [self updateAudioSessionAfterEvent]; + } + // Always treat application becoming active as an interruption end event. + [self notifyDidEndInterruptionWithShouldResumeSession:YES]; +} + +#pragma mark - Private + ++ (NSError *)lockError { + NSDictionary *userInfo = + @{NSLocalizedDescriptionKey : @"Must call lockForConfiguration before calling this method."}; + NSError *error = [[NSError alloc] initWithDomain:kRTCAudioSessionErrorDomain + code:kRTCAudioSessionErrorLockRequired + userInfo:userInfo]; + return error; +} + +- (std::vector<__weak id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)> >)delegates { + @synchronized(self) { + // Note: this returns a copy. + return _delegates; + } +} + +// TODO(tkchin): check for duplicates. +- (void)pushDelegate:(id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)>)delegate { + @synchronized(self) { + _delegates.insert(_delegates.begin(), delegate); + } +} + +- (void)removeZeroedDelegates { + @synchronized(self) { + _delegates.erase( + std::remove_if(_delegates.begin(), + _delegates.end(), + [](id delegate) -> bool { return delegate == nil; }), + _delegates.end()); + } +} + +- (int)activationCount { + return _activationCount.load(); +} + +- (int)incrementActivationCount { + RTCLog(@"Incrementing activation count."); + return _activationCount.fetch_add(1) + 1; +} + +- (NSInteger)decrementActivationCount { + RTCLog(@"Decrementing activation count."); + return _activationCount.fetch_sub(1) - 1; +} + +- (int)webRTCSessionCount { + return _webRTCSessionCount.load(); +} + +- (BOOL)canPlayOrRecord { + return !self.useManualAudio || self.isAudioEnabled; +} + +- (BOOL)isInterrupted { + @synchronized(self) { + return _isInterrupted; + } +} + +- (void)setIsInterrupted:(BOOL)isInterrupted { + @synchronized(self) { + if (_isInterrupted == isInterrupted) { + return; + } + _isInterrupted = isInterrupted; + } +} + +- (BOOL)checkLock:(NSError **)outError { + if (!mutex_locked) { + if (outError) { + *outError = [RTC_OBJC_TYPE(RTCAudioSession) lockError]; + } + return NO; + } + return YES; +} + +- (BOOL)beginWebRTCSession:(NSError **)outError { + if (outError) { + *outError = nil; + } + _webRTCSessionCount.fetch_add(1); + [self notifyDidStartPlayOrRecord]; + return YES; +} + +- (BOOL)endWebRTCSession:(NSError **)outError { + if (outError) { + *outError = nil; + } + _webRTCSessionCount.fetch_sub(1); + [self notifyDidStopPlayOrRecord]; + return YES; +} + +- (BOOL)configureWebRTCSession:(NSError **)outError { + if (outError) { + *outError = nil; + } + RTCLog(@"Configuring audio session for WebRTC."); + + // Configure the AVAudioSession and activate it. + // Provide an error even if there isn't one so we can log it. + NSError *error = nil; + RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *webRTCConfig = + [RTC_OBJC_TYPE(RTCAudioSessionConfiguration) webRTCConfiguration]; + if (![self setConfiguration:webRTCConfig active:YES error:&error]) { + RTCLogError(@"Failed to set WebRTC audio configuration: %@", + error.localizedDescription); + // Do not call setActive:NO if setActive:YES failed. + if (outError) { + *outError = error; + } + return NO; + } + + // Ensure that the device currently supports audio input. + // TODO(tkchin): Figure out if this is really necessary. + if (!self.inputAvailable) { + RTCLogError(@"No audio input path is available!"); + [self unconfigureWebRTCSession:nil]; + if (outError) { + *outError = [self configurationErrorWithDescription:@"No input path."]; + } + return NO; + } + + // It can happen (e.g. in combination with BT devices) that the attempt to set + // the preferred sample rate for WebRTC (48kHz) fails. If so, make a new + // configuration attempt using the sample rate that worked using the active + // audio session. A typical case is that only 8 or 16kHz can be set, e.g. in + // combination with BT headsets. Using this "trick" seems to avoid a state + // where Core Audio asks for a different number of audio frames than what the + // session's I/O buffer duration corresponds to. + // TODO(henrika): this fix resolves bugs.webrtc.org/6004 but it has only been + // tested on a limited set of iOS devices and BT devices. + double sessionSampleRate = self.sampleRate; + double preferredSampleRate = webRTCConfig.sampleRate; + if (sessionSampleRate != preferredSampleRate) { + RTCLogWarning( + @"Current sample rate (%.2f) is not the preferred rate (%.2f)", + sessionSampleRate, preferredSampleRate); + if (![self setPreferredSampleRate:sessionSampleRate + error:&error]) { + RTCLogError(@"Failed to set preferred sample rate: %@", + error.localizedDescription); + if (outError) { + *outError = error; + } + } + } + + return YES; +} + +- (BOOL)unconfigureWebRTCSession:(NSError **)outError { + if (outError) { + *outError = nil; + } + RTCLog(@"Unconfiguring audio session for WebRTC."); + [self setActive:NO error:outError]; + + return YES; +} + +- (NSError *)configurationErrorWithDescription:(NSString *)description { + NSDictionary* userInfo = @{ + NSLocalizedDescriptionKey: description, + }; + return [[NSError alloc] initWithDomain:kRTCAudioSessionErrorDomain + code:kRTCAudioSessionErrorConfiguration + userInfo:userInfo]; +} + +- (void)updateAudioSessionAfterEvent { + BOOL shouldActivate = self.activationCount > 0; + AVAudioSessionSetActiveOptions options = shouldActivate ? + 0 : AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation; + NSError *error = nil; + if ([self.session setActive:shouldActivate + withOptions:options + error:&error]) { + self.isActive = shouldActivate; + } else { + RTCLogError(@"Failed to set session active to %d. Error:%@", + shouldActivate, error.localizedDescription); + } +} + +- (void)updateCanPlayOrRecord { + BOOL canPlayOrRecord = NO; + BOOL shouldNotify = NO; + @synchronized(self) { + canPlayOrRecord = !self.useManualAudio || self.isAudioEnabled; + if (_canPlayOrRecord == canPlayOrRecord) { + return; + } + _canPlayOrRecord = canPlayOrRecord; + shouldNotify = YES; + } + if (shouldNotify) { + [self notifyDidChangeCanPlayOrRecord:canPlayOrRecord]; + } +} + +- (void)audioSessionDidActivate:(AVAudioSession *)session { + if (_session != session) { + RTCLogError(@"audioSessionDidActivate called on different AVAudioSession"); + } + RTCLog(@"Audio session was externally activated."); + [self incrementActivationCount]; + self.isActive = YES; + // When a CallKit call begins, it's possible that we receive an interruption + // begin without a corresponding end. Since we know that we have an activated + // audio session at this point, just clear any saved interruption flag since + // the app may never be foregrounded during the duration of the call. + if (self.isInterrupted) { + RTCLog(@"Clearing interrupted state due to external activation."); + self.isInterrupted = NO; + } + // Treat external audio session activation as an end interruption event. + [self notifyDidEndInterruptionWithShouldResumeSession:YES]; +} + +- (void)audioSessionDidDeactivate:(AVAudioSession *)session { + if (_session != session) { + RTCLogError(@"audioSessionDidDeactivate called on different AVAudioSession"); + } + RTCLog(@"Audio session was externally deactivated."); + self.isActive = NO; + [self decrementActivationCount]; +} + +- (void)observeValueForKeyPath:(NSString *)keyPath + ofObject:(id)object + change:(NSDictionary *)change + context:(void *)context { + if (context == (__bridge void *)RTC_OBJC_TYPE(RTCAudioSession).class) { + if (object == _session) { + NSNumber *newVolume = change[NSKeyValueChangeNewKey]; + RTCLog(@"OutputVolumeDidChange to %f", newVolume.floatValue); + [self notifyDidChangeOutputVolume:newVolume.floatValue]; + } + } else { + [super observeValueForKeyPath:keyPath + ofObject:object + change:change + context:context]; + } +} + +- (void)notifyAudioUnitStartFailedWithError:(OSStatus)error { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSession:audioUnitStartFailedWithError:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self + audioUnitStartFailedWithError:[NSError errorWithDomain:kRTCAudioSessionErrorDomain + code:error + userInfo:nil]]; + } + } +} + +- (void)notifyDidBeginInterruption { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionDidBeginInterruption:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionDidBeginInterruption:self]; + } + } +} + +- (void)notifyDidEndInterruptionWithShouldResumeSession: + (BOOL)shouldResumeSession { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionDidEndInterruption:shouldResumeSession:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionDidEndInterruption:self + shouldResumeSession:shouldResumeSession]; + } + } +} + +- (void)notifyDidChangeRouteWithReason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionDidChangeRoute:reason:previousRoute:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionDidChangeRoute:self + reason:reason + previousRoute:previousRoute]; + } + } +} + +- (void)notifyMediaServicesWereLost { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionMediaServerTerminated:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionMediaServerTerminated:self]; + } + } +} + +- (void)notifyMediaServicesWereReset { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionMediaServerReset:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionMediaServerReset:self]; + } + } +} + +- (void)notifyDidChangeCanPlayOrRecord:(BOOL)canPlayOrRecord { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSession:didChangeCanPlayOrRecord:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self didChangeCanPlayOrRecord:canPlayOrRecord]; + } + } +} + +- (void)notifyDidStartPlayOrRecord { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionDidStartPlayOrRecord:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionDidStartPlayOrRecord:self]; + } + } +} + +- (void)notifyDidStopPlayOrRecord { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSessionDidStopPlayOrRecord:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSessionDidStopPlayOrRecord:self]; + } + } +} + +- (void)notifyDidChangeOutputVolume:(float)volume { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSession:didChangeOutputVolume:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self didChangeOutputVolume:volume]; + } + } +} + +- (void)notifyDidDetectPlayoutGlitch:(int64_t)totalNumberOfGlitches { + for (auto delegate : self.delegates) { + SEL sel = @selector(audioSession:didDetectPlayoutGlitch:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self didDetectPlayoutGlitch:totalNumberOfGlitches]; + } + } +} + +- (void)notifyWillSetActive:(BOOL)active { + for (id delegate : self.delegates) { + SEL sel = @selector(audioSession:willSetActive:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self willSetActive:active]; + } + } +} + +- (void)notifyDidSetActive:(BOOL)active { + for (id delegate : self.delegates) { + SEL sel = @selector(audioSession:didSetActive:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self didSetActive:active]; + } + } +} + +- (void)notifyFailedToSetActive:(BOOL)active error:(NSError *)error { + for (id delegate : self.delegates) { + SEL sel = @selector(audioSession:failedToSetActive:error:); + if ([delegate respondsToSelector:sel]) { + [delegate audioSession:self failedToSetActive:active error:error]; + } + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.h b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.h new file mode 100644 index 0000000000..6c6f808a32 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.h @@ -0,0 +1,46 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_EXTERN const int kRTCAudioSessionPreferredNumberOfChannels; +RTC_EXTERN const double kRTCAudioSessionHighPerformanceSampleRate; +RTC_EXTERN const double kRTCAudioSessionHighPerformanceIOBufferDuration; + +// Struct to hold configuration values. +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioSessionConfiguration) : NSObject + +@property(nonatomic, strong) NSString *category; +@property(nonatomic, assign) AVAudioSessionCategoryOptions categoryOptions; +@property(nonatomic, strong) NSString *mode; +@property(nonatomic, assign) double sampleRate; +@property(nonatomic, assign) NSTimeInterval ioBufferDuration; +@property(nonatomic, assign) NSInteger inputNumberOfChannels; +@property(nonatomic, assign) NSInteger outputNumberOfChannels; + +/** Initializes configuration to defaults. */ +- (instancetype)init NS_DESIGNATED_INITIALIZER; + +/** Returns the current configuration of the audio session. */ ++ (instancetype)currentConfiguration; +/** Returns the configuration that WebRTC needs. */ ++ (instancetype)webRTCConfiguration; +/** Provide a way to override the default configuration. */ ++ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.m b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.m new file mode 100644 index 0000000000..71b0c0cb3a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCAudioSessionConfiguration.m @@ -0,0 +1,112 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSessionConfiguration.h" +#import "RTCAudioSession.h" + +#import "helpers/RTCDispatcher.h" +#import "helpers/UIDevice+RTCDevice.h" + +// Try to use mono to save resources. Also avoids channel format conversion +// in the I/O audio unit. Initial tests have shown that it is possible to use +// mono natively for built-in microphones and for BT headsets but not for +// wired headsets. Wired headsets only support stereo as native channel format +// but it is a low cost operation to do a format conversion to mono in the +// audio unit. Hence, we will not hit a RTC_CHECK in +// VerifyAudioParametersForActiveAudioSession() for a mismatch between the +// preferred number of channels and the actual number of channels. +const int kRTCAudioSessionPreferredNumberOfChannels = 1; + +// Preferred hardware sample rate (unit is in Hertz). The client sample rate +// will be set to this value as well to avoid resampling the the audio unit's +// format converter. Note that, some devices, e.g. BT headsets, only supports +// 8000Hz as native sample rate. +const double kRTCAudioSessionHighPerformanceSampleRate = 48000.0; + +// Use a hardware I/O buffer size (unit is in seconds) that matches the 10ms +// size used by WebRTC. The exact actual size will differ between devices. +// Example: using 48kHz on iPhone 6 results in a native buffer size of +// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will +// take care of any buffering required to convert between native buffers and +// buffers used by WebRTC. It is beneficial for the performance if the native +// size is as an even multiple of 10ms as possible since it results in "clean" +// callback sequence without bursts of callbacks back to back. +const double kRTCAudioSessionHighPerformanceIOBufferDuration = 0.02; + +static RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *gWebRTCConfiguration = nil; + +@implementation RTC_OBJC_TYPE (RTCAudioSessionConfiguration) + +@synthesize category = _category; +@synthesize categoryOptions = _categoryOptions; +@synthesize mode = _mode; +@synthesize sampleRate = _sampleRate; +@synthesize ioBufferDuration = _ioBufferDuration; +@synthesize inputNumberOfChannels = _inputNumberOfChannels; +@synthesize outputNumberOfChannels = _outputNumberOfChannels; + +- (instancetype)init { + if (self = [super init]) { + // Use a category which supports simultaneous recording and playback. + // By default, using this category implies that our app’s audio is + // nonmixable, hence activating the session will interrupt any other + // audio sessions which are also nonmixable. + _category = AVAudioSessionCategoryPlayAndRecord; + _categoryOptions = AVAudioSessionCategoryOptionAllowBluetooth; + + // Specify mode for two-way voice communication (e.g. VoIP). + _mode = AVAudioSessionModeVoiceChat; + + // Use best sample rate and buffer duration if the CPU has more than one + // core. + _sampleRate = kRTCAudioSessionHighPerformanceSampleRate; + _ioBufferDuration = kRTCAudioSessionHighPerformanceIOBufferDuration; + + // We try to use mono in both directions to save resources and format + // conversions in the audio unit. Some devices does only support stereo; + // e.g. wired headset on iPhone 6. + // TODO(henrika): add support for stereo if needed. + _inputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels; + _outputNumberOfChannels = kRTCAudioSessionPreferredNumberOfChannels; + } + return self; +} + ++ (void)initialize { + gWebRTCConfiguration = [[self alloc] init]; +} + ++ (instancetype)currentConfiguration { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *config = + [[RTC_OBJC_TYPE(RTCAudioSessionConfiguration) alloc] init]; + config.category = session.category; + config.categoryOptions = session.categoryOptions; + config.mode = session.mode; + config.sampleRate = session.sampleRate; + config.ioBufferDuration = session.IOBufferDuration; + config.inputNumberOfChannels = session.inputNumberOfChannels; + config.outputNumberOfChannels = session.outputNumberOfChannels; + return config; +} + ++ (instancetype)webRTCConfiguration { + @synchronized(self) { + return (RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)gWebRTCConfiguration; + } +} + ++ (void)setWebRTCConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configuration { + @synchronized(self) { + gWebRTCConfiguration = configuration; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h b/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h new file mode 100644 index 0000000000..6a75f01479 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.h @@ -0,0 +1,33 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioSession.h" + +NS_ASSUME_NONNULL_BEGIN + +namespace webrtc { +class AudioSessionObserver; +} + +/** Adapter that forwards RTCAudioSessionDelegate calls to the appropriate + * methods on the AudioSessionObserver. + */ +@interface RTCNativeAudioSessionDelegateAdapter : NSObject <RTC_OBJC_TYPE (RTCAudioSessionDelegate)> + +- (instancetype)init NS_UNAVAILABLE; + +/** `observer` is a raw pointer and should be kept alive + * for this object's lifetime. + */ +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm b/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm new file mode 100644 index 0000000000..daddf314a4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/audio/RTCNativeAudioSessionDelegateAdapter.mm @@ -0,0 +1,89 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNativeAudioSessionDelegateAdapter.h" + +#include "sdk/objc/native/src/audio/audio_session_observer.h" + +#import "base/RTCLogging.h" + +@implementation RTCNativeAudioSessionDelegateAdapter { + webrtc::AudioSessionObserver *_observer; +} + +- (instancetype)initWithObserver:(webrtc::AudioSessionObserver *)observer { + RTC_DCHECK(observer); + if (self = [super init]) { + _observer = observer; + } + return self; +} + +#pragma mark - RTC_OBJC_TYPE(RTCAudioSessionDelegate) + +- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session { + _observer->OnInterruptionBegin(); +} + +- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session + shouldResumeSession:(BOOL)shouldResumeSession { + _observer->OnInterruptionEnd(); +} + +- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session + reason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute { + switch (reason) { + case AVAudioSessionRouteChangeReasonUnknown: + case AVAudioSessionRouteChangeReasonNewDeviceAvailable: + case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: + case AVAudioSessionRouteChangeReasonCategoryChange: + // It turns out that we see a category change (at least in iOS 9.2) + // when making a switch from a BT device to e.g. Speaker using the + // iOS Control Center and that we therefore must check if the sample + // rate has changed. And if so is the case, restart the audio unit. + case AVAudioSessionRouteChangeReasonOverride: + case AVAudioSessionRouteChangeReasonWakeFromSleep: + case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory: + _observer->OnValidRouteChange(); + break; + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: + // The set of input and output ports has not changed, but their + // configuration has, e.g., a port’s selected data source has + // changed. Ignore this type of route change since we are focusing + // on detecting headset changes. + RTCLog(@"Ignoring RouteConfigurationChange"); + break; + } +} + +- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)session + didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord { + _observer->OnCanPlayOrRecordChange(canPlayOrRecord); +} + +- (void)audioSessionDidStartPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionDidStopPlayOrRecord:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + didChangeOutputVolume:(float)outputVolume { + _observer->OnChangedOutputVolume(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.h b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.h new file mode 100644 index 0000000000..b1f3f64f74 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.h @@ -0,0 +1,59 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoCapturer.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +// Camera capture that implements RTCVideoCapturer. Delivers frames to a +// RTCVideoCapturerDelegate (usually RTCVideoSource). +NS_EXTENSION_UNAVAILABLE_IOS("Camera not available in app extensions.") +@interface RTC_OBJC_TYPE (RTCCameraVideoCapturer) : RTC_OBJC_TYPE(RTCVideoCapturer) + +// Capture session that is used for capturing. Valid from initialization to dealloc. +@property(readonly, nonatomic) AVCaptureSession *captureSession; + +// Returns list of available capture devices that support video capture. ++ (NSArray<AVCaptureDevice *> *)captureDevicesWithDeviceTypes: + (NSArray<AVCaptureDeviceType> *)deviceTypes; +// Returns list of default capture devices types ++ (NSArray<AVCaptureDeviceType> *)defaultCaptureDeviceTypes; +// Returns list of formats that are supported by this class for this device. ++ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device; + +// Returns the most efficient supported output pixel format for this capturer. +- (FourCharCode)preferredOutputPixelFormat; + +// Starts the capture session asynchronously and notifies callback on completion. +// The device will capture video in the format given in the `format` parameter. If the pixel format +// in `format` is supported by the WebRTC pipeline, the same pixel format will be used for the +// output. Otherwise, the format returned by `preferredOutputPixelFormat` will be used. +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps + completionHandler:(nullable void (^)(NSError *_Nullable))completionHandler; +// Stops the capture session asynchronously and notifies callback on completion. +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler; + +// Starts the capture session asynchronously. +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps; +// Stops the capture session asynchronously. +- (void)stopCapture; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m new file mode 100644 index 0000000000..1361207faf --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m @@ -0,0 +1,543 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCameraVideoCapturer.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#if TARGET_OS_IPHONE +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#import "helpers/AVCaptureSession+DevicePosition.h" +#import "helpers/RTCDispatcher+Private.h" +#include "rtc_base/system/gcd_helpers.h" + +const int64_t kNanosecondsPerSecond = 1000000000; + +@interface RTC_OBJC_TYPE (RTCCameraVideoCapturer) +()<AVCaptureVideoDataOutputSampleBufferDelegate> @property(nonatomic, + readonly) dispatch_queue_t frameQueue; +@property(nonatomic, strong) AVCaptureDevice *currentDevice; +@property(nonatomic, assign) BOOL hasRetriedOnFatalError; +@property(nonatomic, assign) BOOL isRunning; +// Will the session be running once all asynchronous operations have been completed? +@property(nonatomic, assign) BOOL willBeRunning; +@end + +@implementation RTC_OBJC_TYPE (RTCCameraVideoCapturer) { + AVCaptureVideoDataOutput *_videoDataOutput; + AVCaptureSession *_captureSession; + FourCharCode _preferredOutputPixelFormat; + FourCharCode _outputPixelFormat; + RTCVideoRotation _rotation; +#if TARGET_OS_IPHONE + UIDeviceOrientation _orientation; + BOOL _generatingOrientationNotifications; +#endif +} + +@synthesize frameQueue = _frameQueue; +@synthesize captureSession = _captureSession; +@synthesize currentDevice = _currentDevice; +@synthesize hasRetriedOnFatalError = _hasRetriedOnFatalError; +@synthesize isRunning = _isRunning; +@synthesize willBeRunning = _willBeRunning; + +- (instancetype)init { + return [self initWithDelegate:nil captureSession:[[AVCaptureSession alloc] init]]; +} + +- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate { + return [self initWithDelegate:delegate captureSession:[[AVCaptureSession alloc] init]]; +} + +// This initializer is used for testing. +- (instancetype)initWithDelegate:(__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate + captureSession:(AVCaptureSession *)captureSession { + if (self = [super initWithDelegate:delegate]) { + // Create the capture session and all relevant inputs and outputs. We need + // to do this in init because the application may want the capture session + // before we start the capturer for e.g. AVCapturePreviewLayer. All objects + // created here are retained until dealloc and never recreated. + if (![self setupCaptureSession:captureSession]) { + return nil; + } + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; +#if TARGET_OS_IPHONE + _orientation = UIDeviceOrientationPortrait; + _rotation = RTCVideoRotation_90; + [center addObserver:self + selector:@selector(deviceOrientationDidChange:) + name:UIDeviceOrientationDidChangeNotification + object:nil]; + [center addObserver:self + selector:@selector(handleCaptureSessionInterruption:) + name:AVCaptureSessionWasInterruptedNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionInterruptionEnded:) + name:AVCaptureSessionInterruptionEndedNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleApplicationDidBecomeActive:) + name:UIApplicationDidBecomeActiveNotification + object:[UIApplication sharedApplication]]; +#endif + [center addObserver:self + selector:@selector(handleCaptureSessionRuntimeError:) + name:AVCaptureSessionRuntimeErrorNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionDidStartRunning:) + name:AVCaptureSessionDidStartRunningNotification + object:_captureSession]; + [center addObserver:self + selector:@selector(handleCaptureSessionDidStopRunning:) + name:AVCaptureSessionDidStopRunningNotification + object:_captureSession]; + } + return self; +} + +- (void)dealloc { + NSAssert(!_willBeRunning, + @"Session was still running in RTC_OBJC_TYPE(RTCCameraVideoCapturer) dealloc. Forgot to " + @"call stopCapture?"); + [[NSNotificationCenter defaultCenter] removeObserver:self]; +} + ++ (NSArray<AVCaptureDevice *> *)captureDevicesWithDeviceTypes: + (NSArray<AVCaptureDeviceType> *)deviceTypes { + AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:deviceTypes + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + return session.devices; +} + ++ (NSArray<AVCaptureDeviceType> *)defaultCaptureDeviceTypes { + NSArray *types = @[ AVCaptureDeviceTypeBuiltInWideAngleCamera ]; +#if !defined(WEBRTC_IOS) + if (@available(macOS 14.0, *)) { + types = [types arrayByAddingObject:AVCaptureDeviceTypeExternal]; + } else { + types = [types arrayByAddingObject:AVCaptureDeviceTypeExternalUnknown]; + } +#endif + return types; +} + ++ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device { + // Support opening the device in any format. We make sure it's converted to a format we + // can handle, if needed, in the method `-setupVideoDataOutput`. + return device.formats; +} + +- (FourCharCode)preferredOutputPixelFormat { + return _preferredOutputPixelFormat; +} + +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps { + [self startCaptureWithDevice:device format:format fps:fps completionHandler:nil]; +} + +- (void)stopCapture { + [self stopCaptureWithCompletionHandler:nil]; +} + +- (void)startCaptureWithDevice:(AVCaptureDevice *)device + format:(AVCaptureDeviceFormat *)format + fps:(NSInteger)fps + completionHandler:(nullable void (^)(NSError *_Nullable error))completionHandler { + _willBeRunning = YES; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLogInfo("startCaptureWithDevice %@ @ %ld fps", format, (long)fps); + +#if TARGET_OS_IPHONE + dispatch_async(dispatch_get_main_queue(), ^{ + if (!self->_generatingOrientationNotifications) { + [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications]; + self->_generatingOrientationNotifications = YES; + } + }); +#endif + + self.currentDevice = device; + + NSError *error = nil; + if (![self.currentDevice lockForConfiguration:&error]) { + RTCLogError(@"Failed to lock device %@. Error: %@", + self.currentDevice, + error.userInfo); + if (completionHandler) { + completionHandler(error); + } + self.willBeRunning = NO; + return; + } + [self reconfigureCaptureSessionInput]; + [self updateOrientation]; + [self updateDeviceCaptureFormat:format fps:fps]; + [self updateVideoDataOutputPixelFormat:format]; + [self.captureSession startRunning]; + [self.currentDevice unlockForConfiguration]; + self.isRunning = YES; + if (completionHandler) { + completionHandler(nil); + } + }]; +} + +- (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHandler { + _willBeRunning = NO; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLogInfo("Stop"); + self.currentDevice = nil; + for (AVCaptureDeviceInput *oldInput in [self.captureSession.inputs copy]) { + [self.captureSession removeInput:oldInput]; + } + [self.captureSession stopRunning]; + +#if TARGET_OS_IPHONE + dispatch_async(dispatch_get_main_queue(), ^{ + if (self->_generatingOrientationNotifications) { + [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications]; + self->_generatingOrientationNotifications = NO; + } + }); +#endif + self.isRunning = NO; + if (completionHandler) { + completionHandler(); + } + }]; +} + +#pragma mark iOS notifications + +#if TARGET_OS_IPHONE +- (void)deviceOrientationDidChange:(NSNotification *)notification { + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + [self updateOrientation]; + }]; +} +#endif + +#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { + NSParameterAssert(captureOutput == _videoDataOutput); + + if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) || + !CMSampleBufferDataIsReady(sampleBuffer)) { + return; + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (pixelBuffer == nil) { + return; + } + +#if TARGET_OS_IPHONE + // Default to portrait orientation on iPhone. + BOOL usingFrontCamera = NO; + // Check the image's EXIF for the camera the image came from as the image could have been + // delayed as we set alwaysDiscardsLateVideoFrames to NO. + AVCaptureDevicePosition cameraPosition = + [AVCaptureSession devicePositionForSampleBuffer:sampleBuffer]; + if (cameraPosition != AVCaptureDevicePositionUnspecified) { + usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition; + } else { + AVCaptureDeviceInput *deviceInput = + (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input; + usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position; + } + switch (_orientation) { + case UIDeviceOrientationPortrait: + _rotation = RTCVideoRotation_90; + break; + case UIDeviceOrientationPortraitUpsideDown: + _rotation = RTCVideoRotation_270; + break; + case UIDeviceOrientationLandscapeLeft: + _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; + break; + case UIDeviceOrientationLandscapeRight: + _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180; + break; + case UIDeviceOrientationFaceUp: + case UIDeviceOrientationFaceDown: + case UIDeviceOrientationUnknown: + // Ignore. + break; + } +#else + // No rotation on Mac. + _rotation = RTCVideoRotation_0; +#endif + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; + int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * + kNanosecondsPerSecond; + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:_rotation + timeStampNs:timeStampNs]; + [self.delegate capturer:self didCaptureVideoFrame:videoFrame]; +} + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { +#if TARGET_OS_IPHONE + CFStringRef droppedReason = + CMGetAttachment(sampleBuffer, kCMSampleBufferAttachmentKey_DroppedFrameReason, nil); +#else + // DroppedFrameReason unavailable on macOS. + CFStringRef droppedReason = nil; +#endif + RTCLogError(@"Dropped sample buffer. Reason: %@", (__bridge NSString *)droppedReason); +} + +#pragma mark - AVCaptureSession notifications + +- (void)handleCaptureSessionInterruption:(NSNotification *)notification { + NSString *reasonString = nil; +#if TARGET_OS_IPHONE + NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey]; + if (reason) { + switch (reason.intValue) { + case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground: + reasonString = @"VideoDeviceNotAvailableInBackground"; + break; + case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient: + reasonString = @"AudioDeviceInUseByAnotherClient"; + break; + case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient: + reasonString = @"VideoDeviceInUseByAnotherClient"; + break; + case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps: + reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps"; + break; + } + } +#endif + RTCLog(@"Capture session interrupted: %@", reasonString); +} + +- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification { + RTCLog(@"Capture session interruption ended."); +} + +- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification { + NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey]; + RTCLogError(@"Capture session runtime error: %@", error); + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ +#if TARGET_OS_IPHONE + if (error.code == AVErrorMediaServicesWereReset) { + [self handleNonFatalError]; + } else { + [self handleFatalError]; + } +#else + [self handleFatalError]; +#endif + }]; +} + +- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification { + RTCLog(@"Capture session started."); + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + // If we successfully restarted after an unknown + // error, allow future retries on fatal errors. + self.hasRetriedOnFatalError = NO; + }]; +} + +- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification { + RTCLog(@"Capture session stopped."); +} + +- (void)handleFatalError { + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + if (!self.hasRetriedOnFatalError) { + RTCLogWarning(@"Attempting to recover from fatal capture error."); + [self handleNonFatalError]; + self.hasRetriedOnFatalError = YES; + } else { + RTCLogError(@"Previous fatal error recovery failed."); + } + }]; +} + +- (void)handleNonFatalError { + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + RTCLog(@"Restarting capture session after error."); + if (self.isRunning) { + [self.captureSession startRunning]; + } + }]; +} + +#if TARGET_OS_IPHONE + +#pragma mark - UIApplication notifications + +- (void)handleApplicationDidBecomeActive:(NSNotification *)notification { + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + if (self.isRunning && !self.captureSession.isRunning) { + RTCLog(@"Restarting capture session on active."); + [self.captureSession startRunning]; + } + }]; +} + +#endif // TARGET_OS_IPHONE + +#pragma mark - Private + +- (dispatch_queue_t)frameQueue { + if (!_frameQueue) { + _frameQueue = RTCDispatchQueueCreateWithTarget( + "org.webrtc.cameravideocapturer.video", + DISPATCH_QUEUE_SERIAL, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0)); + } + return _frameQueue; +} + +- (BOOL)setupCaptureSession:(AVCaptureSession *)captureSession { + NSAssert(_captureSession == nil, @"Setup capture session called twice."); + _captureSession = captureSession; +#if defined(WEBRTC_IOS) + _captureSession.sessionPreset = AVCaptureSessionPresetInputPriority; + _captureSession.usesApplicationAudioSession = NO; +#endif + [self setupVideoDataOutput]; + // Add the output. + if (![_captureSession canAddOutput:_videoDataOutput]) { + RTCLogError(@"Video data output unsupported."); + return NO; + } + [_captureSession addOutput:_videoDataOutput]; + + return YES; +} + +- (void)setupVideoDataOutput { + NSAssert(_videoDataOutput == nil, @"Setup video data output called twice."); + AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; + + // `videoDataOutput.availableVideoCVPixelFormatTypes` returns the pixel formats supported by the + // device with the most efficient output format first. Find the first format that we support. + NSSet<NSNumber *> *supportedPixelFormats = + [RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats]; + NSMutableOrderedSet *availablePixelFormats = + [NSMutableOrderedSet orderedSetWithArray:videoDataOutput.availableVideoCVPixelFormatTypes]; + [availablePixelFormats intersectSet:supportedPixelFormats]; + NSNumber *pixelFormat = availablePixelFormats.firstObject; + NSAssert(pixelFormat, @"Output device has no supported formats."); + + _preferredOutputPixelFormat = [pixelFormat unsignedIntValue]; + _outputPixelFormat = _preferredOutputPixelFormat; + videoDataOutput.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey : pixelFormat}; + videoDataOutput.alwaysDiscardsLateVideoFrames = NO; + [videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue]; + _videoDataOutput = videoDataOutput; +} + +- (void)updateVideoDataOutputPixelFormat:(AVCaptureDeviceFormat *)format { + FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription); + if (![[RTC_OBJC_TYPE(RTCCVPixelBuffer) supportedPixelFormats] containsObject:@(mediaSubType)]) { + mediaSubType = _preferredOutputPixelFormat; + } + + if (mediaSubType != _outputPixelFormat) { + _outputPixelFormat = mediaSubType; + } + + // Update videoSettings with dimensions, as some virtual cameras, e.g. Snap Camera, may not work + // otherwise. + CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription); + _videoDataOutput.videoSettings = @{ + (id)kCVPixelBufferWidthKey : @(dimensions.width), + (id)kCVPixelBufferHeightKey : @(dimensions.height), + (id)kCVPixelBufferPixelFormatTypeKey : @(_outputPixelFormat), + }; +} + +#pragma mark - Private, called inside capture queue + +- (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger)fps { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"updateDeviceCaptureFormat must be called on the capture queue."); + @try { + _currentDevice.activeFormat = format; + _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + } @catch (NSException *exception) { + RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo); + return; + } +} + +- (void)reconfigureCaptureSessionInput { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"reconfigureCaptureSessionInput must be called on the capture queue."); + NSError *error = nil; + AVCaptureDeviceInput *input = + [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error]; + if (!input) { + RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); + return; + } + [_captureSession beginConfiguration]; + for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) { + [_captureSession removeInput:oldInput]; + } + if ([_captureSession canAddInput:input]) { + [_captureSession addInput:input]; + } else { + RTCLogError(@"Cannot add camera as an input to the session."); + } + [_captureSession commitConfiguration]; +} + +- (void)updateOrientation { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], + @"updateOrientation must be called on the capture queue."); +#if TARGET_OS_IPHONE + _orientation = [UIDevice currentDevice].orientation; +#endif +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.h b/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.h new file mode 100644 index 0000000000..19262c64cf --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.h @@ -0,0 +1,51 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCVideoCapturer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Error passing block. + */ +typedef void (^RTCFileVideoCapturerErrorBlock)(NSError *error); + +/** + * Captures buffers from bundled video file. + * + * See @c RTCVideoCapturer for more info on capturers. + */ +RTC_OBJC_EXPORT + +NS_CLASS_AVAILABLE_IOS(10) +@interface RTC_OBJC_TYPE (RTCFileVideoCapturer) : RTC_OBJC_TYPE(RTCVideoCapturer) + +/** + * Starts asynchronous capture of frames from video file. + * + * Capturing is not started if error occurs. Underlying error will be + * relayed in the errorBlock if one is provided. + * Successfully captured video frames will be passed to the delegate. + * + * @param nameOfFile The name of the bundled video file to be read. + * @errorBlock block to be executed upon error. + */ +- (void)startCapturingFromFileNamed:(NSString *)nameOfFile + onError:(__nullable RTCFileVideoCapturerErrorBlock)errorBlock; + +/** + * Immediately stops capture. + */ +- (void)stopCapture; +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.m b/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.m new file mode 100644 index 0000000000..bcf1506259 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.m @@ -0,0 +1,215 @@ +/** + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCFileVideoCapturer.h" + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#include "rtc_base/system/gcd_helpers.h" + +NSString *const kRTCFileVideoCapturerErrorDomain = + @"org.webrtc.RTC_OBJC_TYPE(RTCFileVideoCapturer)"; + +typedef NS_ENUM(NSInteger, RTCFileVideoCapturerErrorCode) { + RTCFileVideoCapturerErrorCode_CapturerRunning = 2000, + RTCFileVideoCapturerErrorCode_FileNotFound +}; + +typedef NS_ENUM(NSInteger, RTCFileVideoCapturerStatus) { + RTCFileVideoCapturerStatusNotInitialized, + RTCFileVideoCapturerStatusStarted, + RTCFileVideoCapturerStatusStopped +}; + +@interface RTC_OBJC_TYPE (RTCFileVideoCapturer) +() @property(nonatomic, assign) CMTime lastPresentationTime; +@property(nonatomic, strong) NSURL *fileURL; +@end + +@implementation RTC_OBJC_TYPE (RTCFileVideoCapturer) { + AVAssetReader *_reader; + AVAssetReaderTrackOutput *_outTrack; + RTCFileVideoCapturerStatus _status; + dispatch_queue_t _frameQueue; +} + +@synthesize lastPresentationTime = _lastPresentationTime; +@synthesize fileURL = _fileURL; + +- (void)startCapturingFromFileNamed:(NSString *)nameOfFile + onError:(RTCFileVideoCapturerErrorBlock)errorBlock { + if (_status == RTCFileVideoCapturerStatusStarted) { + NSError *error = + [NSError errorWithDomain:kRTCFileVideoCapturerErrorDomain + code:RTCFileVideoCapturerErrorCode_CapturerRunning + userInfo:@{NSUnderlyingErrorKey : @"Capturer has been started."}]; + + errorBlock(error); + return; + } else { + _status = RTCFileVideoCapturerStatusStarted; + } + + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + NSString *pathForFile = [self pathForFileName:nameOfFile]; + if (!pathForFile) { + NSString *errorString = + [NSString stringWithFormat:@"File %@ not found in bundle", nameOfFile]; + NSError *error = [NSError errorWithDomain:kRTCFileVideoCapturerErrorDomain + code:RTCFileVideoCapturerErrorCode_FileNotFound + userInfo:@{NSUnderlyingErrorKey : errorString}]; + errorBlock(error); + return; + } + + self.lastPresentationTime = CMTimeMake(0, 0); + + self.fileURL = [NSURL fileURLWithPath:pathForFile]; + [self setupReaderOnError:errorBlock]; + }); +} + +- (void)setupReaderOnError:(RTCFileVideoCapturerErrorBlock)errorBlock { + AVURLAsset *asset = [AVURLAsset URLAssetWithURL:_fileURL options:nil]; + + NSArray *allTracks = [asset tracksWithMediaType:AVMediaTypeVideo]; + NSError *error = nil; + + _reader = [[AVAssetReader alloc] initWithAsset:asset error:&error]; + if (error) { + errorBlock(error); + return; + } + + NSDictionary *options = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) + }; + _outTrack = + [[AVAssetReaderTrackOutput alloc] initWithTrack:allTracks.firstObject outputSettings:options]; + [_reader addOutput:_outTrack]; + + [_reader startReading]; + RTCLog(@"File capturer started reading"); + [self readNextBuffer]; +} +- (void)stopCapture { + _status = RTCFileVideoCapturerStatusStopped; + RTCLog(@"File capturer stopped."); +} + +#pragma mark - Private + +- (nullable NSString *)pathForFileName:(NSString *)fileName { + NSArray *nameComponents = [fileName componentsSeparatedByString:@"."]; + if (nameComponents.count != 2) { + return nil; + } + + NSString *path = + [[NSBundle mainBundle] pathForResource:nameComponents[0] ofType:nameComponents[1]]; + return path; +} + +- (dispatch_queue_t)frameQueue { + if (!_frameQueue) { + _frameQueue = RTCDispatchQueueCreateWithTarget( + "org.webrtc.filecapturer.video", + DISPATCH_QUEUE_SERIAL, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0)); + } + return _frameQueue; +} + +- (void)readNextBuffer { + if (_status == RTCFileVideoCapturerStatusStopped) { + [_reader cancelReading]; + _reader = nil; + return; + } + + if (_reader.status == AVAssetReaderStatusCompleted) { + [_reader cancelReading]; + _reader = nil; + [self setupReaderOnError:nil]; + return; + } + + CMSampleBufferRef sampleBuffer = [_outTrack copyNextSampleBuffer]; + if (!sampleBuffer) { + [self readNextBuffer]; + return; + } + if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) || + !CMSampleBufferDataIsReady(sampleBuffer)) { + CFRelease(sampleBuffer); + [self readNextBuffer]; + return; + } + + [self publishSampleBuffer:sampleBuffer]; +} + +- (void)publishSampleBuffer:(CMSampleBufferRef)sampleBuffer { + CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + Float64 presentationDifference = + CMTimeGetSeconds(CMTimeSubtract(presentationTime, _lastPresentationTime)); + _lastPresentationTime = presentationTime; + int64_t presentationDifferenceRound = lroundf(presentationDifference * NSEC_PER_SEC); + + __block dispatch_source_t timer = [self createStrictTimer]; + // Strict timer that will fire `presentationDifferenceRound` ns from now and never again. + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, presentationDifferenceRound), + DISPATCH_TIME_FOREVER, + 0); + dispatch_source_set_event_handler(timer, ^{ + dispatch_source_cancel(timer); + timer = nil; + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixelBuffer) { + CFRelease(sampleBuffer); + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + [self readNextBuffer]; + }); + return; + } + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer]; + NSTimeInterval timeStampSeconds = CACurrentMediaTime(); + int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer + rotation:0 + timeStampNs:timeStampNs]; + CFRelease(sampleBuffer); + + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + [self readNextBuffer]; + }); + + [self.delegate capturer:self didCaptureVideoFrame:videoFrame]; + }); + dispatch_activate(timer); +} + +- (dispatch_source_t)createStrictTimer { + dispatch_source_t timer = dispatch_source_create( + DISPATCH_SOURCE_TYPE_TIMER, 0, DISPATCH_TIMER_STRICT, [self frameQueue]); + return timer; +} + +- (void)dealloc { + [self stopCapture]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor+Private.h b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor+Private.h new file mode 100644 index 0000000000..b5c786be18 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor+Private.h @@ -0,0 +1,26 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNetworkMonitor.h" + +#include "sdk/objc/native/src/network_monitor_observer.h" + +@interface RTCNetworkMonitor () + +/** `observer` is a raw pointer and should be kept alive + * for this object's lifetime. + */ +- (instancetype)initWithObserver:(webrtc::NetworkMonitorObserver *)observer + NS_DESIGNATED_INITIALIZER; + +/** Stops the receiver from posting updates to `observer`. */ +- (void)stop; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.h b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.h new file mode 100644 index 0000000000..21d22f5463 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.h @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +NS_ASSUME_NONNULL_BEGIN + +/** Listens for NWPathMonitor updates and forwards the results to a C++ + * observer. + */ +@interface RTCNetworkMonitor : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.mm b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.mm new file mode 100644 index 0000000000..7e75b2b4c0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/network/RTCNetworkMonitor.mm @@ -0,0 +1,126 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNetworkMonitor+Private.h" + +#import <Network/Network.h> + +#import "base/RTCLogging.h" +#import "helpers/RTCDispatcher+Private.h" + +#include "rtc_base/string_utils.h" + +namespace { + +rtc::AdapterType AdapterTypeFromInterfaceType(nw_interface_type_t interfaceType) { + rtc::AdapterType adapterType = rtc::ADAPTER_TYPE_UNKNOWN; + switch (interfaceType) { + case nw_interface_type_other: + adapterType = rtc::ADAPTER_TYPE_UNKNOWN; + break; + case nw_interface_type_wifi: + adapterType = rtc::ADAPTER_TYPE_WIFI; + break; + case nw_interface_type_cellular: + adapterType = rtc::ADAPTER_TYPE_CELLULAR; + break; + case nw_interface_type_wired: + adapterType = rtc::ADAPTER_TYPE_ETHERNET; + break; + case nw_interface_type_loopback: + adapterType = rtc::ADAPTER_TYPE_LOOPBACK; + break; + default: + adapterType = rtc::ADAPTER_TYPE_UNKNOWN; + break; + } + return adapterType; +} + +} // namespace + +@implementation RTCNetworkMonitor { + webrtc::NetworkMonitorObserver *_observer; + nw_path_monitor_t _pathMonitor; + dispatch_queue_t _monitorQueue; +} + +- (instancetype)initWithObserver:(webrtc::NetworkMonitorObserver *)observer { + RTC_DCHECK(observer); + if (self = [super init]) { + _observer = observer; + if (@available(iOS 12, *)) { + _pathMonitor = nw_path_monitor_create(); + if (_pathMonitor == nil) { + RTCLog(@"nw_path_monitor_create failed."); + return nil; + } + RTCLog(@"NW path monitor created."); + __weak RTCNetworkMonitor *weakSelf = self; + nw_path_monitor_set_update_handler(_pathMonitor, ^(nw_path_t path) { + if (weakSelf == nil) { + return; + } + RTCNetworkMonitor *strongSelf = weakSelf; + RTCLog(@"NW path monitor: updated."); + nw_path_status_t status = nw_path_get_status(path); + if (status == nw_path_status_invalid) { + RTCLog(@"NW path monitor status: invalid."); + } else if (status == nw_path_status_unsatisfied) { + RTCLog(@"NW path monitor status: unsatisfied."); + } else if (status == nw_path_status_satisfied) { + RTCLog(@"NW path monitor status: satisfied."); + } else if (status == nw_path_status_satisfiable) { + RTCLog(@"NW path monitor status: satisfiable."); + } + std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp> *map = + new std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp>(); + nw_path_enumerate_interfaces( + path, (nw_path_enumerate_interfaces_block_t) ^ (nw_interface_t interface) { + const char *name = nw_interface_get_name(interface); + nw_interface_type_t interfaceType = nw_interface_get_type(interface); + RTCLog(@"NW path monitor available interface: %s", name); + rtc::AdapterType adapterType = AdapterTypeFromInterfaceType(interfaceType); + map->insert(std::pair<std::string, rtc::AdapterType>(name, adapterType)); + }); + @synchronized(strongSelf) { + webrtc::NetworkMonitorObserver *observer = strongSelf->_observer; + if (observer) { + observer->OnPathUpdate(std::move(*map)); + } + } + delete map; + }); + nw_path_monitor_set_queue( + _pathMonitor, + [RTC_OBJC_TYPE(RTCDispatcher) dispatchQueueForType:RTCDispatcherTypeNetworkMonitor]); + nw_path_monitor_start(_pathMonitor); + } + } + return self; +} + +- (void)cancel { + if (@available(iOS 12, *)) { + nw_path_monitor_cancel(_pathMonitor); + } +} +- (void)stop { + [self cancel]; + @synchronized(self) { + _observer = nil; + } +} + +- (void)dealloc { + [self cancel]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h new file mode 100644 index 0000000000..e5987fe22a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.h @@ -0,0 +1,17 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLI420Renderer : RTCMTLRenderer +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm new file mode 100644 index 0000000000..eba8800240 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLI420Renderer.mm @@ -0,0 +1,177 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLI420Renderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCI420Buffer.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" + +#import "RTCMTLRenderer+Private.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } Varyings; + + vertex Varyings vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + unsigned int vid[[vertex_id]]) { + Varyings out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + + return out; + } + + fragment half4 fragmentColorConversion( + Varyings in[[stage_in]], + texture2d<float, access::sample> textureY[[texture(0)]], + texture2d<float, access::sample> textureU[[texture(1)]], + texture2d<float, access::sample> textureV[[texture(2)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + float y; + float u; + float v; + float r; + float g; + float b; + // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php + y = textureY.sample(s, in.texcoord).r; + u = textureU.sample(s, in.texcoord).r; + v = textureV.sample(s, in.texcoord).r; + u = u - 0.5; + v = v - 0.5; + r = y + 1.403 * v; + g = y - 0.344 * u - 0.714 * v; + b = y + 1.770 * u; + + float4 out = float4(r, g, b, 1.0); + + return half4(out); + }); + +@implementation RTCMTLI420Renderer { + // Textures. + id<MTLTexture> _yTexture; + id<MTLTexture> _uTexture; + id<MTLTexture> _vTexture; + + MTLTextureDescriptor *_descriptor; + MTLTextureDescriptor *_chromaDescriptor; + + int _width; + int _height; + int _chromaWidth; + int _chromaHeight; +} + +#pragma mark - Virtual + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + *width = frame.width; + *height = frame.height; + *cropWidth = frame.width; + *cropHeight = frame.height; + *cropX = 0; + *cropY = 0; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (![super setupTexturesForFrame:frame]) { + return NO; + } + + id<MTLDevice> device = [self currentMetalDevice]; + if (!device) { + return NO; + } + + // Chroma size must be >= 1 as per the Apple documentation, so skip ?x1 + // and 1x? frames. + // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=14892 + if (frame.width < 2 || frame.height < 2) { + return NO; + } + + id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420]; + + // Luma (y) texture. + if (!_descriptor || _width != frame.width || _height != frame.height) { + _width = frame.width; + _height = frame.height; + _descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:_width + height:_height + mipmapped:NO]; + _descriptor.usage = MTLTextureUsageShaderRead; + _yTexture = [device newTextureWithDescriptor:_descriptor]; + } + + // Chroma (u,v) textures + [_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height) + mipmapLevel:0 + withBytes:buffer.dataY + bytesPerRow:buffer.strideY]; + + if (!_chromaDescriptor || _chromaWidth != frame.width / 2 || _chromaHeight != frame.height / 2) { + _chromaWidth = frame.width / 2; + _chromaHeight = frame.height / 2; + _chromaDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm + width:_chromaWidth + height:_chromaHeight + mipmapped:NO]; + _chromaDescriptor.usage = MTLTextureUsageShaderRead; + _uTexture = [device newTextureWithDescriptor:_chromaDescriptor]; + _vTexture = [device newTextureWithDescriptor:_chromaDescriptor]; + } + + [_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) + mipmapLevel:0 + withBytes:buffer.dataU + bytesPerRow:buffer.strideU]; + [_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) + mipmapLevel:0 + withBytes:buffer.dataV + bytesPerRow:buffer.strideV]; + + return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil); +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_yTexture atIndex:0]; + [renderEncoder setFragmentTexture:_uTexture atIndex:1]; + [renderEncoder setFragmentTexture:_vTexture atIndex:2]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h new file mode 100644 index 0000000000..5a2e7d380f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.h @@ -0,0 +1,22 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AppKit/AppKit.h> + +#import "RTCVideoRenderer.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMTLNSVideoView) : NSView <RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + ++ (BOOL)isMetalAvailable; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m new file mode 100644 index 0000000000..625fb1caa7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNSVideoView.m @@ -0,0 +1,122 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLNSVideoView.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCVideoFrame.h" + +#import "RTCMTLI420Renderer.h" + +@interface RTC_OBJC_TYPE (RTCMTLNSVideoView) +()<MTKViewDelegate> @property(nonatomic) id<RTCMTLRenderer> renderer; +@property(nonatomic, strong) MTKView *metalView; +@property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@end + +@implementation RTC_OBJC_TYPE (RTCMTLNSVideoView) { + id<RTCMTLRenderer> _renderer; +} + +@synthesize delegate = _delegate; +@synthesize renderer = _renderer; +@synthesize metalView = _metalView; +@synthesize videoFrame = _videoFrame; + +- (instancetype)initWithFrame:(CGRect)frameRect { + self = [super initWithFrame:frameRect]; + if (self) { + [self configure]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aCoder { + self = [super initWithCoder:aCoder]; + if (self) { + [self configure]; + } + return self; +} + +#pragma mark - Private + ++ (BOOL)isMetalAvailable { + return [MTLCopyAllDevices() count] > 0; +} + +- (void)configure { + if ([[self class] isMetalAvailable]) { + _metalView = [[MTKView alloc] initWithFrame:self.bounds]; + [self addSubview:_metalView]; + _metalView.layerContentsPlacement = NSViewLayerContentsPlacementScaleProportionallyToFit; + _metalView.translatesAutoresizingMaskIntoConstraints = NO; + _metalView.framebufferOnly = YES; + _metalView.delegate = self; + + _renderer = [[RTCMTLI420Renderer alloc] init]; + if (![(RTCMTLI420Renderer *)_renderer addRenderingDestination:_metalView]) { + _renderer = nil; + }; + } +} + +- (void)updateConstraints { + NSDictionary *views = NSDictionaryOfVariableBindings(_metalView); + + NSArray *constraintsHorizontal = + [NSLayoutConstraint constraintsWithVisualFormat:@"H:|-0-[_metalView]-0-|" + options:0 + metrics:nil + views:views]; + [self addConstraints:constraintsHorizontal]; + + NSArray *constraintsVertical = + [NSLayoutConstraint constraintsWithVisualFormat:@"V:|-0-[_metalView]-0-|" + options:0 + metrics:nil + views:views]; + [self addConstraints:constraintsVertical]; + [super updateConstraints]; +} + +#pragma mark - MTKViewDelegate methods +- (void)drawInMTKView:(nonnull MTKView *)view { + if (self.videoFrame == nil) { + return; + } + if (view == self.metalView) { + [_renderer drawFrame:self.videoFrame]; + } +} + +- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +- (void)setSize:(CGSize)size { + _metalView.drawableSize = size; + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate videoView:self didChangeVideoSize:size]; + }); + [_metalView draw]; +} + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (frame == nil) { + return; + } + self.videoFrame = [frame newI420VideoFrame]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h new file mode 100644 index 0000000000..866b7ea17e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.h @@ -0,0 +1,18 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLNV12Renderer : RTCMTLRenderer + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm new file mode 100644 index 0000000000..7b037c6dbc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLNV12Renderer.mm @@ -0,0 +1,164 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLNV12Renderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "RTCMTLRenderer+Private.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#include "rtc_base/checks.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } Varyings; + + vertex Varyings vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + unsigned int vid[[vertex_id]]) { + Varyings out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + return out; + } + + // Receiving YCrCb textures. + fragment half4 fragmentColorConversion( + Varyings in[[stage_in]], + texture2d<float, access::sample> textureY[[texture(0)]], + texture2d<float, access::sample> textureCbCr[[texture(1)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + float y; + float2 uv; + y = textureY.sample(s, in.texcoord).r; + uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5); + + // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php + float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0); + + return half4(out); + }); + +@implementation RTCMTLNV12Renderer { + // Textures. + CVMetalTextureCacheRef _textureCache; + id<MTLTexture> _yTexture; + id<MTLTexture> _CrCbTexture; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + if ([super addRenderingDestination:view]) { + return [self initializeTextureCache]; + } + return NO; +} + +- (BOOL)initializeTextureCache { + CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice], + nil, &_textureCache); + if (status != kCVReturnSuccess) { + RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); + return NO; + } + + return YES; +} + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + *width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer); + *height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer); + *cropWidth = pixelBuffer.cropWidth; + *cropHeight = pixelBuffer.cropHeight; + *cropX = pixelBuffer.cropX; + *cropY = pixelBuffer.cropY; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + if (![super setupTexturesForFrame:frame]) { + return NO; + } + CVPixelBufferRef pixelBuffer = ((RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer).pixelBuffer; + + id<MTLTexture> lumaTexture = nil; + id<MTLTexture> chromaTexture = nil; + CVMetalTextureRef outTexture = nullptr; + + // Luma (y) texture. + int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); + int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); + + int indexPlane = 0; + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth, + lumaHeight, indexPlane, &outTexture); + + if (result == kCVReturnSuccess) { + lumaTexture = CVMetalTextureGetTexture(outTexture); + } + + // Same as CFRelease except it can be passed NULL without crashing. + CVBufferRelease(outTexture); + outTexture = nullptr; + + // Chroma (CrCb) texture. + indexPlane = 1; + result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, lumaWidth / 2, + lumaHeight / 2, indexPlane, &outTexture); + if (result == kCVReturnSuccess) { + chromaTexture = CVMetalTextureGetTexture(outTexture); + } + CVBufferRelease(outTexture); + + if (lumaTexture != nil && chromaTexture != nil) { + _yTexture = lumaTexture; + _CrCbTexture = chromaTexture; + return YES; + } + return NO; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_yTexture atIndex:0]; + [renderEncoder setFragmentTexture:_CrCbTexture atIndex:1]; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + } +} +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h new file mode 100644 index 0000000000..9db422cd22 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.h @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMTLRenderer.h" + +/** @abstract RGB/BGR renderer. + * @discussion This renderer handles both kCVPixelFormatType_32BGRA and + * kCVPixelFormatType_32ARGB. + */ +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLRGBRenderer : RTCMTLRenderer + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm new file mode 100644 index 0000000000..e5dc4ef80a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRGBRenderer.mm @@ -0,0 +1,164 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLRGBRenderer.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "RTCMTLRenderer+Private.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#include "rtc_base/checks.h" + +static NSString *const shaderSource = MTL_STRINGIFY( + using namespace metal; + + typedef struct { + packed_float2 position; + packed_float2 texcoord; + } Vertex; + + typedef struct { + float4 position[[position]]; + float2 texcoord; + } VertexIO; + + vertex VertexIO vertexPassthrough(constant Vertex *verticies[[buffer(0)]], + uint vid[[vertex_id]]) { + VertexIO out; + constant Vertex &v = verticies[vid]; + out.position = float4(float2(v.position), 0.0, 1.0); + out.texcoord = v.texcoord; + return out; + } + + fragment half4 fragmentColorConversion(VertexIO in[[stage_in]], + texture2d<half, access::sample> texture[[texture(0)]], + constant bool &isARGB[[buffer(0)]]) { + constexpr sampler s(address::clamp_to_edge, filter::linear); + + half4 out = texture.sample(s, in.texcoord); + if (isARGB) { + out = half4(out.g, out.b, out.a, out.r); + } + + return out; + }); + +@implementation RTCMTLRGBRenderer { + // Textures. + CVMetalTextureCacheRef _textureCache; + id<MTLTexture> _texture; + + // Uniforms. + id<MTLBuffer> _uniformsBuffer; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + if ([super addRenderingDestination:view]) { + return [self initializeTextureCache]; + } + return NO; +} + +- (BOOL)initializeTextureCache { + CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice], + nil, &_textureCache); + if (status != kCVReturnSuccess) { + RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); + return NO; + } + + return YES; +} + +- (NSString *)shaderSource { + return shaderSource; +} + +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *pixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + *width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer); + *height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer); + *cropWidth = pixelBuffer.cropWidth; + *cropHeight = pixelBuffer.cropHeight; + *cropX = pixelBuffer.cropX; + *cropY = pixelBuffer.cropY; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + if (![super setupTexturesForFrame:frame]) { + return NO; + } + CVPixelBufferRef pixelBuffer = ((RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer).pixelBuffer; + + id<MTLTexture> gpuTexture = nil; + CVMetalTextureRef textureOut = nullptr; + bool isARGB; + + int width = CVPixelBufferGetWidth(pixelBuffer); + int height = CVPixelBufferGetHeight(pixelBuffer); + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + + MTLPixelFormat mtlPixelFormat; + if (pixelFormat == kCVPixelFormatType_32BGRA) { + mtlPixelFormat = MTLPixelFormatBGRA8Unorm; + isARGB = false; + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + mtlPixelFormat = MTLPixelFormatRGBA8Unorm; + isARGB = true; + } else { + RTC_DCHECK_NOTREACHED(); + return NO; + } + + CVReturn result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, nil, mtlPixelFormat, + width, height, 0, &textureOut); + if (result == kCVReturnSuccess) { + gpuTexture = CVMetalTextureGetTexture(textureOut); + } + CVBufferRelease(textureOut); + + if (gpuTexture != nil) { + _texture = gpuTexture; + _uniformsBuffer = + [[self currentMetalDevice] newBufferWithBytes:&isARGB + length:sizeof(isARGB) + options:MTLResourceCPUCacheModeDefaultCache]; + return YES; + } + + return NO; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + [renderEncoder setFragmentTexture:_texture atIndex:0]; + [renderEncoder setFragmentBuffer:_uniformsBuffer offset:0 atIndex:0]; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h new file mode 100644 index 0000000000..916d4d4430 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer+Private.h @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Metal/Metal.h> + +#import "RTCMTLRenderer.h" + +#define MTL_STRINGIFY(s) @ #s + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCMTLRenderer (Private) +- (nullable id<MTLDevice>)currentMetalDevice; +- (NSString *)shaderSource; +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder; +- (void)getWidth:(nonnull int *)width + height:(nonnull int *)height + cropWidth:(nonnull int *)cropWidth + cropHeight:(nonnull int *)cropHeight + cropX:(nonnull int *)cropX + cropY:(nonnull int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame; +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h new file mode 100644 index 0000000000..aa31545973 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.h @@ -0,0 +1,61 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#if TARGET_OS_IPHONE +#import <UIKit/UIKit.h> +#else +#import <AppKit/AppKit.h> +#endif + +#import "base/RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN +/** + * Protocol defining ability to render RTCVideoFrame in Metal enabled views. + */ +@protocol RTCMTLRenderer <NSObject> + +/** + * Method to be implemented to perform actual rendering of the provided frame. + * + * @param frame The frame to be rendered. + */ +- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +/** + * Sets the provided view as rendering destination if possible. + * + * If not possible method returns NO and callers of the method are responisble for performing + * cleanups. + */ + +#if TARGET_OS_IOS +- (BOOL)addRenderingDestination:(__kindof UIView *)view; +#else +- (BOOL)addRenderingDestination:(__kindof NSView *)view; +#endif + +@end + +/** + * Implementation of RTCMTLRenderer protocol. + */ +NS_AVAILABLE(10_11, 9_0) +@interface RTCMTLRenderer : NSObject <RTCMTLRenderer> + +/** @abstract A wrapped RTCVideoRotation, or nil. + @discussion When not nil, the rotation of the actual frame is ignored when rendering. + */ +@property(atomic, nullable) NSValue *rotationOverride; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm new file mode 100644 index 0000000000..410590a7b1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLRenderer.mm @@ -0,0 +1,328 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLRenderer+Private.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" + +#include "api/video/video_rotation.h" +#include "rtc_base/checks.h" + +// As defined in shaderSource. +static NSString *const vertexFunctionName = @"vertexPassthrough"; +static NSString *const fragmentFunctionName = @"fragmentColorConversion"; + +static NSString *const pipelineDescriptorLabel = @"RTCPipeline"; +static NSString *const commandBufferLabel = @"RTCCommandBuffer"; +static NSString *const renderEncoderLabel = @"RTCEncoder"; +static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame"; + +// Computes the texture coordinates given rotation and cropping. +static inline void getCubeVertexData(int cropX, + int cropY, + int cropWidth, + int cropHeight, + size_t frameWidth, + size_t frameHeight, + RTCVideoRotation rotation, + float *buffer) { + // The computed values are the adjusted texture coordinates, in [0..1]. + // For the left and top, 0.0 means no cropping and e.g. 0.2 means we're skipping 20% of the + // left/top edge. + // For the right and bottom, 1.0 means no cropping and e.g. 0.8 means we're skipping 20% of the + // right/bottom edge (i.e. render up to 80% of the width/height). + float cropLeft = cropX / (float)frameWidth; + float cropRight = (cropX + cropWidth) / (float)frameWidth; + float cropTop = cropY / (float)frameHeight; + float cropBottom = (cropY + cropHeight) / (float)frameHeight; + + // These arrays map the view coordinates to texture coordinates, taking cropping and rotation + // into account. The first two columns are view coordinates, the last two are texture coordinates. + switch (rotation) { + case RTCVideoRotation_0: { + float values[16] = {-1.0, -1.0, cropLeft, cropBottom, + 1.0, -1.0, cropRight, cropBottom, + -1.0, 1.0, cropLeft, cropTop, + 1.0, 1.0, cropRight, cropTop}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_90: { + float values[16] = {-1.0, -1.0, cropRight, cropBottom, + 1.0, -1.0, cropRight, cropTop, + -1.0, 1.0, cropLeft, cropBottom, + 1.0, 1.0, cropLeft, cropTop}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_180: { + float values[16] = {-1.0, -1.0, cropRight, cropTop, + 1.0, -1.0, cropLeft, cropTop, + -1.0, 1.0, cropRight, cropBottom, + 1.0, 1.0, cropLeft, cropBottom}; + memcpy(buffer, &values, sizeof(values)); + } break; + case RTCVideoRotation_270: { + float values[16] = {-1.0, -1.0, cropLeft, cropTop, + 1.0, -1.0, cropLeft, cropBottom, + -1.0, 1.0, cropRight, cropTop, + 1.0, 1.0, cropRight, cropBottom}; + memcpy(buffer, &values, sizeof(values)); + } break; + } +} + +// The max number of command buffers in flight (submitted to GPU). +// For now setting it up to 1. +// In future we might use triple buffering method if it improves performance. +static const NSInteger kMaxInflightBuffers = 1; + +@implementation RTCMTLRenderer { + __kindof MTKView *_view; + + // Controller. + dispatch_semaphore_t _inflight_semaphore; + + // Renderer. + id<MTLDevice> _device; + id<MTLCommandQueue> _commandQueue; + id<MTLLibrary> _defaultLibrary; + id<MTLRenderPipelineState> _pipelineState; + + // Buffers. + id<MTLBuffer> _vertexBuffer; + + // Values affecting the vertex buffer. Stored for comparison to avoid unnecessary recreation. + int _oldFrameWidth; + int _oldFrameHeight; + int _oldCropWidth; + int _oldCropHeight; + int _oldCropX; + int _oldCropY; + RTCVideoRotation _oldRotation; +} + +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)init { + if (self = [super init]) { + _inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers); + } + + return self; +} + +- (BOOL)addRenderingDestination:(__kindof MTKView *)view { + return [self setupWithView:view]; +} + +#pragma mark - Private + +- (BOOL)setupWithView:(__kindof MTKView *)view { + BOOL success = NO; + if ([self setupMetal]) { + _view = view; + view.device = _device; + view.preferredFramesPerSecond = 30; + view.autoResizeDrawable = NO; + + [self loadAssets]; + + float vertexBufferArray[16] = {0}; + _vertexBuffer = [_device newBufferWithBytes:vertexBufferArray + length:sizeof(vertexBufferArray) + options:MTLResourceCPUCacheModeWriteCombined]; + success = YES; + } + return success; +} +#pragma mark - Inheritance + +- (id<MTLDevice>)currentMetalDevice { + return _device; +} + +- (NSString *)shaderSource { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; + return nil; +} + +- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; +} + +- (void)getWidth:(int *)width + height:(int *)height + cropWidth:(int *)cropWidth + cropHeight:(int *)cropHeight + cropX:(int *)cropX + cropY:(int *)cropY + ofFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + RTC_DCHECK_NOTREACHED() << "Virtual method not implemented in subclass."; +} + +- (BOOL)setupTexturesForFrame:(nonnull RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + // Apply rotation override if set. + RTCVideoRotation rotation; + NSValue *rotationOverride = self.rotationOverride; + if (rotationOverride) { +#if defined(__IPHONE_11_0) && defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && \ + (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_11_0) + if (@available(iOS 11, *)) { + [rotationOverride getValue:&rotation size:sizeof(rotation)]; + } else +#endif + { + [rotationOverride getValue:&rotation]; + } + } else { + rotation = frame.rotation; + } + + int frameWidth, frameHeight, cropWidth, cropHeight, cropX, cropY; + [self getWidth:&frameWidth + height:&frameHeight + cropWidth:&cropWidth + cropHeight:&cropHeight + cropX:&cropX + cropY:&cropY + ofFrame:frame]; + + // Recompute the texture cropping and recreate vertexBuffer if necessary. + if (cropX != _oldCropX || cropY != _oldCropY || cropWidth != _oldCropWidth || + cropHeight != _oldCropHeight || rotation != _oldRotation || frameWidth != _oldFrameWidth || + frameHeight != _oldFrameHeight) { + getCubeVertexData(cropX, + cropY, + cropWidth, + cropHeight, + frameWidth, + frameHeight, + rotation, + (float *)_vertexBuffer.contents); + _oldCropX = cropX; + _oldCropY = cropY; + _oldCropWidth = cropWidth; + _oldCropHeight = cropHeight; + _oldRotation = rotation; + _oldFrameWidth = frameWidth; + _oldFrameHeight = frameHeight; + } + + return YES; +} + +#pragma mark - GPU methods + +- (BOOL)setupMetal { + // Set the view to use the default device. + _device = MTLCreateSystemDefaultDevice(); + if (!_device) { + return NO; + } + + // Create a new command queue. + _commandQueue = [_device newCommandQueue]; + + // Load metal library from source. + NSError *libraryError = nil; + NSString *shaderSource = [self shaderSource]; + + id<MTLLibrary> sourceLibrary = + [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError]; + + if (libraryError) { + RTCLogError(@"Metal: Library with source failed\n%@", libraryError); + return NO; + } + + if (!sourceLibrary) { + RTCLogError(@"Metal: Failed to load library. %@", libraryError); + return NO; + } + _defaultLibrary = sourceLibrary; + + return YES; +} + +- (void)loadAssets { + id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName]; + id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName]; + + MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init]; + pipelineDescriptor.label = pipelineDescriptorLabel; + pipelineDescriptor.vertexFunction = vertexFunction; + pipelineDescriptor.fragmentFunction = fragmentFunction; + pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat; + pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid; + NSError *error = nil; + _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error]; + + if (!_pipelineState) { + RTCLogError(@"Metal: Failed to create pipeline state. %@", error); + } +} + +- (void)render { + id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer]; + commandBuffer.label = commandBufferLabel; + + __block dispatch_semaphore_t block_semaphore = _inflight_semaphore; + [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) { + // GPU work completed. + dispatch_semaphore_signal(block_semaphore); + }]; + + MTLRenderPassDescriptor *renderPassDescriptor = _view.currentRenderPassDescriptor; + if (renderPassDescriptor) { // Valid drawable. + id<MTLRenderCommandEncoder> renderEncoder = + [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor]; + renderEncoder.label = renderEncoderLabel; + + // Set context state. + [renderEncoder pushDebugGroup:renderEncoderDebugGroup]; + [renderEncoder setRenderPipelineState:_pipelineState]; + [renderEncoder setVertexBuffer:_vertexBuffer offset:0 atIndex:0]; + [self uploadTexturesToRenderEncoder:renderEncoder]; + + [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip + vertexStart:0 + vertexCount:4 + instanceCount:1]; + [renderEncoder popDebugGroup]; + [renderEncoder endEncoding]; + + [commandBuffer presentDrawable:_view.currentDrawable]; + } + + // CPU work is completed, GPU work can be started. + [commandBuffer commit]; +} + +#pragma mark - RTCMTLRenderer + +- (void)drawFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + @autoreleasepool { + // Wait until the inflight (curently sent to GPU) command buffer + // has completed the GPU work. + dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER); + + if ([self setupTexturesForFrame:frame]) { + [self render]; + } else { + dispatch_semaphore_signal(_inflight_semaphore); + } + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h new file mode 100644 index 0000000000..3320d12076 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.h @@ -0,0 +1,44 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrame.h" +#import "RTCVideoRenderer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * RTCMTLVideoView is thin wrapper around MTKView. + * + * It has id<RTCVideoRenderer> property that renders video frames in the view's + * bounds using Metal. + */ +NS_CLASS_AVAILABLE_IOS(9) + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCMTLVideoView) : UIView<RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + +@property(nonatomic) UIViewContentMode videoContentMode; + +/** @abstract Enables/disables rendering. + */ +@property(nonatomic, getter=isEnabled) BOOL enabled; + +/** @abstract Wrapped RTCVideoRotation, or nil. + */ +@property(nonatomic, nullable) NSValue* rotationOverride; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m new file mode 100644 index 0000000000..c5d9e4385f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/metal/RTCMTLVideoView.m @@ -0,0 +1,265 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCMTLVideoView.h" + +#import <Metal/Metal.h> +#import <MetalKit/MetalKit.h> + +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#import "RTCMTLI420Renderer.h" +#import "RTCMTLNV12Renderer.h" +#import "RTCMTLRGBRenderer.h" + +// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime. +// Linking errors occur when compiling for architectures that don't support Metal. +#define MTKViewClass NSClassFromString(@"MTKView") +#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer") +#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer") +#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer") + +@interface RTC_OBJC_TYPE (RTCMTLVideoView) +()<MTKViewDelegate> @property(nonatomic) RTCMTLI420Renderer *rendererI420; +@property(nonatomic) RTCMTLNV12Renderer *rendererNV12; +@property(nonatomic) RTCMTLRGBRenderer *rendererRGB; +@property(nonatomic) MTKView *metalView; +@property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@property(nonatomic) CGSize videoFrameSize; +@property(nonatomic) int64_t lastFrameTimeNs; +@end + +@implementation RTC_OBJC_TYPE (RTCMTLVideoView) + +@synthesize delegate = _delegate; +@synthesize rendererI420 = _rendererI420; +@synthesize rendererNV12 = _rendererNV12; +@synthesize rendererRGB = _rendererRGB; +@synthesize metalView = _metalView; +@synthesize videoFrame = _videoFrame; +@synthesize videoFrameSize = _videoFrameSize; +@synthesize lastFrameTimeNs = _lastFrameTimeNs; +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)initWithFrame:(CGRect)frameRect { + self = [super initWithFrame:frameRect]; + if (self) { + [self configure]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aCoder { + self = [super initWithCoder:aCoder]; + if (self) { + [self configure]; + } + return self; +} + +- (BOOL)isEnabled { + return !self.metalView.paused; +} + +- (void)setEnabled:(BOOL)enabled { + self.metalView.paused = !enabled; +} + +- (UIViewContentMode)videoContentMode { + return self.metalView.contentMode; +} + +- (void)setVideoContentMode:(UIViewContentMode)mode { + self.metalView.contentMode = mode; +} + +#pragma mark - Private + ++ (BOOL)isMetalAvailable { + return MTLCreateSystemDefaultDevice() != nil; +} + ++ (MTKView *)createMetalView:(CGRect)frame { + return [[MTKViewClass alloc] initWithFrame:frame]; +} + ++ (RTCMTLNV12Renderer *)createNV12Renderer { + return [[RTCMTLNV12RendererClass alloc] init]; +} + ++ (RTCMTLI420Renderer *)createI420Renderer { + return [[RTCMTLI420RendererClass alloc] init]; +} + ++ (RTCMTLRGBRenderer *)createRGBRenderer { + return [[RTCMTLRGBRenderer alloc] init]; +} + +- (void)configure { + NSAssert([RTC_OBJC_TYPE(RTCMTLVideoView) isMetalAvailable], + @"Metal not availiable on this device"); + + self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds]; + self.metalView.delegate = self; + self.metalView.contentMode = UIViewContentModeScaleAspectFill; + [self addSubview:self.metalView]; + self.videoFrameSize = CGSizeZero; +} + +- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { + [super setMultipleTouchEnabled:multipleTouchEnabled]; + self.metalView.multipleTouchEnabled = multipleTouchEnabled; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + + CGRect bounds = self.bounds; + self.metalView.frame = bounds; + if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) { + self.metalView.drawableSize = [self drawableSize]; + } else { + self.metalView.drawableSize = bounds.size; + } +} + +#pragma mark - MTKViewDelegate methods + +- (void)drawInMTKView:(nonnull MTKView *)view { + NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance."); + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = self.videoFrame; + // Skip rendering if we've already rendered this frame. + if (!videoFrame || videoFrame.width <= 0 || videoFrame.height <= 0 || + videoFrame.timeStampNs == self.lastFrameTimeNs) { + return; + } + + if (CGRectIsEmpty(view.bounds)) { + return; + } + + RTCMTLRenderer *renderer; + if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer; + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); + if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) { + if (!self.rendererRGB) { + self.rendererRGB = [RTC_OBJC_TYPE(RTCMTLVideoView) createRGBRenderer]; + if (![self.rendererRGB addRenderingDestination:self.metalView]) { + self.rendererRGB = nil; + RTCLogError(@"Failed to create RGB renderer"); + return; + } + } + renderer = self.rendererRGB; + } else { + if (!self.rendererNV12) { + self.rendererNV12 = [RTC_OBJC_TYPE(RTCMTLVideoView) createNV12Renderer]; + if (![self.rendererNV12 addRenderingDestination:self.metalView]) { + self.rendererNV12 = nil; + RTCLogError(@"Failed to create NV12 renderer"); + return; + } + } + renderer = self.rendererNV12; + } + } else { + if (!self.rendererI420) { + self.rendererI420 = [RTC_OBJC_TYPE(RTCMTLVideoView) createI420Renderer]; + if (![self.rendererI420 addRenderingDestination:self.metalView]) { + self.rendererI420 = nil; + RTCLogError(@"Failed to create I420 renderer"); + return; + } + } + renderer = self.rendererI420; + } + + renderer.rotationOverride = self.rotationOverride; + + [renderer drawFrame:videoFrame]; + self.lastFrameTimeNs = videoFrame.timeStampNs; +} + +- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { +} + +#pragma mark - + +- (void)setRotationOverride:(NSValue *)rotationOverride { + _rotationOverride = rotationOverride; + + self.metalView.drawableSize = [self drawableSize]; + [self setNeedsLayout]; +} + +- (RTCVideoRotation)frameRotation { + if (self.rotationOverride) { + RTCVideoRotation rotation; + if (@available(iOS 11, *)) { + [self.rotationOverride getValue:&rotation size:sizeof(rotation)]; + } else { + [self.rotationOverride getValue:&rotation]; + } + return rotation; + } + + return self.videoFrame.rotation; +} + +- (CGSize)drawableSize { + // Flip width/height if the rotations are not the same. + CGSize videoFrameSize = self.videoFrameSize; + RTCVideoRotation frameRotation = [self frameRotation]; + + BOOL useLandscape = + (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180); + BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) || + (self.videoFrame.rotation == RTCVideoRotation_180); + + if (useLandscape == sizeIsLandscape) { + return videoFrameSize; + } else { + return CGSizeMake(videoFrameSize.height, videoFrameSize.width); + } +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +- (void)setSize:(CGSize)size { + __weak RTC_OBJC_TYPE(RTCMTLVideoView) *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCMTLVideoView) *strongSelf = weakSelf; + + strongSelf.videoFrameSize = size; + CGSize drawableSize = [strongSelf drawableSize]; + + strongSelf.metalView.drawableSize = drawableSize; + [strongSelf setNeedsLayout]; + [strongSelf.delegate videoView:self didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + if (!self.isEnabled) { + return; + } + + if (frame == nil) { + RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); + return; + } + self.videoFrame = frame; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h new file mode 100644 index 0000000000..71a073ab21 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Default RTCVideoViewShading that will be used in RTCNSGLVideoView + * and RTCEAGLVideoView if no external shader is specified. This shader will render + * the video in a rectangle without any color or geometric transformations. + */ +@interface RTCDefaultShader : NSObject <RTC_OBJC_TYPE (RTCVideoViewShading)> + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm new file mode 100644 index 0000000000..9d686f625c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDefaultShader.mm @@ -0,0 +1,201 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDefaultShader.h" + +#import <OpenGLES/ES3/gl.h> + +#import "RTCOpenGLDefines.h" +#import "RTCShader.h" +#import "base/RTCLogging.h" + +#include "absl/types/optional.h" + +static const int kYTextureUnit = 0; +static const int kUTextureUnit = 1; +static const int kVTextureUnit = 2; +static const int kUvTextureUnit = 1; + +// Fragment shader converts YUV values from input textures into a final RGB +// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. +static const char kI420FragmentShaderSource[] = + SHADER_VERSION + "precision highp float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureU;\n" + "uniform lowp sampler2D s_textureV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " float y, u, v, r, g, b;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" + " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" + " u = u - 0.5;\n" + " v = v - 0.5;\n" + " r = y + 1.403 * v;\n" + " g = y - 0.344 * u - 0.714 * v;\n" + " b = y + 1.770 * u;\n" + " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" + " }\n"; + +static const char kNV12FragmentShaderSource[] = + SHADER_VERSION + "precision mediump float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureUV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " mediump float y;\n" + " mediump vec2 uv;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " uv = " FRAGMENT_SHADER_TEXTURE "(s_textureUV, v_texcoord).ra -\n" + " vec2(0.5, 0.5);\n" + " " FRAGMENT_SHADER_COLOR " = vec4(y + 1.403 * uv.y,\n" + " y - 0.344 * uv.x - 0.714 * uv.y,\n" + " y + 1.770 * uv.x,\n" + " 1.0);\n" + " }\n"; + +@implementation RTCDefaultShader { + GLuint _vertexBuffer; + GLuint _vertexArray; + // Store current rotation and only upload new vertex data when rotation changes. + absl::optional<RTCVideoRotation> _currentRotation; + + GLuint _i420Program; + GLuint _nv12Program; +} + +- (void)dealloc { + glDeleteProgram(_i420Program); + glDeleteProgram(_nv12Program); + glDeleteBuffers(1, &_vertexBuffer); + glDeleteVertexArrays(1, &_vertexArray); +} + +- (BOOL)createAndSetupI420Program { + NSAssert(!_i420Program, @"I420 program already created"); + _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource); + if (!_i420Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_i420Program, "s_textureY"); + GLint uSampler = glGetUniformLocation(_i420Program, "s_textureU"); + GLint vSampler = glGetUniformLocation(_i420Program, "s_textureV"); + + if (ySampler < 0 || uSampler < 0 || vSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in I420 shader"); + glDeleteProgram(_i420Program); + _i420Program = 0; + return NO; + } + + glUseProgram(_i420Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uSampler, kUTextureUnit); + glUniform1i(vSampler, kVTextureUnit); + + return YES; +} + +- (BOOL)createAndSetupNV12Program { + NSAssert(!_nv12Program, @"NV12 program already created"); + _nv12Program = RTCCreateProgramFromFragmentSource(kNV12FragmentShaderSource); + if (!_nv12Program) { + return NO; + } + GLint ySampler = glGetUniformLocation(_nv12Program, "s_textureY"); + GLint uvSampler = glGetUniformLocation(_nv12Program, "s_textureUV"); + + if (ySampler < 0 || uvSampler < 0) { + RTCLog(@"Failed to get uniform variable locations in NV12 shader"); + glDeleteProgram(_nv12Program); + _nv12Program = 0; + return NO; + } + + glUseProgram(_nv12Program); + glUniform1i(ySampler, kYTextureUnit); + glUniform1i(uvSampler, kUvTextureUnit); + + return YES; +} + +- (BOOL)prepareVertexBufferWithRotation:(RTCVideoRotation)rotation { + if (!_vertexBuffer && !RTCCreateVertexBuffer(&_vertexBuffer, &_vertexArray)) { + RTCLog(@"Failed to setup vertex buffer"); + return NO; + } + + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + if (!_currentRotation || rotation != *_currentRotation) { + _currentRotation = absl::optional<RTCVideoRotation>(rotation); + RTCSetVertexData(*_currentRotation); + } + return YES; +} + +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uPlane:(GLuint)uPlane + vPlane:(GLuint)vPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_i420Program && ![self createAndSetupI420Program]) { + RTCLog(@"Failed to setup I420 program"); + return; + } + + glUseProgram(_i420Program); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kUTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kVTextureUnit)); + glBindTexture(GL_TEXTURE_2D, vPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane { + if (![self prepareVertexBufferWithRotation:rotation]) { + return; + } + + if (!_nv12Program && ![self createAndSetupNV12Program]) { + RTCLog(@"Failed to setup NV12 shader"); + return; + } + + glUseProgram(_nv12Program); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kYTextureUnit)); + glBindTexture(GL_TEXTURE_2D, yPlane); + + glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + kUvTextureUnit)); + glBindTexture(GL_TEXTURE_2D, uvPlane); + + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h new file mode 100644 index 0000000000..b78501e9e6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.h @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +// RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen +// refreshes, which should be 30fps. We wrap the display link in order to avoid +// a retain cycle since CADisplayLink takes a strong reference onto its target. +// The timer is paused by default. +@interface RTCDisplayLinkTimer : NSObject + +@property(nonatomic) BOOL isPaused; + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler; +- (void)invalidate; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m new file mode 100644 index 0000000000..906bb898d6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCDisplayLinkTimer.m @@ -0,0 +1,59 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDisplayLinkTimer.h" + +#import <UIKit/UIKit.h> + +@implementation RTCDisplayLinkTimer { + CADisplayLink *_displayLink; + void (^_timerHandler)(void); +} + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler { + NSParameterAssert(timerHandler); + if (self = [super init]) { + _timerHandler = timerHandler; + _displayLink = + [CADisplayLink displayLinkWithTarget:self + selector:@selector(displayLinkDidFire:)]; + _displayLink.paused = YES; +#if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_10_0 + _displayLink.preferredFramesPerSecond = 30; +#else + [_displayLink setFrameInterval:2]; +#endif + [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] + forMode:NSRunLoopCommonModes]; + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)isPaused { + return _displayLink.paused; +} + +- (void)setIsPaused:(BOOL)isPaused { + _displayLink.paused = isPaused; +} + +- (void)invalidate { + [_displayLink invalidate]; +} + +- (void)displayLinkDidFire:(CADisplayLink *)displayLink { + _timerHandler(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h new file mode 100644 index 0000000000..24b26cd602 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.h @@ -0,0 +1,45 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <UIKit/UIKit.h> + +#import "RTCMacros.h" +#import "RTCVideoRenderer.h" +#import "RTCVideoViewShading.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCEAGLVideoView); + +/** + * RTCEAGLVideoView is an RTCVideoRenderer which renders video frames + * in its bounds using OpenGLES 2.0 or OpenGLES 3.0. + */ +RTC_OBJC_EXPORT +NS_EXTENSION_UNAVAILABLE_IOS("Rendering not available in app extensions.") +@interface RTC_OBJC_TYPE (RTCEAGLVideoView) : UIView <RTC_OBJC_TYPE(RTCVideoRenderer)> + +@property(nonatomic, weak) id<RTC_OBJC_TYPE(RTCVideoViewDelegate)> delegate; + +- (instancetype)initWithFrame:(CGRect)frame + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader + NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithCoder:(NSCoder *)aDecoder + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader + NS_DESIGNATED_INITIALIZER; + +/** @abstract Wrapped RTCVideoRotation, or nil. + */ +@property(nonatomic, nullable) NSValue *rotationOverride; +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m new file mode 100644 index 0000000000..89e62d2ce7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCEAGLVideoView.m @@ -0,0 +1,295 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCEAGLVideoView.h" + +#import <GLKit/GLKit.h> + +#import "RTCDefaultShader.h" +#import "RTCDisplayLinkTimer.h" +#import "RTCI420TextureCache.h" +#import "RTCNV12TextureCache.h" +#import "base/RTCLogging.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +// RTC_OBJC_TYPE(RTCEAGLVideoView) wraps a GLKView which is setup with +// enableSetNeedsDisplay = NO for the purpose of gaining control of +// exactly when to call -[GLKView display]. This need for extra +// control is required to avoid triggering method calls on GLKView +// that results in attempting to bind the underlying render buffer +// when the drawable size would be empty which would result in the +// error GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT. -[GLKView display] is +// the method that will trigger the binding of the render +// buffer. Because the standard behaviour of -[UIView setNeedsDisplay] +// is disabled for the reasons above, the RTC_OBJC_TYPE(RTCEAGLVideoView) maintains +// its own `isDirty` flag. + +@interface RTC_OBJC_TYPE (RTCEAGLVideoView) +()<GLKViewDelegate> + // `videoFrame` is set when we receive a frame from a worker thread and is read + // from the display link callback so atomicity is required. + @property(atomic, strong) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; +@property(nonatomic, readonly) GLKView *glkView; +@end + +@implementation RTC_OBJC_TYPE (RTCEAGLVideoView) { + RTCDisplayLinkTimer *_timer; + EAGLContext *_glContext; + // This flag should only be set and read on the main thread (e.g. by + // setNeedsDisplay) + BOOL _isDirty; + id<RTC_OBJC_TYPE(RTCVideoViewShading)> _shader; + RTCNV12TextureCache *_nv12TextureCache; + RTCI420TextureCache *_i420TextureCache; + // As timestamps should be unique between frames, will store last + // drawn frame timestamp instead of the whole frame to reduce memory usage. + int64_t _lastDrawnFrameTimeStampNs; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize glkView = _glkView; +@synthesize rotationOverride = _rotationOverride; + +- (instancetype)initWithFrame:(CGRect)frame { + return [self initWithFrame:frame shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder { + return [self initWithCoder:aDecoder shader:[[RTCDefaultShader alloc] init]]; +} + +- (instancetype)initWithFrame:(CGRect)frame shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader { + if (self = [super initWithFrame:frame]) { + _shader = shader; + if (![self configure]) { + return nil; + } + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder + shader:(id<RTC_OBJC_TYPE(RTCVideoViewShading)>)shader { + if (self = [super initWithCoder:aDecoder]) { + _shader = shader; + if (![self configure]) { + return nil; + } + } + return self; +} + +- (BOOL)configure { + EAGLContext *glContext = + [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; + if (!glContext) { + glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; + } + if (!glContext) { + RTCLogError(@"Failed to create EAGLContext"); + return NO; + } + _glContext = glContext; + + // GLKView manages a framebuffer for us. + _glkView = [[GLKView alloc] initWithFrame:CGRectZero + context:_glContext]; + _glkView.drawableColorFormat = GLKViewDrawableColorFormatRGBA8888; + _glkView.drawableDepthFormat = GLKViewDrawableDepthFormatNone; + _glkView.drawableStencilFormat = GLKViewDrawableStencilFormatNone; + _glkView.drawableMultisample = GLKViewDrawableMultisampleNone; + _glkView.delegate = self; + _glkView.layer.masksToBounds = YES; + _glkView.enableSetNeedsDisplay = NO; + [self addSubview:_glkView]; + + // Listen to application state in order to clean up OpenGL before app goes + // away. + NSNotificationCenter *notificationCenter = + [NSNotificationCenter defaultCenter]; + [notificationCenter addObserver:self + selector:@selector(willResignActive) + name:UIApplicationWillResignActiveNotification + object:nil]; + [notificationCenter addObserver:self + selector:@selector(didBecomeActive) + name:UIApplicationDidBecomeActiveNotification + object:nil]; + + // Frames are received on a separate thread, so we poll for current frame + // using a refresh rate proportional to screen refresh frequency. This + // occurs on the main thread. + __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self; + _timer = [[RTCDisplayLinkTimer alloc] initWithTimerHandler:^{ + RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf; + [strongSelf displayLinkTimerDidFire]; + }]; + if ([[UIApplication sharedApplication] applicationState] == UIApplicationStateActive) { + [self setupGL]; + } + return YES; +} + +- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { + [super setMultipleTouchEnabled:multipleTouchEnabled]; + _glkView.multipleTouchEnabled = multipleTouchEnabled; +} + +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; + UIApplicationState appState = + [UIApplication sharedApplication].applicationState; + if (appState == UIApplicationStateActive) { + [self teardownGL]; + } + [_timer invalidate]; + [self ensureGLContext]; + _shader = nil; + if (_glContext && [EAGLContext currentContext] == _glContext) { + [EAGLContext setCurrentContext:nil]; + } +} + +#pragma mark - UIView + +- (void)setNeedsDisplay { + [super setNeedsDisplay]; + _isDirty = YES; +} + +- (void)setNeedsDisplayInRect:(CGRect)rect { + [super setNeedsDisplayInRect:rect]; + _isDirty = YES; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + _glkView.frame = self.bounds; +} + +#pragma mark - GLKViewDelegate + +// This method is called when the GLKView's content is dirty and needs to be +// redrawn. This occurs on main thread. +- (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { + // The renderer will draw the frame to the framebuffer corresponding to the + // one used by `view`. + RTC_OBJC_TYPE(RTCVideoFrame) *frame = self.videoFrame; + if (!frame || frame.timeStampNs == _lastDrawnFrameTimeStampNs) { + return; + } + RTCVideoRotation rotation = frame.rotation; + if(_rotationOverride != nil) { + [_rotationOverride getValue: &rotation]; + } + [self ensureGLContext]; + glClear(GL_COLOR_BUFFER_BIT); + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + if (!_nv12TextureCache) { + _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; + } + if (_nv12TextureCache) { + [_nv12TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:rotation + yPlane:_nv12TextureCache.yTexture + uvPlane:_nv12TextureCache.uvTexture]; + [_nv12TextureCache releaseTextures]; + + _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs; + } + } else { + if (!_i420TextureCache) { + _i420TextureCache = [[RTCI420TextureCache alloc] initWithContext:_glContext]; + } + [_i420TextureCache uploadFrameToTextures:frame]; + [_shader applyShadingForFrameWithWidth:frame.width + height:frame.height + rotation:rotation + yPlane:_i420TextureCache.yTexture + uPlane:_i420TextureCache.uTexture + vPlane:_i420TextureCache.vTexture]; + + _lastDrawnFrameTimeStampNs = self.videoFrame.timeStampNs; + } +} + +#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + __weak RTC_OBJC_TYPE(RTCEAGLVideoView) *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCEAGLVideoView) *strongSelf = weakSelf; + [strongSelf.delegate videoView:strongSelf didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)displayLinkTimerDidFire { + // Don't render unless video frame have changed or the view content + // has explicitly been marked dirty. + if (!_isDirty && _lastDrawnFrameTimeStampNs == self.videoFrame.timeStampNs) { + return; + } + + // Always reset isDirty at this point, even if -[GLKView display] + // won't be called in the case the drawable size is empty. + _isDirty = NO; + + // Only call -[GLKView display] if the drawable size is + // non-empty. Calling display will make the GLKView setup its + // render buffer if necessary, but that will fail with error + // GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT if size is empty. + if (self.bounds.size.width > 0 && self.bounds.size.height > 0) { + [_glkView display]; + } +} + +- (void)setupGL { + [self ensureGLContext]; + glDisable(GL_DITHER); + _timer.isPaused = NO; +} + +- (void)teardownGL { + self.videoFrame = nil; + _timer.isPaused = YES; + [_glkView deleteDrawable]; + [self ensureGLContext]; + _nv12TextureCache = nil; + _i420TextureCache = nil; +} + +- (void)didBecomeActive { + [self setupGL]; +} + +- (void)willResignActive { + [self teardownGL]; +} + +- (void)ensureGLContext { + NSAssert(_glContext, @"context shouldn't be nil"); + if ([EAGLContext currentContext] != _glContext) { + [EAGLContext setCurrentContext:_glContext]; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h new file mode 100644 index 0000000000..9fdcc5a695 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCOpenGLDefines.h" +#import "base/RTCVideoFrame.h" + +@interface RTCI420TextureCache : NSObject + +@property(nonatomic, readonly) GLuint yTexture; +@property(nonatomic, readonly) GLuint uTexture; +@property(nonatomic, readonly) GLuint vTexture; + +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWithContext:(GlContextType *)context NS_DESIGNATED_INITIALIZER; + +- (void)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm new file mode 100644 index 0000000000..a91e927cb4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCI420TextureCache.mm @@ -0,0 +1,149 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCI420TextureCache.h" + +#import <OpenGLES/ES3/gl.h> + +#import "base/RTCI420Buffer.h" +#import "base/RTCVideoFrameBuffer.h" + +#include <vector> + +// Two sets of 3 textures are used here, one for each of the Y, U and V planes. Having two sets +// alleviates CPU blockage in the event that the GPU is asked to render to a texture that is already +// in use. +static const GLsizei kNumTextureSets = 2; +static const GLsizei kNumTexturesPerSet = 3; +static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; + +@implementation RTCI420TextureCache { + BOOL _hasUnpackRowLength; + GLint _currentTextureSet; + // Handles for OpenGL constructs. + GLuint _textures[kNumTextures]; + // Used to create a non-padded plane for GPU upload when we receive padded frames. + std::vector<uint8_t> _planeBuffer; +} + +- (GLuint)yTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet]; +} + +- (GLuint)uTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 1]; +} + +- (GLuint)vTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 2]; +} + +- (instancetype)initWithContext:(GlContextType *)context { + if (self = [super init]) { + _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3); + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + [self setupTextures]; + } + return self; +} + +- (void)dealloc { + glDeleteTextures(kNumTextures, _textures); +} + +- (void)setupTextures { + glGenTextures(kNumTextures, _textures); + // Set parameters for each of the textures we created. + for (GLsizei i = 0; i < kNumTextures; i++) { + glBindTexture(GL_TEXTURE_2D, _textures[i]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } +} + +- (void)uploadPlane:(const uint8_t *)plane + texture:(GLuint)texture + width:(size_t)width + height:(size_t)height + stride:(int32_t)stride { + glBindTexture(GL_TEXTURE_2D, texture); + + const uint8_t *uploadPlane = plane; + if ((size_t)stride != width) { + if (_hasUnpackRowLength) { + // GLES3 allows us to specify stride. + glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast<GLsizei>(width), + static_cast<GLsizei>(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + return; + } else { + // Make an unpadded copy and upload that instead. Quick profiling showed + // that this is faster than uploading row by row using glTexSubImage2D. + uint8_t *unpaddedPlane = _planeBuffer.data(); + for (size_t y = 0; y < height; ++y) { + memcpy(unpaddedPlane + y * width, plane + y * stride, width); + } + uploadPlane = unpaddedPlane; + } + } + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast<GLsizei>(width), + static_cast<GLsizei>(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); +} + +- (void)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; + + id<RTC_OBJC_TYPE(RTCI420Buffer)> buffer = [frame.buffer toI420]; + + const int chromaWidth = buffer.chromaWidth; + const int chromaHeight = buffer.chromaHeight; + if (buffer.strideY != frame.width || buffer.strideU != chromaWidth || + buffer.strideV != chromaWidth) { + _planeBuffer.resize(buffer.width * buffer.height); + } + + [self uploadPlane:buffer.dataY + texture:self.yTexture + width:buffer.width + height:buffer.height + stride:buffer.strideY]; + + [self uploadPlane:buffer.dataU + texture:self.uTexture + width:chromaWidth + height:chromaHeight + stride:buffer.strideU]; + + [self uploadPlane:buffer.dataV + texture:self.vTexture + width:chromaWidth + height:chromaHeight + stride:buffer.strideV]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h new file mode 100644 index 0000000000..f202b836b5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.h @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <GLKit/GLKit.h> + +#import "base/RTCMacros.h" + +@class RTC_OBJC_TYPE(RTCVideoFrame); + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCNV12TextureCache : NSObject + +@property(nonatomic, readonly) GLuint yTexture; +@property(nonatomic, readonly) GLuint uvTexture; + +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithContext:(EAGLContext *)context NS_DESIGNATED_INITIALIZER; + +- (BOOL)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame; + +- (void)releaseTextures; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m new file mode 100644 index 0000000000..a520ac45b4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCNV12TextureCache.m @@ -0,0 +1,113 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNV12TextureCache.h" + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +@implementation RTCNV12TextureCache { + CVOpenGLESTextureCacheRef _textureCache; + CVOpenGLESTextureRef _yTextureRef; + CVOpenGLESTextureRef _uvTextureRef; +} + +- (GLuint)yTexture { + return CVOpenGLESTextureGetName(_yTextureRef); +} + +- (GLuint)uvTexture { + return CVOpenGLESTextureGetName(_uvTextureRef); +} + +- (instancetype)initWithContext:(EAGLContext *)context { + if (self = [super init]) { + CVReturn ret = CVOpenGLESTextureCacheCreate( + kCFAllocatorDefault, NULL, +#if COREVIDEO_USE_EAGLCONTEXT_CLASS_IN_API + context, +#else + (__bridge void *)context, +#endif + NULL, &_textureCache); + if (ret != kCVReturnSuccess) { + self = nil; + } + } + return self; +} + +- (BOOL)loadTexture:(CVOpenGLESTextureRef *)textureOut + pixelBuffer:(CVPixelBufferRef)pixelBuffer + planeIndex:(int)planeIndex + pixelFormat:(GLenum)pixelFormat { + const int width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex); + const int height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex); + + if (*textureOut) { + CFRelease(*textureOut); + *textureOut = nil; + } + CVReturn ret = CVOpenGLESTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, GL_TEXTURE_2D, pixelFormat, width, + height, pixelFormat, GL_UNSIGNED_BYTE, planeIndex, textureOut); + if (ret != kCVReturnSuccess) { + if (*textureOut) { + CFRelease(*textureOut); + *textureOut = nil; + } + return NO; + } + NSAssert(CVOpenGLESTextureGetTarget(*textureOut) == GL_TEXTURE_2D, + @"Unexpected GLES texture target"); + glBindTexture(GL_TEXTURE_2D, CVOpenGLESTextureGetName(*textureOut)); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + return YES; +} + +- (BOOL)uploadFrameToTextures:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + NSAssert([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]], + @"frame must be CVPixelBuffer backed"); + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + CVPixelBufferRef pixelBuffer = rtcPixelBuffer.pixelBuffer; + return [self loadTexture:&_yTextureRef + pixelBuffer:pixelBuffer + planeIndex:0 + pixelFormat:GL_LUMINANCE] && + [self loadTexture:&_uvTextureRef + pixelBuffer:pixelBuffer + planeIndex:1 + pixelFormat:GL_LUMINANCE_ALPHA]; +} + +- (void)releaseTextures { + if (_uvTextureRef) { + CFRelease(_uvTextureRef); + _uvTextureRef = nil; + } + if (_yTextureRef) { + CFRelease(_yTextureRef); + _yTextureRef = nil; + } +} + +- (void)dealloc { + [self releaseTextures]; + if (_textureCache) { + CFRelease(_textureCache); + _textureCache = nil; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h new file mode 100644 index 0000000000..d84d992278 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCOpenGLDefines.h @@ -0,0 +1,23 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#define RTC_PIXEL_FORMAT GL_LUMINANCE +#define SHADER_VERSION +#define VERTEX_SHADER_IN "attribute" +#define VERTEX_SHADER_OUT "varying" +#define FRAGMENT_SHADER_IN "varying" +#define FRAGMENT_SHADER_OUT +#define FRAGMENT_SHADER_COLOR "gl_FragColor" +#define FRAGMENT_SHADER_TEXTURE "texture2D" + +@class EAGLContext; +typedef EAGLContext GlContextType; diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h new file mode 100644 index 0000000000..d1b91fb643 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.h @@ -0,0 +1,21 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "base/RTCVideoFrame.h" + +RTC_EXTERN const char kRTCVertexShaderSource[]; + +RTC_EXTERN GLuint RTCCreateShader(GLenum type, const GLchar* source); +RTC_EXTERN GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader); +RTC_EXTERN GLuint +RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]); +RTC_EXTERN BOOL RTCCreateVertexBuffer(GLuint* vertexBuffer, + GLuint* vertexArray); +RTC_EXTERN void RTCSetVertexData(RTCVideoRotation rotation); diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm new file mode 100644 index 0000000000..25f6eee34e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCShader.mm @@ -0,0 +1,178 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCShader.h" + +#import <OpenGLES/ES3/gl.h> + +#include <algorithm> +#include <array> +#include <memory> + +#import "RTCOpenGLDefines.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +// Vertex shader doesn't do anything except pass coordinates through. +const char kRTCVertexShaderSource[] = + SHADER_VERSION + VERTEX_SHADER_IN " vec2 position;\n" + VERTEX_SHADER_IN " vec2 texcoord;\n" + VERTEX_SHADER_OUT " vec2 v_texcoord;\n" + "void main() {\n" + " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" + " v_texcoord = texcoord;\n" + "}\n"; + +// Compiles a shader of the given `type` with GLSL source `source` and returns +// the shader handle or 0 on error. +GLuint RTCCreateShader(GLenum type, const GLchar *source) { + GLuint shader = glCreateShader(type); + if (!shader) { + return 0; + } + glShaderSource(shader, 1, &source, NULL); + glCompileShader(shader); + GLint compileStatus = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); + if (compileStatus == GL_FALSE) { + GLint logLength = 0; + // The null termination character is included in the returned log length. + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &logLength); + if (logLength > 0) { + std::unique_ptr<char[]> compileLog(new char[logLength]); + // The returned string is null terminated. + glGetShaderInfoLog(shader, logLength, NULL, compileLog.get()); + RTC_LOG(LS_ERROR) << "Shader compile error: " << compileLog.get(); + } + glDeleteShader(shader); + shader = 0; + } + return shader; +} + +// Links a shader program with the given vertex and fragment shaders and +// returns the program handle or 0 on error. +GLuint RTCCreateProgram(GLuint vertexShader, GLuint fragmentShader) { + if (vertexShader == 0 || fragmentShader == 0) { + return 0; + } + GLuint program = glCreateProgram(); + if (!program) { + return 0; + } + glAttachShader(program, vertexShader); + glAttachShader(program, fragmentShader); + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (linkStatus == GL_FALSE) { + glDeleteProgram(program); + program = 0; + } + return program; +} + +// Creates and links a shader program with the given fragment shader source and +// a plain vertex shader. Returns the program handle or 0 on error. +GLuint RTCCreateProgramFromFragmentSource(const char fragmentShaderSource[]) { + GLuint vertexShader = RTCCreateShader(GL_VERTEX_SHADER, kRTCVertexShaderSource); + RTC_CHECK(vertexShader) << "failed to create vertex shader"; + GLuint fragmentShader = + RTCCreateShader(GL_FRAGMENT_SHADER, fragmentShaderSource); + RTC_CHECK(fragmentShader) << "failed to create fragment shader"; + GLuint program = RTCCreateProgram(vertexShader, fragmentShader); + // Shaders are created only to generate program. + if (vertexShader) { + glDeleteShader(vertexShader); + } + if (fragmentShader) { + glDeleteShader(fragmentShader); + } + + // Set vertex shader variables 'position' and 'texcoord' in program. + GLint position = glGetAttribLocation(program, "position"); + GLint texcoord = glGetAttribLocation(program, "texcoord"); + if (position < 0 || texcoord < 0) { + glDeleteProgram(program); + return 0; + } + + // Read position attribute with size of 2 and stride of 4 beginning at the start of the array. The + // last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); + glEnableVertexAttribArray(position); + + // Read texcoord attribute with size of 2 and stride of 4 beginning at the first texcoord in the + // array. The last argument indicates offset of data within the vertex buffer. + glVertexAttribPointer( + texcoord, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)(2 * sizeof(GLfloat))); + glEnableVertexAttribArray(texcoord); + + return program; +} + +BOOL RTCCreateVertexBuffer(GLuint *vertexBuffer, GLuint *vertexArray) { + glGenBuffers(1, vertexBuffer); + if (*vertexBuffer == 0) { + glDeleteVertexArrays(1, vertexArray); + return NO; + } + glBindBuffer(GL_ARRAY_BUFFER, *vertexBuffer); + glBufferData(GL_ARRAY_BUFFER, 4 * 4 * sizeof(GLfloat), NULL, GL_DYNAMIC_DRAW); + return YES; +} + +// Set vertex data to the currently bound vertex buffer. +void RTCSetVertexData(RTCVideoRotation rotation) { + // When modelview and projection matrices are identity (default) the world is + // contained in the square around origin with unit size 2. Drawing to these + // coordinates is equivalent to drawing to the entire screen. The texture is + // stretched over that square using texture coordinates (u, v) that range + // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically + // here because the incoming frame has origin in upper left hand corner but + // OpenGL expects origin in bottom left corner. + std::array<std::array<GLfloat, 2>, 4> UVCoords = {{ + {{0, 1}}, // Lower left. + {{1, 1}}, // Lower right. + {{1, 0}}, // Upper right. + {{0, 0}}, // Upper left. + }}; + + // Rotate the UV coordinates. + int rotation_offset; + switch (rotation) { + case RTCVideoRotation_0: + rotation_offset = 0; + break; + case RTCVideoRotation_90: + rotation_offset = 1; + break; + case RTCVideoRotation_180: + rotation_offset = 2; + break; + case RTCVideoRotation_270: + rotation_offset = 3; + break; + } + std::rotate(UVCoords.begin(), UVCoords.begin() + rotation_offset, + UVCoords.end()); + + const GLfloat gVertices[] = { + // X, Y, U, V. + -1, -1, UVCoords[0][0], UVCoords[0][1], + 1, -1, UVCoords[1][0], UVCoords[1][1], + 1, 1, UVCoords[2][0], UVCoords[2][1], + -1, 1, UVCoords[3][0], UVCoords[3][1], + }; + + glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(gVertices), gVertices); +} diff --git a/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h new file mode 100644 index 0000000000..9df30a8fa0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/renderer/opengl/RTCVideoViewShading.h @@ -0,0 +1,39 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCVideoFrame.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * RTCVideoViewShading provides a way for apps to customize the OpenGL(ES shaders + * used in rendering for the RTCEAGLVideoView/RTCNSGLVideoView. + */ +RTC_OBJC_EXPORT +@protocol RTC_OBJC_TYPE +(RTCVideoViewShading)<NSObject> + + /** Callback for I420 frames. Each plane is given as a texture. */ + - (void)applyShadingForFrameWithWidth : (int)width height : (int)height rotation + : (RTCVideoRotation)rotation yPlane : (GLuint)yPlane uPlane : (GLuint)uPlane vPlane + : (GLuint)vPlane; + +/** Callback for NV12 frames. Each plane is given as a texture. */ +- (void)applyShadingForFrameWithWidth:(int)width + height:(int)height + rotation:(RTCVideoRotation)rotation + yPlane:(GLuint)yPlane + uvPlane:(GLuint)uvPlane; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h new file mode 100644 index 0000000000..a0cd8515d1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCodecSpecificInfoH264.h" + +#include "modules/video_coding/include/video_codec_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +/* Interfaces for converting to/from internal C++ formats. */ +@interface RTC_OBJC_TYPE (RTCCodecSpecificInfoH264) +() + + - (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h new file mode 100644 index 0000000000..ae3003a115 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h @@ -0,0 +1,27 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCCodecSpecificInfo.h" +#import "RTCMacros.h" + +/** Class for H264 specific config. */ +typedef NS_ENUM(NSUInteger, RTCH264PacketizationMode) { + RTCH264PacketizationModeNonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + RTCH264PacketizationModeSingleNalUnit // Mode 0 - only single NALU allowed +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCodecSpecificInfoH264) : NSObject <RTC_OBJC_TYPE(RTCCodecSpecificInfo)> + +@property(nonatomic, assign) RTCH264PacketizationMode packetizationMode; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm new file mode 100644 index 0000000000..e38ed307b3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCodecSpecificInfoH264+Private.h" + +#import "RTCH264ProfileLevelId.h" + +// H264 specific settings. +@implementation RTC_OBJC_TYPE (RTCCodecSpecificInfoH264) + +@synthesize packetizationMode = _packetizationMode; + +- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo { + webrtc::CodecSpecificInfo codecSpecificInfo; + codecSpecificInfo.codecType = webrtc::kVideoCodecH264; + codecSpecificInfo.codecSpecific.H264.packetization_mode = + (webrtc::H264PacketizationMode)_packetizationMode; + + return codecSpecificInfo; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h new file mode 100644 index 0000000000..de5a9c4684 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoderFactory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** This decoder factory include support for all codecs bundled with WebRTC. If using custom + * codecs, create custom implementations of RTCVideoEncoderFactory and + * RTCVideoDecoderFactory. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoderFactory)> +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m new file mode 100644 index 0000000000..6e3baa8750 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m @@ -0,0 +1,85 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDefaultVideoDecoderFactory.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoDecoderH264.h" +#import "api/video_codec/RTCVideoCodecConstants.h" +#import "api/video_codec/RTCVideoDecoderVP8.h" +#import "api/video_codec/RTCVideoDecoderVP9.h" +#import "base/RTCVideoCodecInfo.h" + +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) +#import "api/video_codec/RTCVideoDecoderAV1.h" // nogncheck +#endif + +@implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory) + +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs { + NSDictionary<NSString *, NSString *> *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedHighInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedHighParams]; + + NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedBaselineInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedBaselineParams]; + + RTC_OBJC_TYPE(RTCVideoCodecInfo) *vp8Info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp8Name]; + + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *result = [@[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, + ] mutableCopy]; + + if ([RTC_OBJC_TYPE(RTCVideoDecoderVP9) isSupported]) { + [result + addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name]]; + } + +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) + [result addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name]]; +#endif + + return result; +} + +- (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)createDecoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + if ([info.name isEqualToString:kRTCVideoCodecH264Name]) { + return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] init]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) { + return [RTC_OBJC_TYPE(RTCVideoDecoderVP8) vp8Decoder]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp9Name] && + [RTC_OBJC_TYPE(RTCVideoDecoderVP9) isSupported]) { + return [RTC_OBJC_TYPE(RTCVideoDecoderVP9) vp9Decoder]; + } + +#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) + if ([info.name isEqualToString:kRTCVideoCodecAv1Name]) { + return [RTC_OBJC_TYPE(RTCVideoDecoderAV1) av1Decoder]; + } +#endif + + return nil; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h new file mode 100644 index 0000000000..92ab40c95b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h @@ -0,0 +1,31 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoderFactory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** This encoder factory include support for all codecs bundled with WebRTC. If using custom + * codecs, create custom implementations of RTCVideoEncoderFactory and + * RTCVideoDecoderFactory. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)> + +@property(nonatomic, retain) RTC_OBJC_TYPE(RTCVideoCodecInfo) *preferredCodec; + ++ (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m new file mode 100644 index 0000000000..8de55bde4a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m @@ -0,0 +1,102 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDefaultVideoEncoderFactory.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoEncoderH264.h" +#import "api/video_codec/RTCVideoCodecConstants.h" +#import "api/video_codec/RTCVideoEncoderVP8.h" +#import "api/video_codec/RTCVideoEncoderVP9.h" +#import "base/RTCVideoCodecInfo.h" + +#if defined(RTC_USE_LIBAOM_AV1_ENCODER) +#import "api/video_codec/RTCVideoEncoderAV1.h" // nogncheck +#endif + +@implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) + +@synthesize preferredCodec; + ++ (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs { + NSDictionary<NSString *, NSString *> *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedHighInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedHighParams]; + + NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedBaselineInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedBaselineParams]; + + RTC_OBJC_TYPE(RTCVideoCodecInfo) *vp8Info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp8Name]; + + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *result = [@[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, + ] mutableCopy]; + + if ([RTC_OBJC_TYPE(RTCVideoEncoderVP9) isSupported]) { + [result + addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name]]; + } + +#if defined(RTC_USE_LIBAOM_AV1_ENCODER) + [result addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name]]; +#endif + + return result; +} + +- (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + if ([info.name isEqualToString:kRTCVideoCodecH264Name]) { + return [[RTC_OBJC_TYPE(RTCVideoEncoderH264) alloc] initWithCodecInfo:info]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) { + return [RTC_OBJC_TYPE(RTCVideoEncoderVP8) vp8Encoder]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp9Name] && + [RTC_OBJC_TYPE(RTCVideoEncoderVP9) isSupported]) { + return [RTC_OBJC_TYPE(RTCVideoEncoderVP9) vp9Encoder]; + } + +#if defined(RTC_USE_LIBAOM_AV1_ENCODER) + if ([info.name isEqualToString:kRTCVideoCodecAv1Name]) { + return [RTC_OBJC_TYPE(RTCVideoEncoderAV1) av1Encoder]; + } +#endif + + return nil; +} + +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs { + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *codecs = + [[[self class] supportedCodecs] mutableCopy]; + + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *orderedCodecs = [NSMutableArray array]; + NSUInteger index = [codecs indexOfObject:self.preferredCodec]; + if (index != NSNotFound) { + [orderedCodecs addObject:[codecs objectAtIndex:index]]; + [codecs removeObjectAtIndex:index]; + } + [orderedCodecs addObjectsFromArray:codecs]; + + return [orderedCodecs copy]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.h new file mode 100644 index 0000000000..dac7bb5610 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.h @@ -0,0 +1,60 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +RTC_OBJC_EXPORT extern NSString *const kRTCVideoCodecH264Name; +RTC_OBJC_EXPORT extern NSString *const kRTCLevel31ConstrainedHigh; +RTC_OBJC_EXPORT extern NSString *const kRTCLevel31ConstrainedBaseline; +RTC_OBJC_EXPORT extern NSString *const kRTCMaxSupportedH264ProfileLevelConstrainedHigh; +RTC_OBJC_EXPORT extern NSString *const kRTCMaxSupportedH264ProfileLevelConstrainedBaseline; + +/** H264 Profiles and levels. */ +typedef NS_ENUM(NSUInteger, RTCH264Profile) { + RTCH264ProfileConstrainedBaseline, + RTCH264ProfileBaseline, + RTCH264ProfileMain, + RTCH264ProfileConstrainedHigh, + RTCH264ProfileHigh, +}; + +typedef NS_ENUM(NSUInteger, RTCH264Level) { + RTCH264Level1_b = 0, + RTCH264Level1 = 10, + RTCH264Level1_1 = 11, + RTCH264Level1_2 = 12, + RTCH264Level1_3 = 13, + RTCH264Level2 = 20, + RTCH264Level2_1 = 21, + RTCH264Level2_2 = 22, + RTCH264Level3 = 30, + RTCH264Level3_1 = 31, + RTCH264Level3_2 = 32, + RTCH264Level4 = 40, + RTCH264Level4_1 = 41, + RTCH264Level4_2 = 42, + RTCH264Level5 = 50, + RTCH264Level5_1 = 51, + RTCH264Level5_2 = 52 +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCH264ProfileLevelId) : NSObject + +@property(nonatomic, readonly) RTCH264Profile profile; +@property(nonatomic, readonly) RTCH264Level level; +@property(nonatomic, readonly) NSString *hexString; + +- (instancetype)initWithHexString:(NSString *)hexString; +- (instancetype)initWithProfile:(RTCH264Profile)profile level:(RTCH264Level)level; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.mm b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.mm new file mode 100644 index 0000000000..f0ef3ec232 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCH264ProfileLevelId.mm @@ -0,0 +1,120 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "RTCH264ProfileLevelId.h" + +#import "helpers/NSString+StdString.h" +#if defined(WEBRTC_IOS) +#import "UIDevice+H264Profile.h" +#endif + +#include "api/video_codecs/h264_profile_level_id.h" +#include "media/base/media_constants.h" + +namespace { + +NSString *MaxSupportedProfileLevelConstrainedHigh(); +NSString *MaxSupportedProfileLevelConstrainedBaseline(); + +} // namespace + +NSString *const kRTCVideoCodecH264Name = @(cricket::kH264CodecName); +NSString *const kRTCLevel31ConstrainedHigh = @"640c1f"; +NSString *const kRTCLevel31ConstrainedBaseline = @"42e01f"; +NSString *const kRTCMaxSupportedH264ProfileLevelConstrainedHigh = + MaxSupportedProfileLevelConstrainedHigh(); +NSString *const kRTCMaxSupportedH264ProfileLevelConstrainedBaseline = + MaxSupportedProfileLevelConstrainedBaseline(); + +namespace { + +#if defined(WEBRTC_IOS) + +NSString *MaxSupportedLevelForProfile(webrtc::H264Profile profile) { + const absl::optional<webrtc::H264ProfileLevelId> profileLevelId = + [UIDevice maxSupportedH264Profile]; + if (profileLevelId && profileLevelId->profile >= profile) { + const absl::optional<std::string> profileString = + H264ProfileLevelIdToString(webrtc::H264ProfileLevelId(profile, profileLevelId->level)); + if (profileString) { + return [NSString stringForStdString:*profileString]; + } + } + return nil; +} +#endif + +NSString *MaxSupportedProfileLevelConstrainedBaseline() { +#if defined(WEBRTC_IOS) + NSString *profile = MaxSupportedLevelForProfile(webrtc::H264Profile::kProfileConstrainedBaseline); + if (profile != nil) { + return profile; + } +#endif + return kRTCLevel31ConstrainedBaseline; +} + +NSString *MaxSupportedProfileLevelConstrainedHigh() { +#if defined(WEBRTC_IOS) + NSString *profile = MaxSupportedLevelForProfile(webrtc::H264Profile::kProfileConstrainedHigh); + if (profile != nil) { + return profile; + } +#endif + return kRTCLevel31ConstrainedHigh; +} + +} // namespace + +@interface RTC_OBJC_TYPE (RTCH264ProfileLevelId) +() + + @property(nonatomic, assign) RTCH264Profile profile; +@property(nonatomic, assign) RTCH264Level level; +@property(nonatomic, strong) NSString *hexString; + +@end + +@implementation RTC_OBJC_TYPE (RTCH264ProfileLevelId) + +@synthesize profile = _profile; +@synthesize level = _level; +@synthesize hexString = _hexString; + +- (instancetype)initWithHexString:(NSString *)hexString { + if (self = [super init]) { + self.hexString = hexString; + + absl::optional<webrtc::H264ProfileLevelId> profile_level_id = + webrtc::ParseH264ProfileLevelId([hexString cStringUsingEncoding:NSUTF8StringEncoding]); + if (profile_level_id.has_value()) { + self.profile = static_cast<RTCH264Profile>(profile_level_id->profile); + self.level = static_cast<RTCH264Level>(profile_level_id->level); + } + } + return self; +} + +- (instancetype)initWithProfile:(RTCH264Profile)profile level:(RTCH264Level)level { + if (self = [super init]) { + self.profile = profile; + self.level = level; + + absl::optional<std::string> hex_string = + webrtc::H264ProfileLevelIdToString(webrtc::H264ProfileLevelId( + static_cast<webrtc::H264Profile>(profile), static_cast<webrtc::H264Level>(level))); + self.hexString = + [NSString stringWithCString:hex_string.value_or("").c_str() encoding:NSUTF8StringEncoding]; + } + return self; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h new file mode 100644 index 0000000000..88bacbbdfe --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h @@ -0,0 +1,18 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoderFactory.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoDecoderFactoryH264) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoderFactory)> +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.m b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.m new file mode 100644 index 0000000000..bdae19d687 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.m @@ -0,0 +1,49 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoDecoderFactoryH264.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoDecoderH264.h" + +@implementation RTC_OBJC_TYPE (RTCVideoDecoderFactoryH264) + +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs { + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *codecs = [NSMutableArray array]; + NSString *codecName = kRTCVideoCodecH264Name; + + NSDictionary<NSString *, NSString *> *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedHighInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:codecName + parameters:constrainedHighParams]; + [codecs addObject:constrainedHighInfo]; + + NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedBaselineInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:codecName + parameters:constrainedBaselineParams]; + [codecs addObject:constrainedBaselineInfo]; + + return [codecs copy]; +} + +- (id<RTC_OBJC_TYPE(RTCVideoDecoder)>)createDecoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] init]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h new file mode 100644 index 0000000000..a12e4212a7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.h @@ -0,0 +1,18 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoDecoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoDecoderH264) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoder)> +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm new file mode 100644 index 0000000000..6708b26c89 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "RTCVideoDecoderH264.h" + +#import <VideoToolbox/VideoToolbox.h> + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "helpers.h" +#import "helpers/scoped_cftyperef.h" + +#if defined(WEBRTC_IOS) +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/components/video_codec/nalu_rewriter.h" + +// Struct that we pass to the decoder per frame to decode. We receive it again +// in the decoder callback. +struct RTCFrameDecodeParams { + RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {} + RTCVideoDecoderCallback callback; + int64_t timestamp; +}; + +@interface RTC_OBJC_TYPE (RTCVideoDecoderH264) +() - (void)setError : (OSStatus)error; +@end + +// This is the callback function that VideoToolbox calls when decode is +// complete. +void decompressionOutputCallback(void *decoderRef, + void *params, + OSStatus status, + VTDecodeInfoFlags infoFlags, + CVImageBufferRef imageBuffer, + CMTime timestamp, + CMTime duration) { + std::unique_ptr<RTCFrameDecodeParams> decodeParams( + reinterpret_cast<RTCFrameDecodeParams *>(params)); + if (status != noErr) { + RTC_OBJC_TYPE(RTCVideoDecoderH264) *decoder = + (__bridge RTC_OBJC_TYPE(RTCVideoDecoderH264) *)decoderRef; + [decoder setError:status]; + RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; + return; + } + // TODO(tkchin): Handle CVO properly. + RTC_OBJC_TYPE(RTCCVPixelBuffer) *frameBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:imageBuffer]; + RTC_OBJC_TYPE(RTCVideoFrame) *decodedFrame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] + initWithBuffer:frameBuffer + rotation:RTCVideoRotation_0 + timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec]; + decodedFrame.timeStamp = decodeParams->timestamp; + decodeParams->callback(decodedFrame); +} + +// Decoder. +@implementation RTC_OBJC_TYPE (RTCVideoDecoderH264) { + CMVideoFormatDescriptionRef _videoFormat; + CMMemoryPoolRef _memoryPool; + VTDecompressionSessionRef _decompressionSession; + RTCVideoDecoderCallback _callback; + OSStatus _error; +} + +- (instancetype)init { + self = [super init]; + if (self) { + _memoryPool = CMMemoryPoolCreate(nil); + } + return self; +} + +- (void)dealloc { + CMMemoryPoolInvalidate(_memoryPool); + CFRelease(_memoryPool); + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; +} + +- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores { + return WEBRTC_VIDEO_CODEC_OK; +} + +// TODO(bugs.webrtc.org/15444): Remove obsolete missingFrames param. +- (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage + missingFrames:(BOOL)missingFrames + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)info + renderTimeMs:(int64_t)renderTimeMs { + RTC_DCHECK(inputImage.buffer); + + if (_error != noErr) { + RTC_LOG(LS_WARNING) << "Last frame decode failed."; + _error = noErr; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + rtc::ScopedCFTypeRef<CMVideoFormatDescriptionRef> inputFormat = + rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length)); + if (inputFormat) { + // Check if the video format has changed, and reinitialize decoder if + // needed. + if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) { + [self setVideoFormat:inputFormat.get()]; + int resetDecompressionSessionError = [self resetDecompressionSession]; + if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) { + return resetDecompressionSessionError; + } + } + } + if (!_videoFormat) { + // We received a frame but we don't have format information so we can't + // decode it. + // This can happen after backgrounding. We need to wait for the next + // sps/pps before we can resume so we request a keyframe by returning an + // error. + RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + CMSampleBufferRef sampleBuffer = nullptr; + if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length, + _videoFormat, + &sampleBuffer, + _memoryPool)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(sampleBuffer); + VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; + std::unique_ptr<RTCFrameDecodeParams> frameDecodeParams; + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + OSStatus status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); +#if defined(WEBRTC_IOS) + // Re-initialize the decoder if we have an invalid session while the app is + // active or decoder malfunctions and retry the decode request. + if ((status == kVTInvalidSessionErr || status == kVTVideoDecoderMalfunctionErr) && + [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_INFO) << "Failed to decode frame with code: " << status + << " retrying decode after decompression session reset"; + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); + } +#endif + CFRelease(sampleBuffer); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoDecoderCallback)callback { + _callback = callback; +} + +- (void)setError:(OSStatus)error { + _error = error; +} + +- (NSInteger)releaseDecoder { + // Need to invalidate the session so that callbacks no longer occur and it + // is safe to null out the callback. + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +#pragma mark - Private + +- (int)resetDecompressionSession { + [self destroyDecompressionSession]; + + // Need to wait for the first SPS to initialize decoder. + if (!_videoFormat) { + return WEBRTC_VIDEO_CODEC_OK; + } + + // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder + // create pixel buffers with GPU backed memory. The intent here is to pass + // the pixel buffers directly so we avoid a texture upload later during + // rendering. This currently is moot because we are converting back to an + // I420 frame after decode, but eventually we will be able to plumb + // CVPixelBuffers directly to the renderer. + // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that + // we can pass CVPixelBuffers as native handles in decoder output. + NSDictionary *attributes = @{ +#if defined(WEBRTC_IOS) && (TARGET_OS_MACCATALYST || TARGET_OS_SIMULATOR) + (NSString *)kCVPixelBufferMetalCompatibilityKey : @(YES), +#elif defined(WEBRTC_IOS) + (NSString *)kCVPixelBufferOpenGLESCompatibilityKey : @(YES), +#elif defined(WEBRTC_MAC) && !defined(WEBRTC_ARCH_ARM64) + (NSString *)kCVPixelBufferOpenGLCompatibilityKey : @(YES), +#endif +#if !(TARGET_OS_SIMULATOR) + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, +#endif + (NSString *) + kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), + }; + + VTDecompressionOutputCallbackRecord record = { + decompressionOutputCallback, (__bridge void *)self, + }; + OSStatus status = VTDecompressionSessionCreate(nullptr, + _videoFormat, + nullptr, + (__bridge CFDictionaryRef)attributes, + &record, + &_decompressionSession); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to create decompression session: " << status; + [self destroyDecompressionSession]; + return WEBRTC_VIDEO_CODEC_ERROR; + } + [self configureDecompressionSession]; + + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureDecompressionSession { + RTC_DCHECK(_decompressionSession); +#if defined(WEBRTC_IOS) + VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); +#endif +} + +- (void)destroyDecompressionSession { + if (_decompressionSession) { +#if defined(WEBRTC_IOS) + VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession); +#endif + VTDecompressionSessionInvalidate(_decompressionSession); + CFRelease(_decompressionSession); + _decompressionSession = nullptr; + } +} + +- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat { + if (_videoFormat == videoFormat) { + return; + } + if (_videoFormat) { + CFRelease(_videoFormat); + } + _videoFormat = videoFormat; + if (_videoFormat) { + CFRetain(_videoFormat); + } +} + +- (NSString *)implementationName { + return @"VideoToolbox"; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h new file mode 100644 index 0000000000..45fc4be2ea --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h @@ -0,0 +1,18 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoEncoderFactory.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderFactoryH264) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)> +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.m b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.m new file mode 100644 index 0000000000..9843849307 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.m @@ -0,0 +1,49 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCVideoEncoderFactoryH264.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoEncoderH264.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactoryH264) + +- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs { + NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *codecs = [NSMutableArray array]; + NSString *codecName = kRTCVideoCodecH264Name; + + NSDictionary<NSString *, NSString *> *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedHighInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:codecName + parameters:constrainedHighParams]; + [codecs addObject:constrainedHighInfo]; + + NSDictionary<NSString *, NSString *> *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *constrainedBaselineInfo = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:codecName + parameters:constrainedBaselineParams]; + [codecs addObject:constrainedBaselineInfo]; + + return [codecs copy]; +} + +- (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + return [[RTC_OBJC_TYPE(RTCVideoEncoderH264) alloc] initWithCodecInfo:info]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.h b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.h new file mode 100644 index 0000000000..9f4f4c7c8d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.h @@ -0,0 +1,22 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" +#import "RTCVideoCodecInfo.h" +#import "RTCVideoEncoder.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderH264) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoder)> + +- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm new file mode 100644 index 0000000000..2160d79ae5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm @@ -0,0 +1,828 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "RTCVideoEncoderH264.h" + +#import <VideoToolbox/VideoToolbox.h> +#include <vector> + +#if defined(WEBRTC_IOS) +#import "helpers/UIDevice+RTCDevice.h" +#endif +#import "RTCCodecSpecificInfoH264.h" +#import "RTCH264ProfileLevelId.h" +#import "api/peerconnection/RTCVideoCodecInfo+Private.h" +#import "base/RTCCodecSpecificInfo.h" +#import "base/RTCI420Buffer.h" +#import "base/RTCVideoEncoder.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "helpers.h" + +#include "api/video_codecs/h264_profile_level_id.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "common_video/include/bitrate_adjuster.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/buffer.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/components/video_codec/nalu_rewriter.h" +#include "third_party/libyuv/include/libyuv/convert_from.h" + +@interface RTC_OBJC_TYPE (RTCVideoEncoderH264) +() + + - (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags sampleBuffer + : (CMSampleBufferRef)sampleBuffer codecSpecificInfo + : (id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo width : (int32_t)width height + : (int32_t)height renderTimeMs : (int64_t)renderTimeMs timestamp : (uint32_t)timestamp rotation + : (RTCVideoRotation)rotation; + +@end + +namespace { // anonymous namespace + +// The ratio between kVTCompressionPropertyKey_DataRateLimits and +// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher +// than the average bit rate to avoid undershooting the target. +const float kLimitToAverageBitRateFactor = 1.5f; +// These thresholds deviate from the default h264 QP thresholds, as they +// have been found to work better on devices that support VideoToolbox +const int kLowH264QpThreshold = 28; +const int kHighH264QpThreshold = 39; + +const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + +// Struct that we pass to the encoder per frame to encode. We receive it again +// in the encoder callback. +struct RTCFrameEncodeParams { + RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH264) * e, + RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) * csi, + int32_t w, + int32_t h, + int64_t rtms, + uint32_t ts, + RTCVideoRotation r) + : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) { + if (csi) { + codecSpecificInfo = csi; + } else { + codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) alloc] init]; + } + } + + RTC_OBJC_TYPE(RTCVideoEncoderH264) * encoder; + RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) * codecSpecificInfo; + int32_t width; + int32_t height; + int64_t render_time_ms; + uint32_t timestamp; + RTCVideoRotation rotation; +}; + +// We receive I420Frames as input, but we need to feed CVPixelBuffers into the +// encoder. This performs the copy and format conversion. +// TODO(tkchin): See if encoder will accept i420 frames and compare performance. +bool CopyVideoFrameToNV12PixelBuffer(id<RTC_OBJC_TYPE(RTCI420Buffer)> frameBuffer, + CVPixelBufferRef pixelBuffer) { + RTC_DCHECK(pixelBuffer); + RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), kNV12PixelFormat); + RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height); + RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width); + + CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + return false; + } + uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + // Convert I420 to NV12. + int ret = libyuv::I420ToNV12(frameBuffer.dataY, + frameBuffer.strideY, + frameBuffer.dataU, + frameBuffer.strideU, + frameBuffer.dataV, + frameBuffer.strideV, + dstY, + dstStrideY, + dstUV, + dstStrideUV, + frameBuffer.width, + frameBuffer.height); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + if (ret) { + RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; + return false; + } + return true; +} + +CVPixelBufferRef CreatePixelBuffer(VTCompressionSessionRef compression_session) { + if (!compression_session) { + RTC_LOG(LS_ERROR) << "Failed to get compression session."; + return nullptr; + } + CVPixelBufferPoolRef pixel_buffer_pool = + VTCompressionSessionGetPixelBufferPool(compression_session); + + if (!pixel_buffer_pool) { + RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool."; + return nullptr; + } + CVPixelBufferRef pixel_buffer; + CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer); + if (ret != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; + // We probably want to drop frames here, since failure probably means + // that the pool is empty. + return nullptr; + } + return pixel_buffer; +} + +// This is the callback function that VideoToolbox calls when encode is +// complete. From inspection this happens on its own queue. +void compressionOutputCallback(void *encoder, + void *params, + OSStatus status, + VTEncodeInfoFlags infoFlags, + CMSampleBufferRef sampleBuffer) { + if (!params) { + // If there are pending callbacks when the encoder is destroyed, this can happen. + return; + } + std::unique_ptr<RTCFrameEncodeParams> encodeParams( + reinterpret_cast<RTCFrameEncodeParams *>(params)); + [encodeParams->encoder frameWasEncoded:status + flags:infoFlags + sampleBuffer:sampleBuffer + codecSpecificInfo:encodeParams->codecSpecificInfo + width:encodeParams->width + height:encodeParams->height + renderTimeMs:encodeParams->render_time_ms + timestamp:encodeParams->timestamp + rotation:encodeParams->rotation]; +} + +// Extract VideoToolbox profile out of the webrtc::SdpVideoFormat. If there is +// no specific VideoToolbox profile for the specified level, AutoLevel will be +// returned. The user must initialize the encoder with a resolution and +// framerate conforming to the selected H264 level regardless. +CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id) { + switch (profile_level_id.profile) { + case webrtc::H264Profile::kProfileConstrainedBaseline: + case webrtc::H264Profile::kProfileBaseline: + switch (profile_level_id.level) { + case webrtc::H264Level::kLevel3: + return kVTProfileLevel_H264_Baseline_3_0; + case webrtc::H264Level::kLevel3_1: + return kVTProfileLevel_H264_Baseline_3_1; + case webrtc::H264Level::kLevel3_2: + return kVTProfileLevel_H264_Baseline_3_2; + case webrtc::H264Level::kLevel4: + return kVTProfileLevel_H264_Baseline_4_0; + case webrtc::H264Level::kLevel4_1: + return kVTProfileLevel_H264_Baseline_4_1; + case webrtc::H264Level::kLevel4_2: + return kVTProfileLevel_H264_Baseline_4_2; + case webrtc::H264Level::kLevel5: + return kVTProfileLevel_H264_Baseline_5_0; + case webrtc::H264Level::kLevel5_1: + return kVTProfileLevel_H264_Baseline_5_1; + case webrtc::H264Level::kLevel5_2: + return kVTProfileLevel_H264_Baseline_5_2; + case webrtc::H264Level::kLevel1: + case webrtc::H264Level::kLevel1_b: + case webrtc::H264Level::kLevel1_1: + case webrtc::H264Level::kLevel1_2: + case webrtc::H264Level::kLevel1_3: + case webrtc::H264Level::kLevel2: + case webrtc::H264Level::kLevel2_1: + case webrtc::H264Level::kLevel2_2: + return kVTProfileLevel_H264_Baseline_AutoLevel; + } + + case webrtc::H264Profile::kProfileMain: + switch (profile_level_id.level) { + case webrtc::H264Level::kLevel3: + return kVTProfileLevel_H264_Main_3_0; + case webrtc::H264Level::kLevel3_1: + return kVTProfileLevel_H264_Main_3_1; + case webrtc::H264Level::kLevel3_2: + return kVTProfileLevel_H264_Main_3_2; + case webrtc::H264Level::kLevel4: + return kVTProfileLevel_H264_Main_4_0; + case webrtc::H264Level::kLevel4_1: + return kVTProfileLevel_H264_Main_4_1; + case webrtc::H264Level::kLevel4_2: + return kVTProfileLevel_H264_Main_4_2; + case webrtc::H264Level::kLevel5: + return kVTProfileLevel_H264_Main_5_0; + case webrtc::H264Level::kLevel5_1: + return kVTProfileLevel_H264_Main_5_1; + case webrtc::H264Level::kLevel5_2: + return kVTProfileLevel_H264_Main_5_2; + case webrtc::H264Level::kLevel1: + case webrtc::H264Level::kLevel1_b: + case webrtc::H264Level::kLevel1_1: + case webrtc::H264Level::kLevel1_2: + case webrtc::H264Level::kLevel1_3: + case webrtc::H264Level::kLevel2: + case webrtc::H264Level::kLevel2_1: + case webrtc::H264Level::kLevel2_2: + return kVTProfileLevel_H264_Main_AutoLevel; + } + + case webrtc::H264Profile::kProfileConstrainedHigh: + case webrtc::H264Profile::kProfileHigh: + case webrtc::H264Profile::kProfilePredictiveHigh444: + switch (profile_level_id.level) { + case webrtc::H264Level::kLevel3: + return kVTProfileLevel_H264_High_3_0; + case webrtc::H264Level::kLevel3_1: + return kVTProfileLevel_H264_High_3_1; + case webrtc::H264Level::kLevel3_2: + return kVTProfileLevel_H264_High_3_2; + case webrtc::H264Level::kLevel4: + return kVTProfileLevel_H264_High_4_0; + case webrtc::H264Level::kLevel4_1: + return kVTProfileLevel_H264_High_4_1; + case webrtc::H264Level::kLevel4_2: + return kVTProfileLevel_H264_High_4_2; + case webrtc::H264Level::kLevel5: + return kVTProfileLevel_H264_High_5_0; + case webrtc::H264Level::kLevel5_1: + return kVTProfileLevel_H264_High_5_1; + case webrtc::H264Level::kLevel5_2: + return kVTProfileLevel_H264_High_5_2; + case webrtc::H264Level::kLevel1: + case webrtc::H264Level::kLevel1_b: + case webrtc::H264Level::kLevel1_1: + case webrtc::H264Level::kLevel1_2: + case webrtc::H264Level::kLevel1_3: + case webrtc::H264Level::kLevel2: + case webrtc::H264Level::kLevel2_1: + case webrtc::H264Level::kLevel2_2: + return kVTProfileLevel_H264_High_AutoLevel; + } + } +} + +// The function returns the max allowed sample rate (pixels per second) that +// can be processed by given encoder with `profile_level_id`. +// See https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.264-201610-S!!PDF-E&type=items +// for details. +NSUInteger GetMaxSampleRate(const webrtc::H264ProfileLevelId &profile_level_id) { + switch (profile_level_id.level) { + case webrtc::H264Level::kLevel3: + return 10368000; + case webrtc::H264Level::kLevel3_1: + return 27648000; + case webrtc::H264Level::kLevel3_2: + return 55296000; + case webrtc::H264Level::kLevel4: + case webrtc::H264Level::kLevel4_1: + return 62914560; + case webrtc::H264Level::kLevel4_2: + return 133693440; + case webrtc::H264Level::kLevel5: + return 150994944; + case webrtc::H264Level::kLevel5_1: + return 251658240; + case webrtc::H264Level::kLevel5_2: + return 530841600; + case webrtc::H264Level::kLevel1: + case webrtc::H264Level::kLevel1_b: + case webrtc::H264Level::kLevel1_1: + case webrtc::H264Level::kLevel1_2: + case webrtc::H264Level::kLevel1_3: + case webrtc::H264Level::kLevel2: + case webrtc::H264Level::kLevel2_1: + case webrtc::H264Level::kLevel2_2: + // Zero means auto rate setting. + return 0; + } +} +} // namespace + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { + RTC_OBJC_TYPE(RTCVideoCodecInfo) * _codecInfo; + std::unique_ptr<webrtc::BitrateAdjuster> _bitrateAdjuster; + uint32_t _targetBitrateBps; + uint32_t _encoderBitrateBps; + uint32_t _encoderFrameRate; + uint32_t _maxAllowedFrameRate; + RTCH264PacketizationMode _packetizationMode; + absl::optional<webrtc::H264ProfileLevelId> _profile_level_id; + RTCVideoEncoderCallback _callback; + int32_t _width; + int32_t _height; + VTCompressionSessionRef _compressionSession; + RTCVideoCodecMode _mode; + + webrtc::H264BitstreamParser _h264BitstreamParser; + std::vector<uint8_t> _frameScaleBuffer; +} + +// .5 is set as a mininum to prevent overcompensating for large temporary +// overshoots. We don't want to degrade video quality too badly. +// .95 is set to prevent oscillations. When a lower bitrate is set on the +// encoder than previously set, its output seems to have a brief period of +// drastically reduced bitrate, so we want to avoid that. In steady state +// conditions, 0.95 seems to give us better overall bitrate over long periods +// of time. +- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo { + if (self = [super init]) { + _codecInfo = codecInfo; + _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95)); + _packetizationMode = RTCH264PacketizationModeNonInterleaved; + _profile_level_id = + webrtc::ParseSdpForH264ProfileLevelId([codecInfo nativeSdpVideoFormat].parameters); + RTC_DCHECK(_profile_level_id); + RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile(*_profile_level_id)); + RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]); + } + return self; +} + +- (void)dealloc { + [self destroyCompressionSession]; +} + +- (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)settings + numberOfCores:(int)numberOfCores { + RTC_DCHECK(settings); + RTC_DCHECK([settings.name isEqualToString:kRTCVideoCodecH264Name]); + + _width = settings.width; + _height = settings.height; + _mode = settings.mode; + + uint32_t aligned_width = (((_width + 15) >> 4) << 4); + uint32_t aligned_height = (((_height + 15) >> 4) << 4); + _maxAllowedFrameRate = static_cast<uint32_t>(GetMaxSampleRate(*_profile_level_id) / + (aligned_width * aligned_height)); + + // We can only set average bitrate on the HW encoder. + _targetBitrateBps = settings.startBitrate * 1000; // startBitrate is in kbps. + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + _encoderFrameRate = MIN(settings.maxFramerate, _maxAllowedFrameRate); + if (settings.maxFramerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) { + RTC_LOG(LS_WARNING) << "Initial encoder frame rate setting " << settings.maxFramerate + << " is larger than the " + << "maximal allowed frame rate " << _maxAllowedFrameRate << "."; + } + + // TODO(tkchin): Try setting payload size via + // kVTCompressionPropertyKey_MaxH264SliceBytes. + + return [self resetCompressionSessionWithPixelFormat:kNV12PixelFormat]; +} + +- (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame + codecSpecificInfo:(nullable id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo + frameTypes:(NSArray<NSNumber *> *)frameTypes { + if (!_callback || !_compressionSession) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + BOOL isKeyframeRequired = NO; + + // Get a pixel buffer from the pool and copy frame data over. + if ([self resetCompressionSessionIfNeededWithFrame:frame]) { + isKeyframeRequired = YES; + } + + CVPixelBufferRef pixelBuffer = nullptr; + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + // Native frame buffer + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + if (![rtcPixelBuffer requiresCropping]) { + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can + // handle that and will output encoded frames in the configured + // resolution regardless of the input pixel buffer resolution. + pixelBuffer = rtcPixelBuffer.pixelBuffer; + CVBufferRetain(pixelBuffer); + } else { + // Cropping required, we need to crop and scale to a new pixel buffer. + pixelBuffer = CreatePixelBuffer(_compressionSession); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + int dstWidth = CVPixelBufferGetWidth(pixelBuffer); + int dstHeight = CVPixelBufferGetHeight(pixelBuffer); + if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { + int size = + [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight]; + _frameScaleBuffer.resize(size); + } else { + _frameScaleBuffer.clear(); + } + _frameScaleBuffer.shrink_to_fit(); + if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_frameScaleBuffer.data()]) { + CVBufferRelease(pixelBuffer); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + } + + if (!pixelBuffer) { + // We did not have a native frame buffer + RTC_DCHECK_EQ(frame.width, _width); + RTC_DCHECK_EQ(frame.height, _height); + + pixelBuffer = CreatePixelBuffer(_compressionSession); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(pixelBuffer); + if (!CopyVideoFrameToNV12PixelBuffer([frame.buffer toI420], pixelBuffer)) { + RTC_LOG(LS_ERROR) << "Failed to copy frame data."; + CVBufferRelease(pixelBuffer); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + // Check if we need a keyframe. + if (!isKeyframeRequired && frameTypes) { + for (NSNumber *frameType in frameTypes) { + if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) { + isKeyframeRequired = YES; + break; + } + } + } + + CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); + CFDictionaryRef frameProperties = nullptr; + if (isKeyframeRequired) { + CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; + CFTypeRef values[] = {kCFBooleanTrue}; + frameProperties = CreateCFTypeDictionary(keys, values, 1); + } + + std::unique_ptr<RTCFrameEncodeParams> encodeParams; + encodeParams.reset(new RTCFrameEncodeParams(self, + codecSpecificInfo, + _width, + _height, + frame.timeStampNs / rtc::kNumNanosecsPerMillisec, + frame.timeStamp, + frame.rotation)); + encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode; + + // Update the bitrate if needed. + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:_encoderFrameRate]; + + OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, + pixelBuffer, + presentationTimeStamp, + kCMTimeInvalid, + frameProperties, + encodeParams.release(), + nullptr); + if (frameProperties) { + CFRelease(frameProperties); + } + if (pixelBuffer) { + CVBufferRelease(pixelBuffer); + } + + if (status == kVTInvalidSessionErr) { + // This error occurs when entering foreground after backgrounding the app. + RTC_LOG(LS_ERROR) << "Invalid compression session, resetting."; + [self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]]; + + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } else if (status == kVTVideoEncoderMalfunctionErr) { + // Sometimes the encoder malfunctions and needs to be restarted. + RTC_LOG(LS_ERROR) + << "Encountered video encoder malfunction error. Resetting compression session."; + [self resetCompressionSessionWithPixelFormat:[self pixelFormatOfFrame:frame]]; + + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } else if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoEncoderCallback)callback { + _callback = callback; +} + +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { + _targetBitrateBps = 1000 * bitrateKbit; + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + if (framerate > _maxAllowedFrameRate && _maxAllowedFrameRate > 0) { + RTC_LOG(LS_WARNING) << "Encoder frame rate setting " << framerate << " is larger than the " + << "maximal allowed frame rate " << _maxAllowedFrameRate << "."; + } + framerate = MIN(framerate, _maxAllowedFrameRate); + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:framerate]; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (NSInteger)resolutionAlignment { + return 1; +} + +- (BOOL)applyAlignmentToAllSimulcastLayers { + return NO; +} + +- (BOOL)supportsNativeHandle { + return YES; +} + +#pragma mark - Private + +- (NSInteger)releaseEncoder { + // Need to destroy so that the session is invalidated and won't use the + // callback anymore. Do not remove callback until the session is invalidated + // since async encoder callbacks can occur until invalidation. + [self destroyCompressionSession]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (OSType)pixelFormatOfFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + // Use NV12 for non-native frames. + if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + return CVPixelBufferGetPixelFormatType(rtcPixelBuffer.pixelBuffer); + } + + return kNV12PixelFormat; +} + +- (BOOL)resetCompressionSessionIfNeededWithFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + BOOL resetCompressionSession = NO; + + // If we're capturing native frames in another pixel format than the compression session is + // configured with, make sure the compression session is reset using the correct pixel format. + OSType framePixelFormat = [self pixelFormatOfFrame:frame]; + + if (_compressionSession) { + // The pool attribute `kCVPixelBufferPixelFormatTypeKey` can contain either an array of pixel + // formats or a single pixel format. + + CVPixelBufferPoolRef pixelBufferPool = + VTCompressionSessionGetPixelBufferPool(_compressionSession); + if (!pixelBufferPool) { + return NO; + } + + NSDictionary *poolAttributes = + (__bridge NSDictionary *)CVPixelBufferPoolGetPixelBufferAttributes(pixelBufferPool); + id pixelFormats = + [poolAttributes objectForKey:(__bridge NSString *)kCVPixelBufferPixelFormatTypeKey]; + NSArray<NSNumber *> *compressionSessionPixelFormats = nil; + if ([pixelFormats isKindOfClass:[NSArray class]]) { + compressionSessionPixelFormats = (NSArray *)pixelFormats; + } else if ([pixelFormats isKindOfClass:[NSNumber class]]) { + compressionSessionPixelFormats = @[ (NSNumber *)pixelFormats ]; + } + + if (![compressionSessionPixelFormats + containsObject:[NSNumber numberWithLong:framePixelFormat]]) { + resetCompressionSession = YES; + RTC_LOG(LS_INFO) << "Resetting compression session due to non-matching pixel format."; + } + } else { + resetCompressionSession = YES; + } + + if (resetCompressionSession) { + [self resetCompressionSessionWithPixelFormat:framePixelFormat]; + } + return resetCompressionSession; +} + +- (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { + [self destroyCompressionSession]; + + // Set source image buffer attributes. These attributes will be present on + // buffers retrieved from the encoder's pixel buffer pool. + NSDictionary *sourceAttributes = @{ +#if defined(WEBRTC_IOS) && (TARGET_OS_MACCATALYST || TARGET_OS_SIMULATOR) + (NSString *)kCVPixelBufferMetalCompatibilityKey : @(YES), +#elif defined(WEBRTC_IOS) + (NSString *)kCVPixelBufferOpenGLESCompatibilityKey : @(YES), +#elif defined(WEBRTC_MAC) && !defined(WEBRTC_ARCH_ARM64) + (NSString *)kCVPixelBufferOpenGLCompatibilityKey : @(YES), +#endif + (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}, + (NSString *)kCVPixelBufferPixelFormatTypeKey : @(framePixelFormat), + }; + + NSDictionary *encoder_specs; +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + // Currently hw accl is supported above 360p on mac, below 360p + // the compression session will be created with hw accl disabled. + encoder_specs = @{ + (NSString *)kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder : @(YES), + }; + +#endif + OSStatus status = VTCompressionSessionCreate( + nullptr, // use default allocator + _width, + _height, + kCMVideoCodecType_H264, + (__bridge CFDictionaryRef)encoder_specs, // use hardware accelerated encoder if available + (__bridge CFDictionaryRef)sourceAttributes, + nullptr, // use default compressed data allocator + compressionOutputCallback, + nullptr, + &_compressionSession); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + CFBooleanRef hwaccl_enabled = nullptr; + status = VTSessionCopyProperty(_compressionSession, + kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, + nullptr, + &hwaccl_enabled); + if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) { + RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled"; + } else { + RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled"; + } +#endif + [self configureCompressionSession]; + + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureCompressionSession { + RTC_DCHECK(_compressionSession); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true); + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_ProfileLevel, + ExtractProfile(*_profile_level_id)); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false); + [self setEncoderBitrateBps:_targetBitrateBps frameRate:_encoderFrameRate]; + // TODO(tkchin): Look at entropy mode and colorspace matrices. + // TODO(tkchin): Investigate to see if there's any way to make this work. + // May need it to interop with Android. Currently this call just fails. + // On inspecting encoder output on iOS8, this value is set to 6. + // internal::SetVTSessionProperty(compression_session_, + // kVTCompressionPropertyKey_MaxFrameDelayCount, + // 1); + + // Set a relatively large value for keyframe emission (7200 frames or 4 minutes). + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); + SetVTSessionProperty( + _compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); +} + +- (void)destroyCompressionSession { + if (_compressionSession) { + VTCompressionSessionInvalidate(_compressionSession); + CFRelease(_compressionSession); + _compressionSession = nullptr; + } +} + +- (NSString *)implementationName { + return @"VideoToolbox"; +} + +- (void)setBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate { + if (_encoderBitrateBps != bitrateBps || _encoderFrameRate != frameRate) { + [self setEncoderBitrateBps:bitrateBps frameRate:frameRate]; + } +} + +- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate { + if (_compressionSession) { + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps); + + // With zero `_maxAllowedFrameRate`, we fall back to automatic frame rate detection. + if (_maxAllowedFrameRate > 0) { + SetVTSessionProperty( + _compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, frameRate); + } + + // TODO(tkchin): Add a helper method to set array value. + int64_t dataLimitBytesPerSecondValue = + static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8); + CFNumberRef bytesPerSecond = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue); + int64_t oneSecondValue = 1; + CFNumberRef oneSecond = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); + const void *nums[2] = {bytesPerSecond, oneSecond}; + CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); + OSStatus status = VTSessionSetProperty( + _compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits); + if (bytesPerSecond) { + CFRelease(bytesPerSecond); + } + if (oneSecond) { + CFRelease(oneSecond); + } + if (dataRateLimits) { + CFRelease(dataRateLimits); + } + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to set data rate limit with code: " << status; + } + + _encoderBitrateBps = bitrateBps; + _encoderFrameRate = frameRate; + } +} + +- (void)frameWasEncoded:(OSStatus)status + flags:(VTEncodeInfoFlags)infoFlags + sampleBuffer:(CMSampleBufferRef)sampleBuffer + codecSpecificInfo:(id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo + width:(int32_t)width + height:(int32_t)height + renderTimeMs:(int64_t)renderTimeMs + timestamp:(uint32_t)timestamp + rotation:(RTCVideoRotation)rotation { + RTCVideoEncoderCallback callback = _callback; + if (!callback) { + return; + } + if (status != noErr) { + RTC_LOG(LS_ERROR) << "H264 encode failed with code: " << status; + return; + } + if (infoFlags & kVTEncodeInfo_FrameDropped) { + RTC_LOG(LS_INFO) << "H264 encode dropped frame."; + return; + } + + BOOL isKeyframe = NO; + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); + if (attachments != nullptr && CFArrayGetCount(attachments)) { + CFDictionaryRef attachment = + static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); + isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); + } + + if (isKeyframe) { + RTC_LOG(LS_INFO) << "Generated keyframe"; + } + + __block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>(); + if (!webrtc::H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) { + return; + } + + RTC_OBJC_TYPE(RTCEncodedImage) *frame = [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] init]; + // This assumes ownership of `buffer` and is responsible for freeing it when done. + frame.buffer = [[NSData alloc] initWithBytesNoCopy:buffer->data() + length:buffer->size() + deallocator:^(void *bytes, NSUInteger size) { + buffer.reset(); + }]; + frame.encodedWidth = width; + frame.encodedHeight = height; + frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta; + frame.captureTimeMs = renderTimeMs; + frame.timeStamp = timestamp; + frame.rotation = rotation; + frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare : + RTCVideoContentTypeUnspecified; + frame.flags = webrtc::VideoSendTiming::kInvalid; + + _h264BitstreamParser.ParseBitstream(*buffer); + frame.qp = @(_h264BitstreamParser.GetLastSliceQp().value_or(-1)); + + BOOL res = callback(frame, codecSpecificInfo); + if (!res) { + RTC_LOG(LS_ERROR) << "Encode callback failed"; + return; + } + _bitrateAdjuster->Update(frame.buffer.length); +} + +- (nullable RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *)scalingSettings { + return [[RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) alloc] + initWithThresholdsLow:kLowH264QpThreshold + high:kHighH264QpThreshold]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.h b/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.h new file mode 100644 index 0000000000..a51debb9fa --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.h @@ -0,0 +1,19 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <UIKit/UIKit.h> + +#include "api/video_codecs/h264_profile_level_id.h" + +@interface UIDevice (H264Profile) + ++ (absl::optional<webrtc::H264ProfileLevelId>)maxSupportedH264Profile; + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.mm b/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.mm new file mode 100644 index 0000000000..cfb2b97605 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/UIDevice+H264Profile.mm @@ -0,0 +1,293 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "UIDevice+H264Profile.h" +#import "helpers/UIDevice+RTCDevice.h" + +#include <algorithm> + +namespace { + +using namespace webrtc; + +struct SupportedH264Profile { + const char* machineName; + const H264ProfileLevelId profile; +}; + +constexpr SupportedH264Profile kH264MaxSupportedProfiles[] = { + // iPhones with at least iOS 12 + {"iPhone15,3", /* https://support.apple.com/kb/SP876 iPhone 14 Pro Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone15,2", /* https://support.apple.com/kb/SP875 iPhone 14 Pro */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone14,8", /* https://support.apple.com/kb/SP874 iPhone 14 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,7", /* https://support.apple.com/kb/SP873 iPhone 14 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,6", /* https://support.apple.com/kb/SP867 iPhone SE 3rd Gen */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,5", /* https://support.apple.com/kb/SP851 iPhone 13 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,4", /* https://support.apple.com/kb/SP847 Phone 13 Mini */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,3", /* https://support.apple.com/kb/SP848 iPhone 13 Pro Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone14,2", /* https://support.apple.com/kb/SP852 iPhone 13 Pro */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone13,4", /* https://support.apple.com/kb/SP832 iPhone 12 Pro Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone13,3", /* https://support.apple.com/kb/SP831 iPhone 12 Pro */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone13,2", /* https://support.apple.com/kb/SP830 iPhone 12 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone13,1", /* https://support.apple.com/kb/SP829 iPhone 12 mini */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone12,8", /* https://support.apple.com/kb/SP820 iPhone SE (2nd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone12,5", /* https://support.apple.com/kb/SP806 iPhone 11 Pro Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone12,3", /* https://support.apple.com/kb/SP805 iPhone 11 pro */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone12,1", /* https://support.apple.com/kb/SP804 iPhone 11 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone11,8", /* https://support.apple.com/kb/SP781 iPhone XR */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone11,6", /* https://support.apple.com/kb/SP780 iPhone XS Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone11,4", /* https://support.apple.com/kb/SP780 iPhone XS Max */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone11,2", /* https://support.apple.com/kb/SP779 iPhone XS */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone10,6", /* https://support.apple.com/kb/SP770 iPhone X */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone10,5", /* https://support.apple.com/kb/SP768 iPhone 8 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone10,4", /* https://support.apple.com/kb/SP767 iPhone 8 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone10,3", /* https://support.apple.com/kb/SP770 iPhone X */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone10,2", /* https://support.apple.com/kb/SP768 iPhone 8 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPhone10,1", /* https://support.apple.com/kb/SP767 iPhone 8 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPhone9,4", /* https://support.apple.com/kb/SP744 iPhone 7 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPhone9,3", /* https://support.apple.com/kb/SP743 iPhone 7 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPhone9,2", /* https://support.apple.com/kb/SP744 iPhone 7 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPhone9,1", /* https://support.apple.com/kb/SP743 iPhone 7 */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + + {"iPhone8,4", /* https://support.apple.com/kb/SP738 iPhone SE */ + {H264Profile::kProfileHigh, H264Level::kLevel4_2}}, + {"iPhone8,2", /* https://support.apple.com/kb/SP727 iPhone 6s Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel4_2}}, + {"iPhone8,1", /* https://support.apple.com/kb/SP726 iPhone 6s */ + {H264Profile::kProfileHigh, H264Level::kLevel4_2}}, + + {"iPhone7,2", /* https://support.apple.com/kb/SP705 iPhone 6 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_2}}, + {"iPhone7,1", /* https://support.apple.com/kb/SP706 iPhone 6 Plus */ + {H264Profile::kProfileHigh, H264Level::kLevel4_2}}, + + {"iPhone6,2", /* https://support.apple.com/kb/SP685 iPhone 5s */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPhone6,1", /* https://support.apple.com/kb/SP685 iPhone 5s */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + + // iPods with at least iOS 12 + {"iPod9,1", /* https://support.apple.com/kb/SP796 iPod touch (7th generation) */ + {H264Profile::kProfileMain, H264Level::kLevel4_1}}, + {"iPod7,1", /* https://support.apple.com/kb/SP720 iPod touch (6th generation) */ + {H264Profile::kProfileMain, H264Level::kLevel4_1}}, + + // iPads with at least iOS 12 + {"iPad14,6", /* https://support.apple.com/kb/SP883 iPad Pro 12.9-inch (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad14,5", /* https://support.apple.com/kb/SP883 iPad Pro 12.9-inch (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad14,4", /* https://support.apple.com/kb/SP882 iPad Pro 11-inch (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad14,3", /* https://support.apple.com/kb/SP882 iPad Pro 11-inch (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad14,2", /* https://support.apple.com/kb/SP850 iPad mini (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad14,1", /* https://support.apple.com/kb/SP850 iPad mini (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPad13,19", /* https://support.apple.com/kb/SP884 iPad (10th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,18", /* https://support.apple.com/kb/SP884 iPad (10th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,17", /* https://support.apple.com/kb/SP866 iPad Air (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,16", /* https://support.apple.com/kb/SP866 iPad Air (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,11", /* https://support.apple.com/kb/SP844 iPad Pro, 12.9-inch (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,10", /* https://support.apple.com/kb/SP844 iPad Pro, 12.9-inch (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,9", /* https://support.apple.com/kb/SP844 iPad Pro, 12.9-inch (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,8", /* https://support.apple.com/kb/SP844 iPad Pro, 12.9-inch (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,7", /* https://support.apple.com/kb/SP843 iPad Pro, 11-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,6", /* https://support.apple.com/kb/SP843 iPad Pro, 11-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,5", /* https://support.apple.com/kb/SP843 iPad Pro, 11-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,4", /* https://support.apple.com/kb/SP843 iPad Pro, 11-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,2", /* https://support.apple.com/kb/SP828 iPad Air (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad13,1", /* https://support.apple.com/kb/SP828 iPad Air (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPad12,2", /* https://support.apple.com/kb/SP849 iPad (9th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad12,1", /* https://support.apple.com/kb/SP849 iPad (9th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + + {"iPad11,7", /* https://support.apple.com/kb/SP822 iPad (8th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad11,6", /* https://support.apple.com/kb/SP822 iPad (8th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad11,4", /* https://support.apple.com/kb/SP787 iPad Air (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad11,3", /* https://support.apple.com/kb/SP787 iPad Air (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad11,2", /* https://support.apple.com/kb/SP788 iPad mini (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad11,1", /* https://support.apple.com/kb/SP788 iPad mini (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + + {"iPad8,12", /* https://support.apple.com/kb/SP815 iPad Pro 12.9-inch (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,11", /* https://support.apple.com/kb/SP815 iPad Pro 12.9-inch (4th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,10", /* https://support.apple.com/kb/SP814 iPad Pro 11-inch (2nd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,9", /* https://support.apple.com/kb/SP814 iPad Pro 11-inch (2nd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,8", /* https://support.apple.com/kb/SP785 iPad Pro 12.9-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,7", /* https://support.apple.com/kb/SP785 iPad Pro 12.9-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,6", /* https://support.apple.com/kb/SP785 iPad Pro 12.9-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,5", /* https://support.apple.com/kb/SP785 iPad Pro 12.9-inch (3rd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,4", /* https://support.apple.com/kb/SP784 iPad Pro 11-inch (1st generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,3", /* https://support.apple.com/kb/SP784 iPad Pro 11-inch (1st generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,2", /* https://support.apple.com/kb/SP784 iPad Pro 11-inch (1st generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + {"iPad8,1", /* https://support.apple.com/kb/SP784 iPad Pro 11-inch (1st generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_2}}, + + {"iPad7,12", /* https://support.apple.com/kb/SP807 iPad (7th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad7,11", /* https://support.apple.com/kb/SP807 iPad (7th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad7,6", /* https://support.apple.com/kb/SP774 iPad (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad7,5", /* https://support.apple.com/kb/SP774 iPad (6th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad7,4", /* https://support.apple.com/kb/SP762 iPad Pro (10.5-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPad7,3", /* https://support.apple.com/kb/SP762 iPad Pro (10.5-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPad7,2", /* https://support.apple.com/kb/SP761 iPad Pro (12.9-inch) (2nd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPad7,1", /* https://support.apple.com/kb/SP761 iPad Pro (12.9-inch) (2nd generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + + {"iPad6,12", /* https://support.apple.com/kb/SP751 iPad (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad6,11", /* https://support.apple.com/kb/SP751 iPad (5th generation) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad6,8", /* https://support.apple.com/kb/SP723 iPad Pro (12.9-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad6,7", /* https://support.apple.com/kb/SP723 iPad Pro (12.9-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad6,4", /* https://support.apple.com/kb/SP739 iPad Pro (9.7-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + {"iPad6,3", /* https://support.apple.com/kb/SP739 iPad Pro (9.7-inch) */ + {H264Profile::kProfileHigh, H264Level::kLevel5_1}}, + + {"iPad5,4", /* https://support.apple.com/kb/sp708 iPad Air 2 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad5,3", /* https://support.apple.com/kb/sp708 iPad Air 2 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad5,2", /* https://support.apple.com/kb/sp725 iPad mini 4 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad5,1", /* https://support.apple.com/kb/sp725 iPad mini 4 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + + {"iPad4,9", /* https://support.apple.com/kb/sp709 iPad mini 3 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,8", /* https://support.apple.com/kb/sp709 iPad mini 3 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,7", /* https://support.apple.com/kb/sp709 iPad mini 3 */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,6", /* https://support.apple.com/kb/sp693 iPad mini 2 with Retina display */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,5", /* https://support.apple.com/kb/sp693 iPad mini 2 with Retina display */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,4", /* https://support.apple.com/kb/sp693 iPad mini 2 with Retina display */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,3", /* https://support.apple.com/kb/SP692 iPad Air */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,2", /* https://support.apple.com/kb/SP692 iPad Air */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, + {"iPad4,1", /* https://support.apple.com/kb/SP692 iPad Air */ + {H264Profile::kProfileHigh, H264Level::kLevel4_1}}, +}; + +absl::optional<H264ProfileLevelId> FindMaxSupportedProfileForDevice(NSString* machineName) { + const auto* result = + std::find_if(std::begin(kH264MaxSupportedProfiles), + std::end(kH264MaxSupportedProfiles), + [machineName](const SupportedH264Profile& supportedProfile) { + return [machineName isEqualToString:@(supportedProfile.machineName)]; + }); + if (result != std::end(kH264MaxSupportedProfiles)) { + return result->profile; + } + if ([machineName hasPrefix:@"iPhone"] || [machineName hasPrefix:@"iPad"]) { + H264ProfileLevelId fallbackProfile{H264Profile::kProfileHigh, H264Level::kLevel4_1}; + return fallbackProfile; + } + if ([machineName hasPrefix:@"iPod"]) { + H264ProfileLevelId fallbackProfile{H264Profile::kProfileMain, H264Level::kLevel4_1}; + return fallbackProfile; + } + return absl::nullopt; +} + +} // namespace + +@implementation UIDevice (H264Profile) + ++ (absl::optional<webrtc::H264ProfileLevelId>)maxSupportedH264Profile { + return FindMaxSupportedProfileForDevice([UIDevice machineName]); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.cc b/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.cc new file mode 100644 index 0000000000..ac957f1b49 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "helpers.h" + +#include <string> + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +// Copies characters from a CFStringRef into a std::string. +std::string CFStringToString(const CFStringRef cf_string) { + RTC_DCHECK(cf_string); + std::string std_string; + // Get the size needed for UTF8 plus terminating character. + size_t buffer_size = + CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), + kCFStringEncodingUTF8) + + 1; + std::unique_ptr<char[]> buffer(new char[buffer_size]); + if (CFStringGetCString(cf_string, buffer.get(), buffer_size, + kCFStringEncodingUTF8)) { + // Copy over the characters. + std_string.assign(buffer.get()); + } + return std_string; +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + int32_t value) { + CFNumberRef cfNum = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); + OSStatus status = VTSessionSetProperty(session, key, cfNum); + CFRelease(cfNum); + if (status != noErr) { + std::string key_string = CFStringToString(key); + RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + uint32_t value) { + int64_t value_64 = value; + CFNumberRef cfNum = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); + OSStatus status = VTSessionSetProperty(session, key, cfNum); + CFRelease(cfNum); + if (status != noErr) { + std::string key_string = CFStringToString(key); + RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { + CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; + OSStatus status = VTSessionSetProperty(session, key, cf_bool); + if (status != noErr) { + std::string key_string = CFStringToString(key); + RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + CFStringRef value) { + OSStatus status = VTSessionSetProperty(session, key, value); + if (status != noErr) { + std::string key_string = CFStringToString(key); + std::string val_string = CFStringToString(value); + RTC_LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << val_string << ": " << status; + } +} diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.h b/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.h new file mode 100644 index 0000000000..9dc05c8d59 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/helpers.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ +#define SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ + +#include <CoreFoundation/CoreFoundation.h> +#include <VideoToolbox/VideoToolbox.h> + +#include <string> + +// Convenience function for creating a dictionary. +inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys, + CFTypeRef* values, + size_t size) { + return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); +} + +// Copies characters from a CFStringRef into a std::string. +std::string CFStringToString(CFStringRef cf_string); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, int32_t value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + uint32_t value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + CFStringRef value); + +#endif // SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.cc b/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.cc new file mode 100644 index 0000000000..73c0ed0abd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.cc @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "sdk/objc/components/video_codec/nalu_rewriter.h" + +#include <CoreFoundation/CoreFoundation.h> + +#include <memory> +#include <vector> + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +using H264::kAud; +using H264::kSps; +using H264::NaluIndex; +using H264::NaluType; +using H264::ParseNaluType; + +const char kAnnexBHeaderBytes[4] = {0, 0, 0, 1}; +const size_t kAvccHeaderByteSize = sizeof(uint32_t); + +bool H264CMSampleBufferToAnnexBBuffer(CMSampleBufferRef avcc_sample_buffer, + bool is_keyframe, + rtc::Buffer* annexb_buffer) { + RTC_DCHECK(avcc_sample_buffer); + + // Get format description from the sample buffer. + CMVideoFormatDescriptionRef description = + CMSampleBufferGetFormatDescription(avcc_sample_buffer); + if (description == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to get sample buffer's description."; + return false; + } + + // Get parameter set information. + int nalu_header_size = 0; + size_t param_set_count = 0; + OSStatus status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + description, 0, nullptr, nullptr, ¶m_set_count, &nalu_header_size); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to get parameter set."; + return false; + } + RTC_CHECK_EQ(nalu_header_size, kAvccHeaderByteSize); + RTC_DCHECK_EQ(param_set_count, 2); + + // Truncate any previous data in the buffer without changing its capacity. + annexb_buffer->SetSize(0); + + // Place all parameter sets at the front of buffer. + if (is_keyframe) { + size_t param_set_size = 0; + const uint8_t* param_set = nullptr; + for (size_t i = 0; i < param_set_count; ++i) { + status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + description, i, ¶m_set, ¶m_set_size, nullptr, nullptr); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to get parameter set."; + return false; + } + // Update buffer. + annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes)); + annexb_buffer->AppendData(reinterpret_cast<const char*>(param_set), + param_set_size); + } + } + + // Get block buffer from the sample buffer. + CMBlockBufferRef block_buffer = + CMSampleBufferGetDataBuffer(avcc_sample_buffer); + if (block_buffer == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to get sample buffer's block buffer."; + return false; + } + CMBlockBufferRef contiguous_buffer = nullptr; + // Make sure block buffer is contiguous. + if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) { + status = CMBlockBufferCreateContiguous( + nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: " + << status; + return false; + } + } else { + contiguous_buffer = block_buffer; + // Retain to make cleanup easier. + CFRetain(contiguous_buffer); + block_buffer = nullptr; + } + + // Now copy the actual data. + char* data_ptr = nullptr; + size_t block_buffer_size = CMBlockBufferGetDataLength(contiguous_buffer); + status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr, nullptr, + &data_ptr); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to get block buffer data."; + CFRelease(contiguous_buffer); + return false; + } + size_t bytes_remaining = block_buffer_size; + while (bytes_remaining > 0) { + // The size type here must match `nalu_header_size`, we expect 4 bytes. + // Read the length of the next packet of data. Must convert from big endian + // to host endian. + RTC_DCHECK_GE(bytes_remaining, (size_t)nalu_header_size); + uint32_t* uint32_data_ptr = reinterpret_cast<uint32_t*>(data_ptr); + uint32_t packet_size = CFSwapInt32BigToHost(*uint32_data_ptr); + // Update buffer. + annexb_buffer->AppendData(kAnnexBHeaderBytes, sizeof(kAnnexBHeaderBytes)); + annexb_buffer->AppendData(data_ptr + nalu_header_size, packet_size); + + size_t bytes_written = packet_size + sizeof(kAnnexBHeaderBytes); + bytes_remaining -= bytes_written; + data_ptr += bytes_written; + } + RTC_DCHECK_EQ(bytes_remaining, (size_t)0); + + CFRelease(contiguous_buffer); + return true; +} + +bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, + size_t annexb_buffer_size, + CMVideoFormatDescriptionRef video_format, + CMSampleBufferRef* out_sample_buffer, + CMMemoryPoolRef memory_pool) { + RTC_DCHECK(annexb_buffer); + RTC_DCHECK(out_sample_buffer); + RTC_DCHECK(video_format); + *out_sample_buffer = nullptr; + + AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size); + if (reader.SeekToNextNaluOfType(kSps)) { + // Buffer contains an SPS NALU - skip it and the following PPS + const uint8_t* data; + size_t data_len; + if (!reader.ReadNalu(&data, &data_len)) { + RTC_LOG(LS_ERROR) << "Failed to read SPS"; + return false; + } + if (!reader.ReadNalu(&data, &data_len)) { + RTC_LOG(LS_ERROR) << "Failed to read PPS"; + return false; + } + } else { + // No SPS NALU - start reading from the first NALU in the buffer + reader.SeekToStart(); + } + + // Allocate memory as a block buffer. + CMBlockBufferRef block_buffer = nullptr; + CFAllocatorRef block_allocator = CMMemoryPoolGetAllocator(memory_pool); + OSStatus status = CMBlockBufferCreateWithMemoryBlock( + kCFAllocatorDefault, nullptr, reader.BytesRemaining(), block_allocator, + nullptr, 0, reader.BytesRemaining(), kCMBlockBufferAssureMemoryNowFlag, + &block_buffer); + if (status != kCMBlockBufferNoErr) { + RTC_LOG(LS_ERROR) << "Failed to create block buffer."; + return false; + } + + // Make sure block buffer is contiguous. + CMBlockBufferRef contiguous_buffer = nullptr; + if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) { + status = CMBlockBufferCreateContiguous(kCFAllocatorDefault, block_buffer, + block_allocator, nullptr, 0, 0, 0, + &contiguous_buffer); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: " + << status; + CFRelease(block_buffer); + return false; + } + } else { + contiguous_buffer = block_buffer; + block_buffer = nullptr; + } + + // Get a raw pointer into allocated memory. + size_t block_buffer_size = 0; + char* data_ptr = nullptr; + status = CMBlockBufferGetDataPointer(contiguous_buffer, 0, nullptr, + &block_buffer_size, &data_ptr); + if (status != kCMBlockBufferNoErr) { + RTC_LOG(LS_ERROR) << "Failed to get block buffer data pointer."; + CFRelease(contiguous_buffer); + return false; + } + RTC_DCHECK(block_buffer_size == reader.BytesRemaining()); + + // Write Avcc NALUs into block buffer memory. + AvccBufferWriter writer(reinterpret_cast<uint8_t*>(data_ptr), + block_buffer_size); + while (reader.BytesRemaining() > 0) { + const uint8_t* nalu_data_ptr = nullptr; + size_t nalu_data_size = 0; + if (reader.ReadNalu(&nalu_data_ptr, &nalu_data_size)) { + writer.WriteNalu(nalu_data_ptr, nalu_data_size); + } + } + + // Create sample buffer. + status = CMSampleBufferCreate(kCFAllocatorDefault, contiguous_buffer, true, + nullptr, nullptr, video_format, 1, 0, nullptr, + 0, nullptr, out_sample_buffer); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to create sample buffer."; + CFRelease(contiguous_buffer); + return false; + } + CFRelease(contiguous_buffer); + return true; +} + +CMVideoFormatDescriptionRef CreateVideoFormatDescription( + const uint8_t* annexb_buffer, + size_t annexb_buffer_size) { + const uint8_t* param_set_ptrs[2] = {}; + size_t param_set_sizes[2] = {}; + AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size); + // Skip everyting before the SPS, then read the SPS and PPS + if (!reader.SeekToNextNaluOfType(kSps)) { + return nullptr; + } + if (!reader.ReadNalu(¶m_set_ptrs[0], ¶m_set_sizes[0])) { + RTC_LOG(LS_ERROR) << "Failed to read SPS"; + return nullptr; + } + if (!reader.ReadNalu(¶m_set_ptrs[1], ¶m_set_sizes[1])) { + RTC_LOG(LS_ERROR) << "Failed to read PPS"; + return nullptr; + } + + // Parse the SPS and PPS into a CMVideoFormatDescription. + CMVideoFormatDescriptionRef description = nullptr; + OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets( + kCFAllocatorDefault, 2, param_set_ptrs, param_set_sizes, 4, &description); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to create video format description."; + return nullptr; + } + return description; +} + +AnnexBBufferReader::AnnexBBufferReader(const uint8_t* annexb_buffer, + size_t length) + : start_(annexb_buffer), length_(length) { + RTC_DCHECK(annexb_buffer); + offsets_ = H264::FindNaluIndices(annexb_buffer, length); + offset_ = offsets_.begin(); +} + +AnnexBBufferReader::~AnnexBBufferReader() = default; + +bool AnnexBBufferReader::ReadNalu(const uint8_t** out_nalu, + size_t* out_length) { + RTC_DCHECK(out_nalu); + RTC_DCHECK(out_length); + *out_nalu = nullptr; + *out_length = 0; + + if (offset_ == offsets_.end()) { + return false; + } + *out_nalu = start_ + offset_->payload_start_offset; + *out_length = offset_->payload_size; + ++offset_; + return true; +} + +size_t AnnexBBufferReader::BytesRemaining() const { + if (offset_ == offsets_.end()) { + return 0; + } + return length_ - offset_->start_offset; +} + +void AnnexBBufferReader::SeekToStart() { + offset_ = offsets_.begin(); +} + +bool AnnexBBufferReader::SeekToNextNaluOfType(NaluType type) { + for (; offset_ != offsets_.end(); ++offset_) { + if (offset_->payload_size < 1) + continue; + if (ParseNaluType(*(start_ + offset_->payload_start_offset)) == type) + return true; + } + return false; +} +AvccBufferWriter::AvccBufferWriter(uint8_t* const avcc_buffer, size_t length) + : start_(avcc_buffer), offset_(0), length_(length) { + RTC_DCHECK(avcc_buffer); +} + +bool AvccBufferWriter::WriteNalu(const uint8_t* data, size_t data_size) { + // Check if we can write this length of data. + if (data_size + kAvccHeaderByteSize > BytesRemaining()) { + return false; + } + // Write length header, which needs to be big endian. + uint32_t big_endian_length = CFSwapInt32HostToBig(data_size); + memcpy(start_ + offset_, &big_endian_length, sizeof(big_endian_length)); + offset_ += sizeof(big_endian_length); + // Write data. + memcpy(start_ + offset_, data, data_size); + offset_ += data_size; + return true; +} + +size_t AvccBufferWriter::BytesRemaining() const { + return length_ - offset_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.h b/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.h new file mode 100644 index 0000000000..c2b9e4875e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_codec/nalu_rewriter.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_NALU_REWRITER_H_ +#define SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_NALU_REWRITER_H_ + +#include <CoreMedia/CoreMedia.h> + +#include <vector> + +#include "common_video/h264/h264_common.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/buffer.h" + +using webrtc::H264::NaluIndex; + +namespace webrtc { + +// Converts a sample buffer emitted from the VideoToolbox encoder into a buffer +// suitable for RTP. The sample buffer is in avcc format whereas the rtp buffer +// needs to be in Annex B format. Data is written directly to `annexb_buffer`. +bool H264CMSampleBufferToAnnexBBuffer(CMSampleBufferRef avcc_sample_buffer, + bool is_keyframe, + rtc::Buffer* annexb_buffer); + +// Converts a buffer received from RTP into a sample buffer suitable for the +// VideoToolbox decoder. The RTP buffer is in annex b format whereas the sample +// buffer is in avcc format. +// If `is_keyframe` is true then `video_format` is ignored since the format will +// be read from the buffer. Otherwise `video_format` must be provided. +// Caller is responsible for releasing the created sample buffer. +bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, + size_t annexb_buffer_size, + CMVideoFormatDescriptionRef video_format, + CMSampleBufferRef* out_sample_buffer, + CMMemoryPoolRef memory_pool); + +// Returns a video format description created from the sps/pps information in +// the Annex B buffer. If there is no such information, nullptr is returned. +// The caller is responsible for releasing the description. +CMVideoFormatDescriptionRef CreateVideoFormatDescription( + const uint8_t* annexb_buffer, + size_t annexb_buffer_size); + +// Helper class for reading NALUs from an RTP Annex B buffer. +class AnnexBBufferReader final { + public: + AnnexBBufferReader(const uint8_t* annexb_buffer, size_t length); + ~AnnexBBufferReader(); + AnnexBBufferReader(const AnnexBBufferReader& other) = delete; + void operator=(const AnnexBBufferReader& other) = delete; + + // Returns a pointer to the beginning of the next NALU slice without the + // header bytes and its length. Returns false if no more slices remain. + bool ReadNalu(const uint8_t** out_nalu, size_t* out_length); + + // Returns the number of unread NALU bytes, including the size of the header. + // If the buffer has no remaining NALUs this will return zero. + size_t BytesRemaining() const; + + // Reset the reader to start reading from the first NALU + void SeekToStart(); + + // Seek to the next position that holds a NALU of the desired type, + // or the end if no such NALU is found. + // Return true if a NALU of the desired type is found, false if we + // reached the end instead + bool SeekToNextNaluOfType(H264::NaluType type); + + private: + // Returns the the next offset that contains NALU data. + size_t FindNextNaluHeader(const uint8_t* start, + size_t length, + size_t offset) const; + + const uint8_t* const start_; + std::vector<NaluIndex> offsets_; + std::vector<NaluIndex>::iterator offset_; + const size_t length_; +}; + +// Helper class for writing NALUs using avcc format into a buffer. +class AvccBufferWriter final { + public: + AvccBufferWriter(uint8_t* const avcc_buffer, size_t length); + ~AvccBufferWriter() {} + AvccBufferWriter(const AvccBufferWriter& other) = delete; + void operator=(const AvccBufferWriter& other) = delete; + + // Writes the data slice into the buffer. Returns false if there isn't + // enough space left. + bool WriteNalu(const uint8_t* data, size_t data_size); + + // Returns the unused bytes in the buffer. + size_t BytesRemaining() const; + + private: + uint8_t* const start_; + size_t offset_; + const size_t length_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_NALU_REWRITER_H_ diff --git a/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h new file mode 100644 index 0000000000..664d9bb904 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h @@ -0,0 +1,52 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#import "RTCMacros.h" +#import "RTCVideoFrameBuffer.h" + +NS_ASSUME_NONNULL_BEGIN + +/** RTCVideoFrameBuffer containing a CVPixelBufferRef */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCVPixelBuffer) : NSObject <RTC_OBJC_TYPE(RTCVideoFrameBuffer)> + +@property(nonatomic, readonly) CVPixelBufferRef pixelBuffer; +@property(nonatomic, readonly) int cropX; +@property(nonatomic, readonly) int cropY; +@property(nonatomic, readonly) int cropWidth; +@property(nonatomic, readonly) int cropHeight; + ++ (NSSet<NSNumber *> *)supportedPixelFormats; + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer; +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY; + +- (BOOL)requiresCropping; +- (BOOL)requiresScalingToWidth:(int)width height:(int)height; +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height; + +/** The minimum size of the `tmpBuffer` must be the number of bytes returned from the + * bufferSizeForCroppingAndScalingToWidth:height: method. + * If that size is 0, the `tmpBuffer` may be nil. + */ +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t *)tmpBuffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm new file mode 100644 index 0000000000..1a9b672d1a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm @@ -0,0 +1,367 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCVPixelBuffer.h" + +#import "api/video_frame_buffer/RTCNativeMutableI420Buffer.h" + +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv.h" + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +#import <UIKit/UIKit.h> +#import <VideoToolbox/VideoToolbox.h> +#endif + +@implementation RTC_OBJC_TYPE (RTCCVPixelBuffer) { + int _width; + int _height; + int _bufferWidth; + int _bufferHeight; + int _cropWidth; + int _cropHeight; +} + +@synthesize pixelBuffer = _pixelBuffer; +@synthesize cropX = _cropX; +@synthesize cropY = _cropY; +@synthesize cropWidth = _cropWidth; +@synthesize cropHeight = _cropHeight; + ++ (NSSet<NSNumber*>*)supportedPixelFormats { + return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), + @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + @(kCVPixelFormatType_32BGRA), + @(kCVPixelFormatType_32ARGB), + nil]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { + return [self initWithPixelBuffer:pixelBuffer + adaptedWidth:CVPixelBufferGetWidth(pixelBuffer) + adaptedHeight:CVPixelBufferGetHeight(pixelBuffer) + cropWidth:CVPixelBufferGetWidth(pixelBuffer) + cropHeight:CVPixelBufferGetHeight(pixelBuffer) + cropX:0 + cropY:0]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY { + if (self = [super init]) { + _width = adaptedWidth; + _height = adaptedHeight; + _pixelBuffer = pixelBuffer; + _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer); + _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer); + _cropWidth = cropWidth; + _cropHeight = cropHeight; + // Can only crop at even pixels. + _cropX = cropX & ~1; + _cropY = cropY & ~1; + CVBufferRetain(_pixelBuffer); + } + + return self; +} + +- (void)dealloc { + CVBufferRelease(_pixelBuffer); +} + +- (int)width { + return _width; +} + +- (int)height { + return _height; +} + +- (BOOL)requiresCropping { + return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight; +} + +- (BOOL)requiresScalingToWidth:(int)width height:(int)height { + return _cropWidth != width || _cropHeight != height; +} + +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height { + const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + switch (srcPixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + int srcChromaWidth = (_cropWidth + 1) / 2; + int srcChromaHeight = (_cropHeight + 1) / 2; + int dstChromaWidth = (width + 1) / 2; + int dstChromaHeight = (height + 1) / 2; + + return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + return 0; // Scaling RGBA frames does not require a temporary buffer. + } + } + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + return 0; +} + +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer + withTempBuffer:(nullable uint8_t*)tmpBuffer { + const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer); + + switch (srcPixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + if (dstWidth > 0 && dstHeight > 0) { + RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || + dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + if ([self requiresScalingToWidth:dstWidth height:dstHeight]) { + RTC_DCHECK(tmpBuffer); + } + [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer]; + } + break; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + RTC_DCHECK(srcPixelFormat == dstPixelFormat); + [self cropAndScaleARGBTo:outputPixelBuffer]; + break; + } + default: { + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + } + } + + return YES; +} +- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX + offsetY:(int)offsetY + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + scaleWidth:(int)scaleWidth + scaleHeight:(int)scaleHeight { + return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] + initWithPixelBuffer:_pixelBuffer + adaptedWidth:scaleWidth + adaptedHeight:scaleHeight + cropWidth:cropWidth * _cropWidth / _width + cropHeight:cropHeight * _cropHeight / _height + cropX:_cropX + offsetX * _cropWidth / _width + cropY:_cropY + offsetY * _cropHeight / _height]; +} + +- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 { + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + + RTC_OBJC_TYPE(RTCMutableI420Buffer)* i420Buffer = + [[RTC_OBJC_TYPE(RTCMutableI420Buffer) alloc] initWithWidth:[self width] height:[self height]]; + + switch (pixelFormat) { + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { + const uint8_t* srcY = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + // TODO(magjed): Use a frame buffer pool. + webrtc::NV12ToI420Scaler nv12ToI420Scaler; + nv12ToI420Scaler.NV12ToI420Scale(srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + break; + } + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_32ARGB: { + CVPixelBufferRef scaledPixelBuffer = NULL; + CVPixelBufferRef sourcePixelBuffer = NULL; + if ([self requiresCropping] || + [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) { + CVPixelBufferCreate( + NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer); + [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL]; + + CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + sourcePixelBuffer = scaledPixelBuffer; + } else { + sourcePixelBuffer = _pixelBuffer; + } + const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(sourcePixelBuffer)); + const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer); + + if (pixelFormat == kCVPixelFormatType_32BGRA) { + // Corresponds to libyuv::FOURCC_ARGB + libyuv::ARGBToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + // Corresponds to libyuv::FOURCC_BGRA + libyuv::BGRAToI420(src, + bytesPerRow, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + } + + if (scaledPixelBuffer) { + CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); + CVBufferRelease(scaledPixelBuffer); + } + break; + } + default: { + RTC_DCHECK_NOTREACHED() << "Unsupported pixel format."; + } + } + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + + return i420Buffer; +} + +#pragma mark - Debugging + +#if !defined(NDEBUG) && defined(WEBRTC_IOS) +- (id)debugQuickLookObject { + CGImageRef cgImage; + VTCreateCGImageFromCVPixelBuffer(_pixelBuffer, NULL, &cgImage); + UIImage *image = [UIImage imageWithCGImage:cgImage scale:1.0 orientation:UIImageOrientationUp]; + CGImageRelease(cgImage); + return image; +} +#endif + +#pragma mark - Private + +- (void)cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { + // Prepare output pointers. + CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + } + const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + uint8_t* dstY = + reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); + const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); + uint8_t* dstUV = + reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1)); + const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); + + // Prepare source pointers. + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + webrtc::NV12Scale(tmpBuffer, + srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + dstY, + dstYStride, + dstUV, + dstUVStride, + dstWidth, + dstHeight); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); +} + +- (void)cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer { + // Prepare output pointers. + CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + } + const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + + uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer)); + const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer); + + // Prepare source pointers. + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(_pixelBuffer)); + const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer); + + // Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding + // to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly. + const int bytesPerPixel = 4; + src += srcStride * _cropY + (_cropX * bytesPerPixel); + + // kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB + libyuv::ARGBScale(src, + srcStride, + _cropWidth, + _cropHeight, + dst, + dstStride, + dstWidth, + dstHeight, + libyuv::kFilterBox); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.h b/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.h new file mode 100644 index 0000000000..32ab6877f0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <CoreMedia/CoreMedia.h> + +NS_ASSUME_NONNULL_BEGIN + +@interface AVCaptureSession (DevicePosition) + +// Check the image's EXIF for the camera the image came from. ++ (AVCaptureDevicePosition)devicePositionForSampleBuffer:(CMSampleBufferRef)sampleBuffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.mm b/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.mm new file mode 100644 index 0000000000..0814ecc6c5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/AVCaptureSession+DevicePosition.mm @@ -0,0 +1,51 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "AVCaptureSession+DevicePosition.h" + +BOOL CFStringContainsString(CFStringRef theString, CFStringRef stringToFind) { + return CFStringFindWithOptions(theString, + stringToFind, + CFRangeMake(0, CFStringGetLength(theString)), + kCFCompareCaseInsensitive, + nil); +} + +@implementation AVCaptureSession (DevicePosition) + ++ (AVCaptureDevicePosition)devicePositionForSampleBuffer:(CMSampleBufferRef)sampleBuffer { + // Check the image's EXIF for the camera the image came from. + AVCaptureDevicePosition cameraPosition = AVCaptureDevicePositionUnspecified; + CFDictionaryRef attachments = CMCopyDictionaryOfAttachments( + kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate); + if (attachments) { + int size = CFDictionaryGetCount(attachments); + if (size > 0) { + CFDictionaryRef cfExifDictVal = nil; + if (CFDictionaryGetValueIfPresent( + attachments, (const void *)CFSTR("{Exif}"), (const void **)&cfExifDictVal)) { + CFStringRef cfLensModelStrVal; + if (CFDictionaryGetValueIfPresent(cfExifDictVal, + (const void *)CFSTR("LensModel"), + (const void **)&cfLensModelStrVal)) { + if (CFStringContainsString(cfLensModelStrVal, CFSTR("front"))) { + cameraPosition = AVCaptureDevicePositionFront; + } else if (CFStringContainsString(cfLensModelStrVal, CFSTR("back"))) { + cameraPosition = AVCaptureDevicePositionBack; + } + } + } + } + CFRelease(attachments); + } + return cameraPosition; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.h b/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.h new file mode 100644 index 0000000000..b0324e8a19 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.h @@ -0,0 +1,34 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#include <string> + +#include "absl/strings/string_view.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface NSString (StdString) + +@property(nonatomic, readonly) std::string stdString; + ++ (std::string)stdStringForString:(NSString *)nsString; ++ (NSString *)stringForStdString:(const std::string &)stdString; + +@end + +@interface NSString (AbslStringView) + ++ (NSString *)stringForAbslStringView:(const absl::string_view)abslStringView; + +@end + +NS_ASSUME_NONNULL_END diff --git a/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.mm b/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.mm new file mode 100644 index 0000000000..c98432c445 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/NSString+StdString.mm @@ -0,0 +1,45 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "NSString+StdString.h" + +#include "absl/strings/string_view.h" + +@implementation NSString (StdString) + +- (std::string)stdString { + return [NSString stdStringForString:self]; +} + ++ (std::string)stdStringForString:(NSString *)nsString { + NSData *charData = [nsString dataUsingEncoding:NSUTF8StringEncoding]; + return std::string(reinterpret_cast<const char *>(charData.bytes), + charData.length); +} + ++ (NSString *)stringForStdString:(const std::string&)stdString { + // std::string may contain null termination character so we construct + // using length. + return [[NSString alloc] initWithBytes:stdString.data() + length:stdString.length() + encoding:NSUTF8StringEncoding]; +} + +@end + +@implementation NSString (AbslStringView) + ++ (NSString *)stringForAbslStringView:(const absl::string_view)abslStringView { + return [[NSString alloc] initWithBytes:abslStringView.data() + length:abslStringView.length() + encoding:NSUTF8StringEncoding]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.h b/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.h new file mode 100644 index 0000000000..db9b15a45c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.h @@ -0,0 +1,30 @@ +/* + * Copyright 2015 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <UIKit/UIKit.h> + +#import "RTCMacros.h" + +@class AVCaptureSession; + +/** RTCCameraPreviewView is a view that renders local video from an + * AVCaptureSession. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCCameraPreviewView) : UIView + +/** The capture session being rendered in the view. Capture session + * is assigned to AVCaptureVideoPreviewLayer async in the same + * queue that the AVCaptureSession is started/stopped. + */ +@property(nonatomic, strong) AVCaptureSession* captureSession; + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.m b/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.m new file mode 100644 index 0000000000..12e87d8d64 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/RTCCameraPreviewView.m @@ -0,0 +1,123 @@ +/* + * Copyright 2015 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCCameraPreviewView.h" + +#import <AVFoundation/AVFoundation.h> +#import <UIKit/UIKit.h> + +#import "RTCDispatcher+Private.h" + +@implementation RTC_OBJC_TYPE (RTCCameraPreviewView) + +@synthesize captureSession = _captureSession; + ++ (Class)layerClass { + return [AVCaptureVideoPreviewLayer class]; +} + +- (instancetype)initWithFrame:(CGRect)aRect { + self = [super initWithFrame:aRect]; + if (self) { + [self addOrientationObserver]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder*)aDecoder { + self = [super initWithCoder:aDecoder]; + if (self) { + [self addOrientationObserver]; + } + return self; +} + +- (void)dealloc { + [self removeOrientationObserver]; +} + +- (void)setCaptureSession:(AVCaptureSession *)captureSession { + if (_captureSession == captureSession) { + return; + } + _captureSession = captureSession; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeMain + block:^{ + AVCaptureVideoPreviewLayer *previewLayer = [self previewLayer]; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + previewLayer.session = captureSession; + [RTC_OBJC_TYPE(RTCDispatcher) + dispatchAsyncOnType:RTCDispatcherTypeMain + block:^{ + [self setCorrectVideoOrientation]; + }]; + }]; + }]; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + + // Update the video orientation based on the device orientation. + [self setCorrectVideoOrientation]; +} + +-(void)orientationChanged:(NSNotification *)notification { + [self setCorrectVideoOrientation]; +} + +- (void)setCorrectVideoOrientation { + // Get current device orientation. + UIDeviceOrientation deviceOrientation = [UIDevice currentDevice].orientation; + AVCaptureVideoPreviewLayer *previewLayer = [self previewLayer]; + + // First check if we are allowed to set the video orientation. + if (previewLayer.connection.isVideoOrientationSupported) { + // Set the video orientation based on device orientation. + if (deviceOrientation == UIInterfaceOrientationPortraitUpsideDown) { + previewLayer.connection.videoOrientation = + AVCaptureVideoOrientationPortraitUpsideDown; + } else if (deviceOrientation == UIInterfaceOrientationLandscapeRight) { + previewLayer.connection.videoOrientation = + AVCaptureVideoOrientationLandscapeRight; + } else if (deviceOrientation == UIInterfaceOrientationLandscapeLeft) { + previewLayer.connection.videoOrientation = + AVCaptureVideoOrientationLandscapeLeft; + } else if (deviceOrientation == UIInterfaceOrientationPortrait) { + previewLayer.connection.videoOrientation = + AVCaptureVideoOrientationPortrait; + } + // If device orientation switches to FaceUp or FaceDown, don't change video orientation. + } +} + +#pragma mark - Private + +- (void)addOrientationObserver { + [[NSNotificationCenter defaultCenter] addObserver:self + selector:@selector(orientationChanged:) + name:UIDeviceOrientationDidChangeNotification + object:nil]; +} + +- (void)removeOrientationObserver { + [[NSNotificationCenter defaultCenter] removeObserver:self + name:UIDeviceOrientationDidChangeNotification + object:nil]; +} + +- (AVCaptureVideoPreviewLayer *)previewLayer { + return (AVCaptureVideoPreviewLayer *)self.layer; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher+Private.h b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher+Private.h new file mode 100644 index 0000000000..195c651790 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher+Private.h @@ -0,0 +1,18 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDispatcher.h" + +@interface RTC_OBJC_TYPE (RTCDispatcher) +() + + + (dispatch_queue_t)dispatchQueueForType : (RTCDispatcherQueueType)dispatchType; + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.h b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.h new file mode 100644 index 0000000000..e148af6dea --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.h @@ -0,0 +1,46 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> + +#import "RTCMacros.h" + +typedef NS_ENUM(NSInteger, RTCDispatcherQueueType) { + // Main dispatcher queue. + RTCDispatcherTypeMain, + // Used for starting/stopping AVCaptureSession, and assigning + // capture session to AVCaptureVideoPreviewLayer. + RTCDispatcherTypeCaptureSession, + // Used for operations on AVAudioSession. + RTCDispatcherTypeAudioSession, + // Used for operations on NWPathMonitor. + RTCDispatcherTypeNetworkMonitor, +}; + +/** Dispatcher that asynchronously dispatches blocks to a specific + * shared dispatch queue. + */ +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDispatcher) : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +/** Dispatch the block asynchronously on the queue for dispatchType. + * @param dispatchType The queue type to dispatch on. + * @param block The block to dispatch asynchronously. + */ ++ (void)dispatchAsyncOnType:(RTCDispatcherQueueType)dispatchType block:(dispatch_block_t)block; + +/** Returns YES if run on queue for the dispatchType otherwise NO. + * Useful for asserting that a method is run on a correct queue. + */ ++ (BOOL)isOnQueueForType:(RTCDispatcherQueueType)dispatchType; + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.m b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.m new file mode 100644 index 0000000000..4df19bc297 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/RTCDispatcher.m @@ -0,0 +1,65 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCDispatcher+Private.h" + +static dispatch_queue_t kAudioSessionQueue = nil; +static dispatch_queue_t kCaptureSessionQueue = nil; +static dispatch_queue_t kNetworkMonitorQueue = nil; + +@implementation RTC_OBJC_TYPE (RTCDispatcher) + ++ (void)initialize { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + kAudioSessionQueue = dispatch_queue_create( + "org.webrtc.RTCDispatcherAudioSession", + DISPATCH_QUEUE_SERIAL); + kCaptureSessionQueue = dispatch_queue_create( + "org.webrtc.RTCDispatcherCaptureSession", + DISPATCH_QUEUE_SERIAL); + kNetworkMonitorQueue = + dispatch_queue_create("org.webrtc.RTCDispatcherNetworkMonitor", DISPATCH_QUEUE_SERIAL); + }); +} + ++ (void)dispatchAsyncOnType:(RTCDispatcherQueueType)dispatchType + block:(dispatch_block_t)block { + dispatch_queue_t queue = [self dispatchQueueForType:dispatchType]; + dispatch_async(queue, block); +} + ++ (BOOL)isOnQueueForType:(RTCDispatcherQueueType)dispatchType { + dispatch_queue_t targetQueue = [self dispatchQueueForType:dispatchType]; + const char* targetLabel = dispatch_queue_get_label(targetQueue); + const char* currentLabel = dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL); + + NSAssert(strlen(targetLabel) > 0, @"Label is required for the target queue."); + NSAssert(strlen(currentLabel) > 0, @"Label is required for the current queue."); + + return strcmp(targetLabel, currentLabel) == 0; +} + +#pragma mark - Private + ++ (dispatch_queue_t)dispatchQueueForType:(RTCDispatcherQueueType)dispatchType { + switch (dispatchType) { + case RTCDispatcherTypeMain: + return dispatch_get_main_queue(); + case RTCDispatcherTypeCaptureSession: + return kCaptureSessionQueue; + case RTCDispatcherTypeAudioSession: + return kAudioSessionQueue; + case RTCDispatcherTypeNetworkMonitor: + return kNetworkMonitorQueue; + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.h b/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.h new file mode 100644 index 0000000000..4d04f38f8c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.h @@ -0,0 +1,17 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <UIKit/UIKit.h> + +@interface UIDevice (RTCDevice) + ++ (NSString *)machineName; + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.mm b/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.mm new file mode 100644 index 0000000000..f3c21991b8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/UIDevice+RTCDevice.mm @@ -0,0 +1,25 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "UIDevice+RTCDevice.h" + +#import <sys/utsname.h> +#include <memory> + +@implementation UIDevice (RTCDevice) + ++ (NSString *)machineName { + struct utsname systemInfo; + uname(&systemInfo); + return [[NSString alloc] initWithCString:systemInfo.machine + encoding:NSUTF8StringEncoding]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/helpers/noop.mm b/third_party/libwebrtc/sdk/objc/helpers/noop.mm new file mode 100644 index 0000000000..16a8e6d5c1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/noop.mm @@ -0,0 +1,13 @@ +/* + * Copyright 2015 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is only needed to make ninja happy on some platforms. +// On some platforms it is not possible to link an rtc_static_library +// without any source file listed in the GN target. diff --git a/third_party/libwebrtc/sdk/objc/helpers/scoped_cftyperef.h b/third_party/libwebrtc/sdk/objc/helpers/scoped_cftyperef.h new file mode 100644 index 0000000000..092f02b3af --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/helpers/scoped_cftyperef.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef SDK_OBJC_HELPERS_SCOPED_CFTYPEREF_H_ +#define SDK_OBJC_HELPERS_SCOPED_CFTYPEREF_H_ + +#include <CoreFoundation/CoreFoundation.h> +namespace rtc { + +// RETAIN: ScopedTypeRef should retain the object when it takes +// ownership. +// ASSUME: Assume the object already has already been retained. +// ScopedTypeRef takes over ownership. +enum class RetainPolicy { RETAIN, ASSUME }; + +namespace internal { +template <typename T> +struct CFTypeRefTraits { + static T InvalidValue() { return nullptr; } + static void Release(T ref) { CFRelease(ref); } + static T Retain(T ref) { + CFRetain(ref); + return ref; + } +}; + +template <typename T, typename Traits> +class ScopedTypeRef { + public: + ScopedTypeRef() : ptr_(Traits::InvalidValue()) {} + explicit ScopedTypeRef(T ptr) : ptr_(ptr) {} + ScopedTypeRef(T ptr, RetainPolicy policy) : ScopedTypeRef(ptr) { + if (ptr_ && policy == RetainPolicy::RETAIN) + Traits::Retain(ptr_); + } + + ScopedTypeRef(const ScopedTypeRef<T, Traits>& rhs) : ptr_(rhs.ptr_) { + if (ptr_) + ptr_ = Traits::Retain(ptr_); + } + + ~ScopedTypeRef() { + if (ptr_) { + Traits::Release(ptr_); + } + } + + T get() const { return ptr_; } + T operator->() const { return ptr_; } + explicit operator bool() const { return ptr_; } + + bool operator!() const { return !ptr_; } + + ScopedTypeRef& operator=(const T& rhs) { + if (ptr_) + Traits::Release(ptr_); + ptr_ = rhs; + return *this; + } + + ScopedTypeRef& operator=(const ScopedTypeRef<T, Traits>& rhs) { + reset(rhs.get(), RetainPolicy::RETAIN); + return *this; + } + + // This is intended to take ownership of objects that are + // created by pass-by-pointer initializers. + T* InitializeInto() { + RTC_DCHECK(!ptr_); + return &ptr_; + } + + void reset(T ptr, RetainPolicy policy = RetainPolicy::ASSUME) { + if (ptr && policy == RetainPolicy::RETAIN) + Traits::Retain(ptr); + if (ptr_) + Traits::Release(ptr_); + ptr_ = ptr; + } + + T release() { + T temp = ptr_; + ptr_ = Traits::InvalidValue(); + return temp; + } + + private: + T ptr_; +}; +} // namespace internal + +template <typename T> +using ScopedCFTypeRef = + internal::ScopedTypeRef<T, internal::CFTypeRefTraits<T>>; + +template <typename T> +static ScopedCFTypeRef<T> AdoptCF(T cftype) { + return ScopedCFTypeRef<T>(cftype, RetainPolicy::RETAIN); +} + +template <typename T> +static ScopedCFTypeRef<T> ScopedCF(T cftype) { + return ScopedCFTypeRef<T>(cftype); +} + +} // namespace rtc + +#endif // SDK_OBJC_HELPERS_SCOPED_CFTYPEREF_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.h b/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.h new file mode 100644 index 0000000000..3405469709 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.h @@ -0,0 +1,30 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_AUDIO_DEVICE_MODULE_H_ +#define SDK_OBJC_NATIVE_API_AUDIO_DEVICE_MODULE_H_ + +#include <memory> + +#include "modules/audio_device/include/audio_device.h" + +namespace webrtc { + +// If `bypass_voice_processing` is true, WebRTC will attempt to disable hardware +// audio processing on iOS. +// Warning: Setting `bypass_voice_processing` will have unpredictable +// consequences for the audio path in the device. It is not advisable to use in +// most scenarios. +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( + bool bypass_voice_processing = false); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_AUDIO_DEVICE_MODULE_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.mm b/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.mm new file mode 100644 index 0000000000..4e7b681e69 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/audio_device_module.mm @@ -0,0 +1,29 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio_device_module.h" + +#include "api/make_ref_counted.h" +#include "rtc_base/logging.h" + +#include "sdk/objc/native/src/audio/audio_device_module_ios.h" + +namespace webrtc { + +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(bool bypass_voice_processing) { + RTC_DLOG(LS_INFO) << __FUNCTION__; +#if defined(WEBRTC_IOS) + return rtc::make_ref_counted<ios_adm::AudioDeviceModuleIOS>(bypass_voice_processing); +#else + RTC_LOG(LS_ERROR) << "current platform is not supported => this module will self destruct!"; + return nullptr; +#endif +} +} diff --git a/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.h b/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.h new file mode 100644 index 0000000000..903c66893d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.h @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_NETWORK_MONITOR_FACTORY_H_ +#define SDK_OBJC_NATIVE_API_NETWORK_MONITOR_FACTORY_H_ + +#include <memory> + +#include "rtc_base/network_monitor_factory.h" + +namespace webrtc { + +std::unique_ptr<rtc::NetworkMonitorFactory> CreateNetworkMonitorFactory(); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_NETWORK_MONITOR_FACTORY_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.mm b/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.mm new file mode 100644 index 0000000000..acde634b1d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/network_monitor_factory.mm @@ -0,0 +1,30 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "network_monitor_factory.h" + +#if defined(WEBRTC_IOS) +#include "sdk/objc/native/src/objc_network_monitor.h" +#endif + +#include "rtc_base/logging.h" + +namespace webrtc { + +std::unique_ptr<rtc::NetworkMonitorFactory> CreateNetworkMonitorFactory() { + RTC_DLOG(LS_INFO) << __FUNCTION__; +#if defined(WEBRTC_IOS) + return std::make_unique<ObjCNetworkMonitorFactory>(); +#else + return nullptr; +#endif +} + +} diff --git a/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.h b/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.h new file mode 100644 index 0000000000..0fe2dda4a0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.h @@ -0,0 +1,24 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ +#define SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ + +#import "components/audio/RTCAudioDevice.h" +#include "modules/audio_device/include/audio_device.h" + +namespace webrtc { + +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( + id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.mm b/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.mm new file mode 100644 index 0000000000..76edd45605 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/objc_audio_device_module.mm @@ -0,0 +1,26 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "objc_audio_device_module.h" + +#include "api/make_ref_counted.h" +#include "rtc_base/logging.h" + +#include "sdk/objc/native/src/objc_audio_device.h" + +namespace webrtc { + +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( + id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + return rtc::make_ref_counted<objc_adm::ObjCAudioDeviceModule>(audio_device); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.h b/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.h new file mode 100644 index 0000000000..35ab1be9a8 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.h @@ -0,0 +1,26 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_SSL_CERTIFICATE_VERIFIER_H_ +#define SDK_OBJC_NATIVE_API_SSL_CERTIFICATE_VERIFIER_H_ + +#include <memory> + +#import "RTCSSLCertificateVerifier.h" +#include "rtc_base/ssl_certificate.h" + +namespace webrtc { + +std::unique_ptr<rtc::SSLCertificateVerifier> ObjCToNativeCertificateVerifier( + id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)> objc_certificate_verifier); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_SSL_CERTIFICATE_VERIFIER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.mm b/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.mm new file mode 100644 index 0000000000..4437402b9c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/ssl_certificate_verifier.mm @@ -0,0 +1,48 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "ssl_certificate_verifier.h" + +#include "rtc_base/buffer.h" + +namespace { + +class SSLCertificateVerifierAdapter final : public rtc::SSLCertificateVerifier { + public: + SSLCertificateVerifierAdapter( + id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)> objc_certificate_verifier) + : objc_certificate_verifier_(objc_certificate_verifier) { + RTC_DCHECK(objc_certificate_verifier_ != nil); + } + + bool Verify(const rtc::SSLCertificate& certificate) override { + @autoreleasepool { + rtc::Buffer der_buffer; + certificate.ToDER(&der_buffer); + NSData* serialized_certificate = [[NSData alloc] initWithBytes:der_buffer.data() + length:der_buffer.size()]; + return [objc_certificate_verifier_ verify:serialized_certificate]; + } + } + + private: + id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)> objc_certificate_verifier_; +}; + +} + +namespace webrtc { + +std::unique_ptr<rtc::SSLCertificateVerifier> ObjCToNativeCertificateVerifier( + id<RTC_OBJC_TYPE(RTCSSLCertificateVerifier)> objc_certificate_verifier) { + return std::make_unique<SSLCertificateVerifierAdapter>(objc_certificate_verifier); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_capturer.h b/third_party/libwebrtc/sdk/objc/native/api/video_capturer.h new file mode 100644 index 0000000000..c1dfb07868 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_capturer.h @@ -0,0 +1,31 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_CAPTURER_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_CAPTURER_H_ + +// import +#import "base/RTCVideoCapturer.h" + +// include +#include "api/media_stream_interface.h" +#include "api/scoped_refptr.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> ObjCToNativeVideoCapturer( + RTC_OBJC_TYPE(RTCVideoCapturer) * objc_video_capturer, + rtc::Thread* signaling_thread, + rtc::Thread* worker_thread); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_CAPTURER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_capturer.mm b/third_party/libwebrtc/sdk/objc/native/api/video_capturer.mm new file mode 100644 index 0000000000..a7260ab802 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_capturer.mm @@ -0,0 +1,35 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_capturer.h" + +#include "absl/memory/memory.h" +#include "api/video_track_source_proxy_factory.h" +#include "sdk/objc/native/src/objc_video_track_source.h" + +namespace webrtc { + +rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> ObjCToNativeVideoCapturer( + RTC_OBJC_TYPE(RTCVideoCapturer) * objc_video_capturer, + rtc::Thread *signaling_thread, + rtc::Thread *worker_thread) { + RTCObjCVideoSourceAdapter *adapter = [[RTCObjCVideoSourceAdapter alloc] init]; + rtc::scoped_refptr<webrtc::ObjCVideoTrackSource> objc_video_track_source = + rtc::make_ref_counted<webrtc::ObjCVideoTrackSource>(adapter); + rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> video_source = + webrtc::CreateVideoTrackSourceProxy( + signaling_thread, worker_thread, objc_video_track_source.get()); + + objc_video_capturer.delegate = adapter; + + return video_source; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.h b/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.h new file mode 100644 index 0000000000..9ba11d65a3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.h @@ -0,0 +1,26 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_DECODER_FACTORY_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_DECODER_FACTORY_H_ + +#include <memory> + +#include "api/video_codecs/video_decoder_factory.h" +#import "base/RTCVideoDecoderFactory.h" + +namespace webrtc { + +std::unique_ptr<VideoDecoderFactory> ObjCToNativeVideoDecoderFactory( + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> objc_video_decoder_factory); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.mm b/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.mm new file mode 100644 index 0000000000..d418f2fe6f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_decoder_factory.mm @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_decoder_factory.h" + +#include <memory> + +#include "sdk/objc/native/src/objc_video_decoder_factory.h" + +namespace webrtc { + +std::unique_ptr<VideoDecoderFactory> ObjCToNativeVideoDecoderFactory( + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> objc_video_decoder_factory) { + return std::make_unique<ObjCVideoDecoderFactory>(objc_video_decoder_factory); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.h b/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.h new file mode 100644 index 0000000000..ecd9ab090b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.h @@ -0,0 +1,26 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_ENCODER_FACTORY_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_ENCODER_FACTORY_H_ + +#include <memory> + +#include "api/video_codecs/video_encoder_factory.h" +#import "base/RTCVideoEncoderFactory.h" + +namespace webrtc { + +std::unique_ptr<VideoEncoderFactory> ObjCToNativeVideoEncoderFactory( + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> objc_video_encoder_factory); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.mm b/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.mm new file mode 100644 index 0000000000..6fa5563f75 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_encoder_factory.mm @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_encoder_factory.h" + +#include <memory> + +#include "sdk/objc/native/src/objc_video_encoder_factory.h" + +namespace webrtc { + +std::unique_ptr<VideoEncoderFactory> ObjCToNativeVideoEncoderFactory( + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> objc_video_encoder_factory) { + return std::make_unique<ObjCVideoEncoderFactory>(objc_video_encoder_factory); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_frame.h b/third_party/libwebrtc/sdk/objc/native/api/video_frame.h new file mode 100644 index 0000000000..4ca469f2f2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_frame.h @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_FRAME_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_FRAME_H_ + +#include "api/video/video_frame.h" +#import "base/RTCVideoFrame.h" + +namespace webrtc { + +RTC_OBJC_TYPE(RTCVideoFrame) * NativeToObjCVideoFrame(const VideoFrame& frame); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_FRAME_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_frame.mm b/third_party/libwebrtc/sdk/objc/native/api/video_frame.mm new file mode 100644 index 0000000000..b82994fd5f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_frame.mm @@ -0,0 +1,21 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_frame.h" + +#include "sdk/objc/native/src/objc_video_frame.h" + +namespace webrtc { + +RTC_OBJC_TYPE(RTCVideoFrame) * NativeToObjCVideoFrame(const VideoFrame& frame) { + return ToObjCVideoFrame(frame); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.h b/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.h new file mode 100644 index 0000000000..68a8543d26 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.h @@ -0,0 +1,31 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_FRAME_BUFFER_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_FRAME_BUFFER_H_ + +// import +#import "base/RTCVideoFrameBuffer.h" + +// include +#include "api/scoped_refptr.h" +#include "common_video/include/video_frame_buffer.h" + +namespace webrtc { + +rtc::scoped_refptr<VideoFrameBuffer> ObjCToNativeVideoFrameBuffer( + id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> objc_video_frame_buffer); + +id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> NativeToObjCVideoFrameBuffer( + const rtc::scoped_refptr<VideoFrameBuffer>& buffer); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_FRAME_BUFFER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.mm b/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.mm new file mode 100644 index 0000000000..4fe9037bce --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_frame_buffer.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_frame_buffer.h" + +#include "api/make_ref_counted.h" +#include "sdk/objc/native/src/objc_frame_buffer.h" + +namespace webrtc { + +rtc::scoped_refptr<VideoFrameBuffer> ObjCToNativeVideoFrameBuffer( + id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> objc_video_frame_buffer) { + return rtc::make_ref_counted<ObjCFrameBuffer>(objc_video_frame_buffer); +} + +id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> NativeToObjCVideoFrameBuffer( + const rtc::scoped_refptr<VideoFrameBuffer> &buffer) { + return ToObjCVideoFrameBuffer(buffer); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_renderer.h b/third_party/libwebrtc/sdk/objc/native/api/video_renderer.h new file mode 100644 index 0000000000..279857a860 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_renderer.h @@ -0,0 +1,27 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_VIDEO_RENDERER_H_ +#define SDK_OBJC_NATIVE_API_VIDEO_RENDERER_H_ + +#include <memory> + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#import "base/RTCVideoRenderer.h" + +namespace webrtc { + +std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> ObjCToNativeVideoRenderer( + id<RTC_OBJC_TYPE(RTCVideoRenderer)> objc_video_renderer); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_VIDEO_RENDERER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/api/video_renderer.mm b/third_party/libwebrtc/sdk/objc/native/api/video_renderer.mm new file mode 100644 index 0000000000..e92d47d1e3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/api/video_renderer.mm @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/api/video_renderer.h" + +#include <memory> + +#include "sdk/objc/native/src/objc_video_renderer.h" + +namespace webrtc { + +std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> ObjCToNativeVideoRenderer( + id<RTC_OBJC_TYPE(RTCVideoRenderer)> objc_video_renderer) { + return std::make_unique<ObjCVideoRenderer>(objc_video_renderer); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.h b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.h new file mode 100644 index 0000000000..a86acb56fe --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.h @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ +#define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ + +#include <atomic> +#include <memory> + +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "audio_session_observer.h" +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/buffer.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "sdk/objc/base/RTCMacros.h" +#include "voice_processing_audio_unit.h" + +RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter); + +namespace webrtc { + +class FineAudioBuffer; + +namespace ios_adm { + +// Implements full duplex 16-bit mono PCM audio support for iOS using a +// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit +// supports audio echo cancellation. It also adds automatic gain control, +// adjustment of voice-processing quality and muting. +// +// An instance must be created and destroyed on one and the same thread. +// All supported public methods must also be called on the same thread. +// A thread checker will RTC_DCHECK if any supported method is called on an +// invalid thread. +// +// Recorded audio will be delivered on a real-time internal I/O thread in the +// audio unit. The audio unit will also ask for audio data to play out on this +// same thread. +class AudioDeviceIOS : public AudioDeviceGeneric, + public AudioSessionObserver, + public VoiceProcessingAudioUnitObserver { + public: + explicit AudioDeviceIOS(bool bypass_voice_processing); + ~AudioDeviceIOS() override; + + void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; + + InitStatus Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // These methods returns hard-coded delay values and not dynamic delay + // estimates. The reason is that iOS supports a built-in AEC and the WebRTC + // AEC will always be disabled in the Libjingle layer to avoid running two + // AEC implementations at the same time. And, it saves resources to avoid + // updating these delay values continuously. + // TODO(henrika): it would be possible to mark these two methods as not + // implemented since they are only called for A/V-sync purposes today and + // A/V-sync is not supported on iOS. However, we avoid adding error messages + // the log by using these dummy implementations instead. + int32_t PlayoutDelay(uint16_t& delayMS) const override; + + // No implementation for playout underrun on iOS. We override it to avoid a + // periodic log that it isn't available from the base class. + int32_t GetPlayoutUnderrunCount() const override { return -1; } + + // Native audio parameters stored during construction. + // These methods are unique for the iOS implementation. + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; + + // These methods are currently not fully implemented on iOS: + + // See audio_device_not_implemented.cc for trivial implementations. + int32_t ActiveAudioLayer( + AudioDeviceModule::AudioLayer& audioLayer) const override; + int32_t PlayoutIsAvailable(bool& available) override; + int32_t RecordingIsAvailable(bool& available) override; + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice( + AudioDeviceModule::WindowsDeviceType device) override; + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + int32_t SpeakerVolumeIsAvailable(bool& available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t& volume) const override; + int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t& minVolume) const override; + int32_t MicrophoneVolumeIsAvailable(bool& available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t& volume) const override; + int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t& minVolume) const override; + int32_t MicrophoneMuteIsAvailable(bool& available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool& enabled) const override; + int32_t SpeakerMuteIsAvailable(bool& available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool& enabled) const override; + int32_t StereoPlayoutIsAvailable(bool& available) override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool& enabled) const override; + int32_t StereoRecordingIsAvailable(bool& available) override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool& enabled) const override; + + // AudioSessionObserver methods. May be called from any thread. + void OnInterruptionBegin() override; + void OnInterruptionEnd() override; + void OnValidRouteChange() override; + void OnCanPlayOrRecordChange(bool can_play_or_record) override; + void OnChangedOutputVolume() override; + + // VoiceProcessingAudioUnitObserver methods. + OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) override; + + bool IsInterrupted(); + + private: + // Called by the relevant AudioSessionObserver methods on `thread_`. + void HandleInterruptionBegin(); + void HandleInterruptionEnd(); + void HandleValidRouteChange(); + void HandleCanPlayOrRecordChange(bool can_play_or_record); + void HandleSampleRateChange(); + void HandlePlayoutGlitchDetected(); + void HandleOutputVolumeChange(); + + // Uses current `playout_parameters_` and `record_parameters_` to inform the + // audio device buffer (ADB) about our internal audio parameters. + void UpdateAudioDeviceBuffer(); + + // Since the preferred audio parameters are only hints to the OS, the actual + // values may be different once the AVAudioSession has been activated. + // This method asks for the current hardware parameters and takes actions + // if they should differ from what we have asked for initially. It also + // defines `playout_parameters_` and `record_parameters_`. + void SetupAudioBuffersForActiveAudioSession(); + + // Creates the audio unit. + bool CreateAudioUnit(); + + // Updates the audio unit state based on current state. + void UpdateAudioUnit(bool can_play_or_record); + + // Configures the audio session for WebRTC. + bool ConfigureAudioSession(); + + // Like above, but requires caller to already hold session lock. + bool ConfigureAudioSessionLocked(); + + // Unconfigures the audio session. + void UnconfigureAudioSession(); + + // Activates our audio session, creates and initializes the voice-processing + // audio unit and verifies that we got the preferred native audio parameters. + bool InitPlayOrRecord(); + + // Closes and deletes the voice-processing I/O unit. + void ShutdownPlayOrRecord(); + + // Resets thread-checkers before a call is restarted. + void PrepareForNewStart(); + + // Determines whether voice processing should be enabled or disabled. + const bool bypass_voice_processing_; + + // Native I/O audio thread checker. + SequenceChecker io_thread_checker_; + + // Thread that this object is created on. + rtc::Thread* thread_; + + // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the + // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). + // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance + // and therefore outlives this object. + AudioDeviceBuffer* audio_device_buffer_; + + // Contains audio parameters (sample rate, #channels, buffer size etc.) for + // the playout and recording sides. These structure is set in two steps: + // first, native sample rate and #channels are defined in Init(). Next, the + // audio session is activated and we verify that the preferred parameters + // were granted by the OS. At this stage it is also possible to add a third + // component to the parameters; the native I/O buffer duration. + // A RTC_CHECK will be hit if we for some reason fail to open an audio session + // using the specified parameters. + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + + // The AudioUnit used to play and record audio. + std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_; + + // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data + // in chunks of 10ms. It then allows for this data to be pulled in + // a finer or coarser granularity. I.e. interacting with this class instead + // of directly with the AudioDeviceBuffer one can ask for any number of + // audio data samples. Is also supports a similar scheme for the recording + // side. + // Example: native buffer size can be 128 audio frames at 16kHz sample rate. + // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128 + // in each callback (one every 8ms). This class can then ask for 128 and the + // FineAudioBuffer will ask WebRTC for new data only when needed and also + // cache non-utilized audio between callbacks. On the recording side, iOS + // can provide audio data frames of size 128 and these are accumulated until + // enough data to supply one 10ms call exists. This 10ms chunk is then sent + // to WebRTC and the remaining part is stored. + std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; + + // Temporary storage for recorded data. AudioUnitRender() renders into this + // array as soon as a frame of the desired buffer size has been recorded. + // On real iOS devices, the size will be fixed and set once. For iOS + // simulators, the size can vary from callback to callback and the size + // will be changed dynamically to account for this behavior. + rtc::BufferT<int16_t> record_audio_buffer_; + + // Set to 1 when recording is active and 0 otherwise. + std::atomic<int> recording_; + + // Set to 1 when playout is active and 0 otherwise. + std::atomic<int> playing_; + + // Set to true after successful call to Init(), false otherwise. + bool initialized_ RTC_GUARDED_BY(thread_); + + // Set to true after successful call to InitRecording() or InitPlayout(), + // false otherwise. + bool audio_is_initialized_; + + // Set to true if audio session is interrupted, false otherwise. + bool is_interrupted_; + + // Audio interruption observer instance. + RTCNativeAudioSessionDelegateAdapter* audio_session_observer_ + RTC_GUARDED_BY(thread_); + + // Set to true if we've activated the audio session. + bool has_configured_session_ RTC_GUARDED_BY(thread_); + + // Counts number of detected audio glitches on the playout side. + int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_); + int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_); + + // Counts number of playout callbacks per call. + // The value is updated on the native I/O thread and later read on the + // creating `thread_` but at this stage no audio is active. + // Hence, it is a "thread safe" design and no lock is needed. + int64_t num_playout_callbacks_; + + // Contains the time for when the last output volume change was detected. + int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_); + + // Avoids running pending task after `this` is Terminated. + rtc::scoped_refptr<PendingTaskSafetyFlag> safety_ = + PendingTaskSafetyFlag::Create(); +}; +} // namespace ios_adm +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.mm b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.mm new file mode 100644 index 0000000000..dd2c11bdd2 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_ios.mm @@ -0,0 +1,1127 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> + +#include "audio_device_ios.h" + +#include <cmath> + +#include "api/array_view.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "helpers.h" +#include "modules/audio_device/fine_audio_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +#import "base/RTCLogging.h" +#import "components/audio/RTCAudioSession+Private.h" +#import "components/audio/RTCAudioSession.h" +#import "components/audio/RTCAudioSessionConfiguration.h" +#import "components/audio/RTCNativeAudioSessionDelegateAdapter.h" + +namespace webrtc { +namespace ios_adm { + +#define LOGI() RTC_LOG(LS_INFO) << "AudioDeviceIOS::" + +#define LOG_AND_RETURN_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + return false; \ + } \ + } while (0) + +#define LOG_IF_ERROR(error, message) \ + do { \ + OSStatus err = error; \ + if (err) { \ + RTC_LOG(LS_ERROR) << message << ": " << err; \ + } \ + } while (0) + +// Hardcoded delay estimates based on real measurements. +// TODO(henrika): these value is not used in combination with built-in AEC. +// Can most likely be removed. +const UInt16 kFixedPlayoutDelayEstimate = 30; +const UInt16 kFixedRecordDelayEstimate = 30; + +using ios::CheckAndLogError; + +#if !defined(NDEBUG) +// Returns true when the code runs on a device simulator. +static bool DeviceIsSimulator() { + return ios::GetDeviceName() == "x86_64"; +} + +// Helper method that logs essential device information strings. +static void LogDeviceInfo() { + RTC_LOG(LS_INFO) << "LogDeviceInfo"; + @autoreleasepool { + RTC_LOG(LS_INFO) << " system name: " << ios::GetSystemName(); + RTC_LOG(LS_INFO) << " system version: " << ios::GetSystemVersionAsString(); + RTC_LOG(LS_INFO) << " device type: " << ios::GetDeviceType(); + RTC_LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); + RTC_LOG(LS_INFO) << " process name: " << ios::GetProcessName(); + RTC_LOG(LS_INFO) << " process ID: " << ios::GetProcessID(); + RTC_LOG(LS_INFO) << " OS version: " << ios::GetOSVersionString(); + RTC_LOG(LS_INFO) << " processing cores: " << ios::GetProcessorCount(); + RTC_LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); +#if TARGET_IPHONE_SIMULATOR + RTC_LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; +#endif + RTC_LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); + } +} +#endif // !defined(NDEBUG) + +AudioDeviceIOS::AudioDeviceIOS(bool bypass_voice_processing) + : bypass_voice_processing_(bypass_voice_processing), + audio_device_buffer_(nullptr), + audio_unit_(nullptr), + recording_(0), + playing_(0), + initialized_(false), + audio_is_initialized_(false), + is_interrupted_(false), + has_configured_session_(false), + num_detected_playout_glitches_(0), + last_playout_time_(0), + num_playout_callbacks_(0), + last_output_volume_change_time_(0) { + LOGI() << "ctor" << ios::GetCurrentThreadDescription() + << ",bypass_voice_processing=" << bypass_voice_processing_; + io_thread_checker_.Detach(); + thread_ = rtc::Thread::Current(); + + audio_session_observer_ = [[RTCNativeAudioSessionDelegateAdapter alloc] initWithObserver:this]; +} + +AudioDeviceIOS::~AudioDeviceIOS() { + RTC_DCHECK_RUN_ON(thread_); + LOGI() << "~dtor" << ios::GetCurrentThreadDescription(); + safety_->SetNotAlive(); + Terminate(); + audio_session_observer_ = nil; +} + +void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { + LOGI() << "AttachAudioBuffer"; + RTC_DCHECK(audioBuffer); + RTC_DCHECK_RUN_ON(thread_); + audio_device_buffer_ = audioBuffer; +} + +AudioDeviceGeneric::InitStatus AudioDeviceIOS::Init() { + LOGI() << "Init"; + io_thread_checker_.Detach(); + + RTC_DCHECK_RUN_ON(thread_); + if (initialized_) { + return InitStatus::OK; + } +#if !defined(NDEBUG) + LogDeviceInfo(); +#endif + // Store the preferred sample rate and preferred number of channels already + // here. They have not been set and confirmed yet since configureForWebRTC + // is not called until audio is about to start. However, it makes sense to + // store the parameters now and then verify at a later stage. + RTC_OBJC_TYPE(RTCAudioSessionConfiguration)* config = + [RTC_OBJC_TYPE(RTCAudioSessionConfiguration) webRTCConfiguration]; + playout_parameters_.reset(config.sampleRate, config.outputNumberOfChannels); + record_parameters_.reset(config.sampleRate, config.inputNumberOfChannels); + // Ensure that the audio device buffer (ADB) knows about the internal audio + // parameters. Note that, even if we are unable to get a mono audio session, + // we will always tell the I/O audio unit to do a channel format conversion + // to guarantee mono on the "input side" of the audio unit. + UpdateAudioDeviceBuffer(); + initialized_ = true; + return InitStatus::OK; +} + +int32_t AudioDeviceIOS::Terminate() { + LOGI() << "Terminate"; + RTC_DCHECK_RUN_ON(thread_); + if (!initialized_) { + return 0; + } + StopPlayout(); + StopRecording(); + initialized_ = false; + return 0; +} + +bool AudioDeviceIOS::Initialized() const { + RTC_DCHECK_RUN_ON(thread_); + return initialized_; +} + +int32_t AudioDeviceIOS::InitPlayout() { + LOGI() << "InitPlayout"; + RTC_DCHECK_RUN_ON(thread_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!playing_.load()); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +bool AudioDeviceIOS::PlayoutIsInitialized() const { + RTC_DCHECK_RUN_ON(thread_); + return audio_is_initialized_; +} + +bool AudioDeviceIOS::RecordingIsInitialized() const { + RTC_DCHECK_RUN_ON(thread_); + return audio_is_initialized_; +} + +int32_t AudioDeviceIOS::InitRecording() { + LOGI() << "InitRecording"; + RTC_DCHECK_RUN_ON(thread_); + RTC_DCHECK(initialized_); + RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!recording_.load()); + if (!audio_is_initialized_) { + if (!InitPlayOrRecord()) { + RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; + return -1; + } + } + audio_is_initialized_ = true; + return 0; +} + +int32_t AudioDeviceIOS::StartPlayout() { + LOGI() << "StartPlayout"; + RTC_DCHECK_RUN_ON(thread_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!playing_.load()); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetPlayout(); + } + if (!recording_.load() && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + OSStatus result = audio_unit_->Start(); + if (result != noErr) { + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session notifyAudioUnitStartFailedWithError:result]; + RTCLogError(@"StartPlayout failed to start audio unit, reason %d", result); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + playing_.store(1, std::memory_order_release); + num_playout_callbacks_ = 0; + num_detected_playout_glitches_ = 0; + return 0; +} + +int32_t AudioDeviceIOS::StopPlayout() { + LOGI() << "StopPlayout"; + RTC_DCHECK_RUN_ON(thread_); + if (!audio_is_initialized_ || !playing_.load()) { + return 0; + } + if (!recording_.load()) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + playing_.store(0, std::memory_order_release); + + // Derive average number of calls to OnGetPlayoutData() between detected + // audio glitches and add the result to a histogram. + int average_number_of_playout_callbacks_between_glitches = 100000; + RTC_DCHECK_GE(num_playout_callbacks_, num_detected_playout_glitches_); + if (num_detected_playout_glitches_ > 0) { + average_number_of_playout_callbacks_between_glitches = + num_playout_callbacks_ / num_detected_playout_glitches_; + } + RTC_HISTOGRAM_COUNTS_100000("WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches", + average_number_of_playout_callbacks_between_glitches); + RTCLog(@"Average number of playout callbacks between glitches: %d", + average_number_of_playout_callbacks_between_glitches); + return 0; +} + +bool AudioDeviceIOS::Playing() const { + return playing_.load(); +} + +int32_t AudioDeviceIOS::StartRecording() { + LOGI() << "StartRecording"; + RTC_DCHECK_RUN_ON(thread_); + RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(!recording_.load()); + RTC_DCHECK(audio_unit_); + if (fine_audio_buffer_) { + fine_audio_buffer_->ResetRecord(); + } + if (!playing_.load() && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + OSStatus result = audio_unit_->Start(); + if (result != noErr) { + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session notifyAudioUnitStartFailedWithError:result]; + RTCLogError(@"StartRecording failed to start audio unit, reason %d", result); + return -1; + } + RTC_LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started"; + } + recording_.store(1, std::memory_order_release); + return 0; +} + +int32_t AudioDeviceIOS::StopRecording() { + LOGI() << "StopRecording"; + RTC_DCHECK_RUN_ON(thread_); + if (!audio_is_initialized_ || !recording_.load()) { + return 0; + } + if (!playing_.load()) { + ShutdownPlayOrRecord(); + audio_is_initialized_ = false; + } + recording_.store(0, std::memory_order_release); + return 0; +} + +bool AudioDeviceIOS::Recording() const { + return recording_.load(); +} + +int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { + delayMS = kFixedPlayoutDelayEstimate; + return 0; +} + +int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { + LOGI() << "GetPlayoutAudioParameters"; + RTC_DCHECK(playout_parameters_.is_valid()); + RTC_DCHECK_RUN_ON(thread_); + *params = playout_parameters_; + return 0; +} + +int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { + LOGI() << "GetRecordAudioParameters"; + RTC_DCHECK(record_parameters_.is_valid()); + RTC_DCHECK_RUN_ON(thread_); + *params = record_parameters_; + return 0; +} + +void AudioDeviceIOS::OnInterruptionBegin() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionBegin"; + thread_->PostTask(SafeTask(safety_, [this] { HandleInterruptionBegin(); })); +} + +void AudioDeviceIOS::OnInterruptionEnd() { + RTC_DCHECK(thread_); + LOGI() << "OnInterruptionEnd"; + thread_->PostTask(SafeTask(safety_, [this] { HandleInterruptionEnd(); })); +} + +void AudioDeviceIOS::OnValidRouteChange() { + RTC_DCHECK(thread_); + thread_->PostTask(SafeTask(safety_, [this] { HandleValidRouteChange(); })); +} + +void AudioDeviceIOS::OnCanPlayOrRecordChange(bool can_play_or_record) { + RTC_DCHECK(thread_); + thread_->PostTask(SafeTask( + safety_, [this, can_play_or_record] { HandleCanPlayOrRecordChange(can_play_or_record); })); +} + +void AudioDeviceIOS::OnChangedOutputVolume() { + RTC_DCHECK(thread_); + thread_->PostTask(SafeTask(safety_, [this] { HandleOutputVolumeChange(); })); +} + +OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* /* io_data */) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + OSStatus result = noErr; + // Simply return if recording is not enabled. + if (!recording_.load(std::memory_order_acquire)) return result; + + // Set the size of our own audio buffer and clear it first to avoid copying + // in combination with potential reallocations. + // On real iOS devices, the size will only be set once (at first callback). + record_audio_buffer_.Clear(); + record_audio_buffer_.SetSize(num_frames); + + // Allocate AudioBuffers to be used as storage for the received audio. + // The AudioBufferList structure works as a placeholder for the + // AudioBuffer structure, which holds a pointer to the actual data buffer + // in `record_audio_buffer_`. Recorded audio will be rendered into this memory + // at each input callback when calling AudioUnitRender(). + AudioBufferList audio_buffer_list; + audio_buffer_list.mNumberBuffers = 1; + AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; + audio_buffer->mNumberChannels = record_parameters_.channels(); + audio_buffer->mDataByteSize = + record_audio_buffer_.size() * VoiceProcessingAudioUnit::kBytesPerSample; + audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data()); + + // Obtain the recorded audio samples by initiating a rendering cycle. + // Since it happens on the input bus, the `io_data` parameter is a reference + // to the preallocated audio buffer list that the audio unit renders into. + // We can make the audio unit provide a buffer instead in io_data, but we + // currently just use our own. + // TODO(henrika): should error handling be improved? + result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list); + if (result != noErr) { + RTCLogError(@"Failed to render audio."); + return result; + } + + // Get a pointer to the recorded audio and send it to the WebRTC ADB. + // Use the FineAudioBuffer instance to convert between native buffer size + // and the 10ms buffer size used by WebRTC. + fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate); + return noErr; +} + +OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + // Verify 16-bit, noninterleaved mono PCM signal format. + RTC_DCHECK_EQ(1, io_data->mNumberBuffers); + AudioBuffer* audio_buffer = &io_data->mBuffers[0]; + RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels); + + // Produce silence and give audio unit a hint about it if playout is not + // activated. + if (!playing_.load(std::memory_order_acquire)) { + const size_t size_in_bytes = audio_buffer->mDataByteSize; + RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); + *flags |= kAudioUnitRenderAction_OutputIsSilence; + memset(static_cast<int8_t*>(audio_buffer->mData), 0, size_in_bytes); + return noErr; + } + + // Measure time since last call to OnGetPlayoutData() and see if it is larger + // than a well defined threshold which depends on the current IO buffer size. + // If so, we have an indication of a glitch in the output audio since the + // core audio layer will most likely run dry in this state. + ++num_playout_callbacks_; + const int64_t now_time = rtc::TimeMillis(); + if (time_stamp->mSampleTime != num_frames) { + const int64_t delta_time = now_time - last_playout_time_; + const int glitch_threshold = 1.6 * playout_parameters_.GetBufferSizeInMilliseconds(); + if (delta_time > glitch_threshold) { + RTCLogWarning(@"Possible playout audio glitch detected.\n" + " Time since last OnGetPlayoutData was %lld ms.\n", + delta_time); + // Exclude extreme delta values since they do most likely not correspond + // to a real glitch. Instead, the most probable cause is that a headset + // has been plugged in or out. There are more direct ways to detect + // audio device changes (see HandleValidRouteChange()) but experiments + // show that using it leads to more complex implementations. + // TODO(henrika): more tests might be needed to come up with an even + // better upper limit. + if (glitch_threshold < 120 && delta_time > 120) { + RTCLog(@"Glitch warning is ignored. Probably caused by device switch."); + } else { + thread_->PostTask(SafeTask(safety_, [this] { HandlePlayoutGlitchDetected(); })); + } + } + } + last_playout_time_ = now_time; + + // Read decoded 16-bit PCM samples from WebRTC (using a size that matches + // the native I/O audio unit) and copy the result to the audio buffer in the + // `io_data` destination. + fine_audio_buffer_->GetPlayoutData( + rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames), + kFixedPlayoutDelayEstimate); + return noErr; +} + +void AudioDeviceIOS::HandleInterruptionBegin() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_); + if (audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + RTCLog(@"Stopping the audio unit due to interruption begin."); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop the audio unit for interruption begin."); + } + PrepareForNewStart(); + } + is_interrupted_ = true; +} + +void AudioDeviceIOS::HandleInterruptionEnd() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Interruption ended. IsInterrupted changed from %d to 0. " + "Updating audio unit state.", + is_interrupted_); + is_interrupted_ = false; + if (!audio_unit_) return; + if (webrtc::field_trial::IsEnabled("WebRTC-Audio-iOS-Holding")) { + // Work around an issue where audio does not restart properly after an interruption + // by restarting the audio unit when the interruption ends. + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + PrepareForNewStart(); + } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + // Allocate new buffers given the potentially new stream format. + SetupAudioBuffersForActiveAudioSession(); + } + UpdateAudioUnit([RTC_OBJC_TYPE(RTCAudioSession) sharedInstance].canPlayOrRecord); +} + +void AudioDeviceIOS::HandleValidRouteChange() { + RTC_DCHECK_RUN_ON(thread_); + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + RTCLog(@"%@", session); + HandleSampleRateChange(); +} + +void AudioDeviceIOS::HandleCanPlayOrRecordChange(bool can_play_or_record) { + RTCLog(@"Handling CanPlayOrRecord change to: %d", can_play_or_record); + UpdateAudioUnit(can_play_or_record); +} + +void AudioDeviceIOS::HandleSampleRateChange() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Handling sample rate change."); + + // Don't do anything if we're interrupted. + if (is_interrupted_) { + RTCLog(@"Ignoring sample rate change due to interruption."); + return; + } + + // If we don't have an audio unit yet, or the audio unit is uninitialized, + // there is no work to do. + if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { + return; + } + + // The audio unit is already initialized or started. + // Check to see if the sample rate or buffer size has changed. + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + const double new_sample_rate = session.sampleRate; + const NSTimeInterval session_buffer_duration = session.IOBufferDuration; + const size_t new_frames_per_buffer = + static_cast<size_t>(new_sample_rate * session_buffer_duration + .5); + const double current_sample_rate = playout_parameters_.sample_rate(); + const size_t current_frames_per_buffer = playout_parameters_.frames_per_buffer(); + RTCLog(@"Handling playout sample rate change:\n" + " Session sample rate: %f frames_per_buffer: %lu\n" + " ADM sample rate: %f frames_per_buffer: %lu", + new_sample_rate, + (unsigned long)new_frames_per_buffer, + current_sample_rate, + (unsigned long)current_frames_per_buffer); + + // Sample rate and buffer size are the same, no work to do. + if (std::abs(current_sample_rate - new_sample_rate) <= DBL_EPSILON && + current_frames_per_buffer == new_frames_per_buffer) { + RTCLog(@"Ignoring sample rate change since audio parameters are intact."); + return; + } + + // Extra sanity check to ensure that the new sample rate is valid. + if (new_sample_rate <= 0.0) { + RTCLogError(@"Sample rate is invalid: %f", new_sample_rate); + return; + } + + // We need to adjust our format and buffer sizes. + // The stream format is about to be changed and it requires that we first + // stop and uninitialize the audio unit to deallocate its resources. + RTCLog(@"Stopping and uninitializing audio unit to adjust buffers."); + bool restart_audio_unit = false; + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + restart_audio_unit = true; + PrepareForNewStart(); + } + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + + // Allocate new buffers given the new stream format. + SetupAudioBuffersForActiveAudioSession(); + + // Initialize the audio unit again with the new sample rate. + if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %d", + playout_parameters_.sample_rate()); + return; + } + + // Restart the audio unit if it was already running. + if (restart_audio_unit) { + OSStatus result = audio_unit_->Start(); + if (result != noErr) { + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session notifyAudioUnitStartFailedWithError:result]; + RTCLogError(@"Failed to start audio unit with sample rate: %d, reason %d", + playout_parameters_.sample_rate(), + result); + return; + } + } + RTCLog(@"Successfully handled sample rate change."); +} + +void AudioDeviceIOS::HandlePlayoutGlitchDetected() { + RTC_DCHECK_RUN_ON(thread_); + // Don't update metrics if we're interrupted since a "glitch" is expected + // in this state. + if (is_interrupted_) { + RTCLog(@"Ignoring audio glitch due to interruption."); + return; + } + // Avoid doing glitch detection for two seconds after a volume change + // has been detected to reduce the risk of false alarm. + if (last_output_volume_change_time_ > 0 && + rtc::TimeSince(last_output_volume_change_time_) < 2000) { + RTCLog(@"Ignoring audio glitch due to recent output volume change."); + return; + } + num_detected_playout_glitches_++; + RTCLog(@"Number of detected playout glitches: %lld", num_detected_playout_glitches_); + + int64_t glitch_count = num_detected_playout_glitches_; + dispatch_async(dispatch_get_main_queue(), ^{ + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session notifyDidDetectPlayoutGlitch:glitch_count]; + }); +} + +void AudioDeviceIOS::HandleOutputVolumeChange() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Output volume change detected."); + // Store time of this detection so it can be used to defer detection of + // glitches too close in time to this event. + last_output_volume_change_time_ = rtc::TimeMillis(); +} + +void AudioDeviceIOS::UpdateAudioDeviceBuffer() { + LOGI() << "UpdateAudioDevicebuffer"; + // AttachAudioBuffer() is called at construction by the main class but check + // just in case. + RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; + RTC_DCHECK_GT(playout_parameters_.sample_rate(), 0); + RTC_DCHECK_GT(record_parameters_.sample_rate(), 0); + RTC_DCHECK_EQ(playout_parameters_.channels(), 1); + RTC_DCHECK_EQ(record_parameters_.channels(), 1); + // Inform the audio device buffer (ADB) about the new audio format. + audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); + audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); + audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); + audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); +} + +void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { + LOGI() << "SetupAudioBuffersForActiveAudioSession"; + // Verify the current values once the audio session has been activated. + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + double sample_rate = session.sampleRate; + NSTimeInterval io_buffer_duration = session.IOBufferDuration; + RTCLog(@"%@", session); + + // Log a warning message for the case when we are unable to set the preferred + // hardware sample rate but continue and use the non-ideal sample rate after + // reinitializing the audio parameters. Most BT headsets only support 8kHz or + // 16kHz. + RTC_OBJC_TYPE(RTCAudioSessionConfiguration)* webRTCConfig = + [RTC_OBJC_TYPE(RTCAudioSessionConfiguration) webRTCConfiguration]; + if (sample_rate != webRTCConfig.sampleRate) { + RTC_LOG(LS_WARNING) << "Unable to set the preferred sample rate"; + } + + // Crash reports indicates that it can happen in rare cases that the reported + // sample rate is less than or equal to zero. If that happens and if a valid + // sample rate has already been set during initialization, the best guess we + // can do is to reuse the current sample rate. + if (sample_rate <= DBL_EPSILON && playout_parameters_.sample_rate() > 0) { + RTCLogError(@"Reported rate is invalid: %f. " + "Using %d as sample rate instead.", + sample_rate, playout_parameters_.sample_rate()); + sample_rate = playout_parameters_.sample_rate(); + } + + // At this stage, we also know the exact IO buffer duration and can add + // that info to the existing audio parameters where it is converted into + // number of audio frames. + // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. + // Hence, 128 is the size we expect to see in upcoming render callbacks. + playout_parameters_.reset(sample_rate, playout_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(playout_parameters_.is_complete()); + record_parameters_.reset(sample_rate, record_parameters_.channels(), io_buffer_duration); + RTC_DCHECK(record_parameters_.is_complete()); + RTC_LOG(LS_INFO) << " frames per I/O buffer: " << playout_parameters_.frames_per_buffer(); + RTC_LOG(LS_INFO) << " bytes per I/O buffer: " << playout_parameters_.GetBytesPerBuffer(); + RTC_DCHECK_EQ(playout_parameters_.GetBytesPerBuffer(), record_parameters_.GetBytesPerBuffer()); + + // Update the ADB parameters since the sample rate might have changed. + UpdateAudioDeviceBuffer(); + + // Create a modified audio buffer class which allows us to ask for, + // or deliver, any number of samples (and not only multiple of 10ms) to match + // the native audio unit buffer size. + RTC_DCHECK(audio_device_buffer_); + fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_)); +} + +bool AudioDeviceIOS::CreateAudioUnit() { + RTC_DCHECK(!audio_unit_); + + audio_unit_.reset(new VoiceProcessingAudioUnit(bypass_voice_processing_, this)); + if (!audio_unit_->Init()) { + audio_unit_.reset(); + return false; + } + + return true; +} + +void AudioDeviceIOS::UpdateAudioUnit(bool can_play_or_record) { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Updating audio unit state. CanPlayOrRecord=%d IsInterrupted=%d", + can_play_or_record, + is_interrupted_); + + if (is_interrupted_) { + RTCLog(@"Ignoring audio unit update due to interruption."); + return; + } + + // If we're not initialized we don't need to do anything. Audio unit will + // be initialized on initialization. + if (!audio_is_initialized_) return; + + // If we're initialized, we must have an audio unit. + RTC_DCHECK(audio_unit_); + + bool should_initialize_audio_unit = false; + bool should_uninitialize_audio_unit = false; + bool should_start_audio_unit = false; + bool should_stop_audio_unit = false; + + switch (audio_unit_->GetState()) { + case VoiceProcessingAudioUnit::kInitRequired: + RTCLog(@"VPAU state: InitRequired"); + RTC_DCHECK_NOTREACHED(); + break; + case VoiceProcessingAudioUnit::kUninitialized: + RTCLog(@"VPAU state: Uninitialized"); + should_initialize_audio_unit = can_play_or_record; + should_start_audio_unit = + should_initialize_audio_unit && (playing_.load() || recording_.load()); + break; + case VoiceProcessingAudioUnit::kInitialized: + RTCLog(@"VPAU state: Initialized"); + should_start_audio_unit = can_play_or_record && (playing_.load() || recording_.load()); + should_uninitialize_audio_unit = !can_play_or_record; + break; + case VoiceProcessingAudioUnit::kStarted: + RTCLog(@"VPAU state: Started"); + RTC_DCHECK(playing_.load() || recording_.load()); + should_stop_audio_unit = !can_play_or_record; + should_uninitialize_audio_unit = should_stop_audio_unit; + break; + } + + if (should_initialize_audio_unit) { + RTCLog(@"Initializing audio unit for UpdateAudioUnit"); + ConfigureAudioSession(); + SetupAudioBuffersForActiveAudioSession(); + if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + RTCLogError(@"Failed to initialize audio unit."); + return; + } + } + + if (should_start_audio_unit) { + RTCLog(@"Starting audio unit for UpdateAudioUnit"); + // Log session settings before trying to start audio streaming. + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + RTCLog(@"%@", session); + OSStatus result = audio_unit_->Start(); + if (result != noErr) { + [session notifyAudioUnitStartFailedWithError:result]; + RTCLogError(@"Failed to start audio unit, reason %d", result); + return; + } + } + + if (should_stop_audio_unit) { + RTCLog(@"Stopping audio unit for UpdateAudioUnit"); + if (!audio_unit_->Stop()) { + RTCLogError(@"Failed to stop audio unit."); + PrepareForNewStart(); + return; + } + PrepareForNewStart(); + } + + if (should_uninitialize_audio_unit) { + RTCLog(@"Uninitializing audio unit for UpdateAudioUnit"); + audio_unit_->Uninitialize(); + UnconfigureAudioSession(); + } +} + +bool AudioDeviceIOS::ConfigureAudioSession() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Configuring audio session."); + if (has_configured_session_) { + RTCLogWarning(@"Audio session already configured."); + return false; + } + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session lockForConfiguration]; + bool success = [session configureWebRTCSession:nil]; + [session unlockForConfiguration]; + if (success) { + has_configured_session_ = true; + RTCLog(@"Configured audio session."); + } else { + RTCLog(@"Failed to configure audio session."); + } + return success; +} + +bool AudioDeviceIOS::ConfigureAudioSessionLocked() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Configuring audio session."); + if (has_configured_session_) { + RTCLogWarning(@"Audio session already configured."); + return false; + } + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + bool success = [session configureWebRTCSession:nil]; + if (success) { + has_configured_session_ = true; + RTCLog(@"Configured audio session."); + } else { + RTCLog(@"Failed to configure audio session."); + } + return success; +} + +void AudioDeviceIOS::UnconfigureAudioSession() { + RTC_DCHECK_RUN_ON(thread_); + RTCLog(@"Unconfiguring audio session."); + if (!has_configured_session_) { + RTCLogWarning(@"Audio session already unconfigured."); + return; + } + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session lockForConfiguration]; + [session unconfigureWebRTCSession:nil]; + [session endWebRTCSession:nil]; + [session unlockForConfiguration]; + has_configured_session_ = false; + RTCLog(@"Unconfigured audio session."); +} + +bool AudioDeviceIOS::InitPlayOrRecord() { + LOGI() << "InitPlayOrRecord"; + RTC_DCHECK_RUN_ON(thread_); + + // There should be no audio unit at this point. + if (!CreateAudioUnit()) { + return false; + } + + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + // Subscribe to audio session events. + [session pushDelegate:audio_session_observer_]; + is_interrupted_ = session.isInterrupted ? true : false; + + // Lock the session to make configuration changes. + [session lockForConfiguration]; + NSError* error = nil; + if (![session beginWebRTCSession:&error]) { + [session unlockForConfiguration]; + RTCLogError(@"Failed to begin WebRTC session: %@", error.localizedDescription); + audio_unit_.reset(); + return false; + } + + // If we are ready to play or record, and if the audio session can be + // configured, then initialize the audio unit. + if (session.canPlayOrRecord) { + if (!ConfigureAudioSessionLocked()) { + // One possible reason for failure is if an attempt was made to use the + // audio session during or after a Media Services failure. + // See AVAudioSessionErrorCodeMediaServicesFailed for details. + [session unlockForConfiguration]; + audio_unit_.reset(); + return false; + } + SetupAudioBuffersForActiveAudioSession(); + audio_unit_->Initialize(playout_parameters_.sample_rate()); + } + + // Release the lock. + [session unlockForConfiguration]; + return true; +} + +void AudioDeviceIOS::ShutdownPlayOrRecord() { + LOGI() << "ShutdownPlayOrRecord"; + RTC_DCHECK_RUN_ON(thread_); + + // Stop the audio unit to prevent any additional audio callbacks. + audio_unit_->Stop(); + + // Close and delete the voice-processing I/O unit. + audio_unit_.reset(); + + // Detach thread checker for the AURemoteIO::IOThread to ensure that the + // next session uses a fresh thread id. + io_thread_checker_.Detach(); + + // Remove audio session notification observers. + RTC_OBJC_TYPE(RTCAudioSession)* session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session removeDelegate:audio_session_observer_]; + + // All I/O should be stopped or paused prior to deactivating the audio + // session, hence we deactivate as last action. + UnconfigureAudioSession(); +} + +void AudioDeviceIOS::PrepareForNewStart() { + LOGI() << "PrepareForNewStart"; + // The audio unit has been stopped and preparations are needed for an upcoming + // restart. It will result in audio callbacks from a new native I/O thread + // which means that we must detach thread checkers here to be prepared for an + // upcoming new audio stream. + io_thread_checker_.Detach(); +} + +bool AudioDeviceIOS::IsInterrupted() { + return is_interrupted_; +} + +#pragma mark - Not Implemented + +int32_t AudioDeviceIOS::ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const { + audioLayer = AudioDeviceModule::kPlatformDefaultAudio; + return 0; +} + +int16_t AudioDeviceIOS::PlayoutDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int16_t AudioDeviceIOS::RecordingDevices() { + // TODO(henrika): improve. + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return (int16_t)1; +} + +int32_t AudioDeviceIOS::InitSpeaker() { + return 0; +} + +bool AudioDeviceIOS::SpeakerIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::SpeakerVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerVolume(uint32_t volume) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerVolume(uint32_t& volume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxSpeakerVolume(uint32_t& maxVolume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinSpeakerVolume(uint32_t& minVolume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetSpeakerMute(bool enable) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SpeakerMute(bool& enabled) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::InitMicrophone() { + return 0; +} + +bool AudioDeviceIOS::MicrophoneIsInitialized() const { + return true; +} + +int32_t AudioDeviceIOS::MicrophoneMuteIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneMute(bool enable) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneMute(bool& enabled) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecordingIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoRecording(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoRecording(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::StereoPlayoutIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetStereoPlayout(bool enable) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::StereoPlayout(bool& enabled) const { + enabled = false; + return 0; +} + +int32_t AudioDeviceIOS::MicrophoneVolumeIsAvailable(bool& available) { + available = false; + return 0; +} + +int32_t AudioDeviceIOS::SetMicrophoneVolume(uint32_t volume) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MicrophoneVolume(uint32_t& volume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MaxMicrophoneVolume(uint32_t& maxVolume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::MinMicrophoneVolume(uint32_t& minVolume) const { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(uint16_t index) { + RTC_LOG_F(LS_WARNING) << "Not implemented"; + return 0; +} + +int32_t AudioDeviceIOS::SetRecordingDevice(AudioDeviceModule::WindowsDeviceType) { + RTC_DCHECK_NOTREACHED() << "Not implemented"; + return -1; +} + +int32_t AudioDeviceIOS::PlayoutIsAvailable(bool& available) { + available = true; + return 0; +} + +int32_t AudioDeviceIOS::RecordingIsAvailable(bool& available) { + available = true; + return 0; +} + +} // namespace ios_adm +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.h b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.h new file mode 100644 index 0000000000..189d7e6c9c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_ +#define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_ + +#include <memory> + +#include "api/task_queue/task_queue_factory.h" +#include "audio_device_ios.h" +#include "modules/audio_device/audio_device_buffer.h" +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +class AudioDeviceGeneric; + +namespace ios_adm { + +class AudioDeviceModuleIOS : public AudioDeviceModule { + public: + int32_t AttachAudioBuffer(); + + explicit AudioDeviceModuleIOS(bool bypass_voice_processing); + ~AudioDeviceModuleIOS() override; + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; + + // Full-duplex transportation of PCM audio + int32_t RegisterAudioCallback(AudioTransport* audioCallback) override; + + // Main initializaton and termination + int32_t Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice(WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice(WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool* available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool* available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool* available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t* volume) const override; + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t* minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool* available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool* available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool* enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool* available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool* enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool* available) const override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool* enabled) const override; + int32_t StereoRecordingIsAvailable(bool* available) const override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool* enabled) const override; + + // Delay information and control + int32_t PlayoutDelay(uint16_t* delayMS) const override; + + bool BuiltInAECIsAvailable() const override; + int32_t EnableBuiltInAEC(bool enable) override; + bool BuiltInAGCIsAvailable() const override; + int32_t EnableBuiltInAGC(bool enable) override; + bool BuiltInNSIsAvailable() const override; + int32_t EnableBuiltInNS(bool enable) override; + + int32_t GetPlayoutUnderrunCount() const override; + +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; +#endif // WEBRTC_IOS + private: + const bool bypass_voice_processing_; + bool initialized_ = false; + const std::unique_ptr<TaskQueueFactory> task_queue_factory_; + std::unique_ptr<AudioDeviceIOS> audio_device_; + std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_; +}; +} // namespace ios_adm +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_MODULE_IOS_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.mm b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.mm new file mode 100644 index 0000000000..5effef3abd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_device_module_ios.mm @@ -0,0 +1,669 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio_device_module_ios.h" + +#include "api/task_queue/default_task_queue_factory.h" +#include "modules/audio_device/audio_device_config.h" +#include "modules/audio_device/audio_device_generic.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_count.h" +#include "system_wrappers/include/metrics.h" + +#if defined(WEBRTC_IOS) +#include "audio_device_ios.h" +#endif + +#define CHECKinitialized_() \ + { \ + if (!initialized_) { \ + return -1; \ + }; \ + } + +#define CHECKinitialized__BOOL() \ + { \ + if (!initialized_) { \ + return false; \ + }; \ + } + +namespace webrtc { +namespace ios_adm { + +AudioDeviceModuleIOS::AudioDeviceModuleIOS(bool bypass_voice_processing) + : bypass_voice_processing_(bypass_voice_processing), + task_queue_factory_(CreateDefaultTaskQueueFactory()) { + RTC_LOG(LS_INFO) << "current platform is IOS"; + RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized."; +} + + int32_t AudioDeviceModuleIOS::AttachAudioBuffer() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + audio_device_->AttachAudioBuffer(audio_device_buffer_.get()); + return 0; + } + + AudioDeviceModuleIOS::~AudioDeviceModuleIOS() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + } + + int32_t AudioDeviceModuleIOS::ActiveAudioLayer(AudioLayer* audioLayer) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + AudioLayer activeAudio; + if (audio_device_->ActiveAudioLayer(activeAudio) == -1) { + return -1; + } + *audioLayer = activeAudio; + return 0; + } + + int32_t AudioDeviceModuleIOS::Init() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + if (initialized_) + return 0; + + audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_.get())); + audio_device_.reset(new ios_adm::AudioDeviceIOS(bypass_voice_processing_)); + RTC_CHECK(audio_device_); + + this->AttachAudioBuffer(); + + AudioDeviceGeneric::InitStatus status = audio_device_->Init(); + RTC_HISTOGRAM_ENUMERATION( + "WebRTC.Audio.InitializationResult", static_cast<int>(status), + static_cast<int>(AudioDeviceGeneric::InitStatus::NUM_STATUSES)); + if (status != AudioDeviceGeneric::InitStatus::OK) { + RTC_LOG(LS_ERROR) << "Audio device initialization failed."; + return -1; + } + initialized_ = true; + return 0; + } + + int32_t AudioDeviceModuleIOS::Terminate() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + if (!initialized_) + return 0; + if (audio_device_->Terminate() == -1) { + return -1; + } + initialized_ = false; + return 0; + } + + bool AudioDeviceModuleIOS::Initialized() const { + RTC_DLOG(LS_INFO) << __FUNCTION__ << ": " << initialized_; + return initialized_; + } + + int32_t AudioDeviceModuleIOS::InitSpeaker() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->InitSpeaker(); + } + + int32_t AudioDeviceModuleIOS::InitMicrophone() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->InitMicrophone(); + } + + int32_t AudioDeviceModuleIOS::SpeakerVolumeIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->SpeakerVolumeIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetSpeakerVolume(uint32_t volume) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")"; + CHECKinitialized_(); + return audio_device_->SetSpeakerVolume(volume); + } + + int32_t AudioDeviceModuleIOS::SpeakerVolume(uint32_t* volume) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + uint32_t level = 0; + if (audio_device_->SpeakerVolume(level) == -1) { + return -1; + } + *volume = level; + RTC_DLOG(LS_INFO) << "output: " << *volume; + return 0; + } + + bool AudioDeviceModuleIOS::SpeakerIsInitialized() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isInitialized = audio_device_->SpeakerIsInitialized(); + RTC_DLOG(LS_INFO) << "output: " << isInitialized; + return isInitialized; + } + + bool AudioDeviceModuleIOS::MicrophoneIsInitialized() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isInitialized = audio_device_->MicrophoneIsInitialized(); + RTC_DLOG(LS_INFO) << "output: " << isInitialized; + return isInitialized; + } + + int32_t AudioDeviceModuleIOS::MaxSpeakerVolume(uint32_t* maxVolume) const { + CHECKinitialized_(); + uint32_t maxVol = 0; + if (audio_device_->MaxSpeakerVolume(maxVol) == -1) { + return -1; + } + *maxVolume = maxVol; + return 0; + } + + int32_t AudioDeviceModuleIOS::MinSpeakerVolume(uint32_t* minVolume) const { + CHECKinitialized_(); + uint32_t minVol = 0; + if (audio_device_->MinSpeakerVolume(minVol) == -1) { + return -1; + } + *minVolume = minVol; + return 0; + } + + int32_t AudioDeviceModuleIOS::SpeakerMuteIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->SpeakerMuteIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetSpeakerMute(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + return audio_device_->SetSpeakerMute(enable); + } + + int32_t AudioDeviceModuleIOS::SpeakerMute(bool* enabled) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool muted = false; + if (audio_device_->SpeakerMute(muted) == -1) { + return -1; + } + *enabled = muted; + RTC_DLOG(LS_INFO) << "output: " << muted; + return 0; + } + + int32_t AudioDeviceModuleIOS::MicrophoneMuteIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->MicrophoneMuteIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetMicrophoneMute(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + return (audio_device_->SetMicrophoneMute(enable)); + } + + int32_t AudioDeviceModuleIOS::MicrophoneMute(bool* enabled) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool muted = false; + if (audio_device_->MicrophoneMute(muted) == -1) { + return -1; + } + *enabled = muted; + RTC_DLOG(LS_INFO) << "output: " << muted; + return 0; + } + + int32_t AudioDeviceModuleIOS::MicrophoneVolumeIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->MicrophoneVolumeIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetMicrophoneVolume(uint32_t volume) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << volume << ")"; + CHECKinitialized_(); + return (audio_device_->SetMicrophoneVolume(volume)); + } + + int32_t AudioDeviceModuleIOS::MicrophoneVolume(uint32_t* volume) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + uint32_t level = 0; + if (audio_device_->MicrophoneVolume(level) == -1) { + return -1; + } + *volume = level; + RTC_DLOG(LS_INFO) << "output: " << *volume; + return 0; + } + + int32_t AudioDeviceModuleIOS::StereoRecordingIsAvailable( + bool* available) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->StereoRecordingIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetStereoRecording(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + if (enable) { + RTC_LOG(LS_WARNING) << "recording in stereo is not supported"; + } + return -1; + } + + int32_t AudioDeviceModuleIOS::StereoRecording(bool* enabled) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool stereo = false; + if (audio_device_->StereoRecording(stereo) == -1) { + return -1; + } + *enabled = stereo; + RTC_DLOG(LS_INFO) << "output: " << stereo; + return 0; + } + + int32_t AudioDeviceModuleIOS::StereoPlayoutIsAvailable(bool* available) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->StereoPlayoutIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::SetStereoPlayout(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + if (audio_device_->PlayoutIsInitialized()) { + RTC_LOG(LS_ERROR) << "unable to set stereo mode while playing side is initialized"; + return -1; + } + if (audio_device_->SetStereoPlayout(enable)) { + RTC_LOG(LS_WARNING) << "stereo playout is not supported"; + return -1; + } + int8_t nChannels(1); + if (enable) { + nChannels = 2; + } + audio_device_buffer_.get()->SetPlayoutChannels(nChannels); + return 0; + } + + int32_t AudioDeviceModuleIOS::StereoPlayout(bool* enabled) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool stereo = false; + if (audio_device_->StereoPlayout(stereo) == -1) { + return -1; + } + *enabled = stereo; + RTC_DLOG(LS_INFO) << "output: " << stereo; + return 0; + } + + int32_t AudioDeviceModuleIOS::PlayoutIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->PlayoutIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::RecordingIsAvailable(bool* available) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + bool isAvailable = false; + if (audio_device_->RecordingIsAvailable(isAvailable) == -1) { + return -1; + } + *available = isAvailable; + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return 0; + } + + int32_t AudioDeviceModuleIOS::MaxMicrophoneVolume(uint32_t* maxVolume) const { + CHECKinitialized_(); + uint32_t maxVol(0); + if (audio_device_->MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + *maxVolume = maxVol; + return 0; + } + + int32_t AudioDeviceModuleIOS::MinMicrophoneVolume(uint32_t* minVolume) const { + CHECKinitialized_(); + uint32_t minVol(0); + if (audio_device_->MinMicrophoneVolume(minVol) == -1) { + return -1; + } + *minVolume = minVol; + return 0; + } + + int16_t AudioDeviceModuleIOS::PlayoutDevices() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + uint16_t nPlayoutDevices = audio_device_->PlayoutDevices(); + RTC_DLOG(LS_INFO) << "output: " << nPlayoutDevices; + return (int16_t)(nPlayoutDevices); + } + + int32_t AudioDeviceModuleIOS::SetPlayoutDevice(uint16_t index) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")"; + CHECKinitialized_(); + return audio_device_->SetPlayoutDevice(index); + } + + int32_t AudioDeviceModuleIOS::SetPlayoutDevice(WindowsDeviceType device) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->SetPlayoutDevice(device); + } + + int32_t AudioDeviceModuleIOS::PlayoutDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)"; + CHECKinitialized_(); + if (name == NULL) { + return -1; + } + if (audio_device_->PlayoutDeviceName(index, name, guid) == -1) { + return -1; + } + if (name != NULL) { + RTC_DLOG(LS_INFO) << "output: name = " << name; + } + if (guid != NULL) { + RTC_DLOG(LS_INFO) << "output: guid = " << guid; + } + return 0; + } + + int32_t AudioDeviceModuleIOS::RecordingDeviceName( + uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ", ...)"; + CHECKinitialized_(); + if (name == NULL) { + return -1; + } + if (audio_device_->RecordingDeviceName(index, name, guid) == -1) { + return -1; + } + if (name != NULL) { + RTC_DLOG(LS_INFO) << "output: name = " << name; + } + if (guid != NULL) { + RTC_DLOG(LS_INFO) << "output: guid = " << guid; + } + return 0; + } + + int16_t AudioDeviceModuleIOS::RecordingDevices() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + uint16_t nRecordingDevices = audio_device_->RecordingDevices(); + RTC_DLOG(LS_INFO) << "output: " << nRecordingDevices; + return (int16_t)nRecordingDevices; + } + + int32_t AudioDeviceModuleIOS::SetRecordingDevice(uint16_t index) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << index << ")"; + CHECKinitialized_(); + return audio_device_->SetRecordingDevice(index); + } + + int32_t AudioDeviceModuleIOS::SetRecordingDevice(WindowsDeviceType device) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + return audio_device_->SetRecordingDevice(device); + } + + int32_t AudioDeviceModuleIOS::InitPlayout() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + if (PlayoutIsInitialized()) { + return 0; + } + int32_t result = audio_device_->InitPlayout(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitPlayoutSuccess", + static_cast<int>(result == 0)); + return result; + } + + int32_t AudioDeviceModuleIOS::InitRecording() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + if (RecordingIsInitialized()) { + return 0; + } + int32_t result = audio_device_->InitRecording(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.InitRecordingSuccess", + static_cast<int>(result == 0)); + return result; + } + + bool AudioDeviceModuleIOS::PlayoutIsInitialized() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->PlayoutIsInitialized(); + } + + bool AudioDeviceModuleIOS::RecordingIsInitialized() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->RecordingIsInitialized(); + } + + int32_t AudioDeviceModuleIOS::StartPlayout() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + if (Playing()) { + return 0; + } + audio_device_buffer_.get()->StartPlayout(); + int32_t result = audio_device_->StartPlayout(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartPlayoutSuccess", + static_cast<int>(result == 0)); + return result; + } + + int32_t AudioDeviceModuleIOS::StopPlayout() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + int32_t result = audio_device_->StopPlayout(); + audio_device_buffer_.get()->StopPlayout(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopPlayoutSuccess", + static_cast<int>(result == 0)); + return result; + } + + bool AudioDeviceModuleIOS::Playing() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->Playing(); + } + + int32_t AudioDeviceModuleIOS::StartRecording() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + if (Recording()) { + return 0; + } + audio_device_buffer_.get()->StartRecording(); + int32_t result = audio_device_->StartRecording(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess", + static_cast<int>(result == 0)); + return result; + } + + int32_t AudioDeviceModuleIOS::StopRecording() { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized_(); + int32_t result = audio_device_->StopRecording(); + audio_device_buffer_.get()->StopRecording(); + RTC_DLOG(LS_INFO) << "output: " << result; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StopRecordingSuccess", + static_cast<int>(result == 0)); + return result; + } + + bool AudioDeviceModuleIOS::Recording() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + return audio_device_->Recording(); + } + + int32_t AudioDeviceModuleIOS::RegisterAudioCallback( + AudioTransport* audioCallback) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + return audio_device_buffer_.get()->RegisterAudioCallback(audioCallback); + } + + int32_t AudioDeviceModuleIOS::PlayoutDelay(uint16_t* delayMS) const { + CHECKinitialized_(); + uint16_t delay = 0; + if (audio_device_->PlayoutDelay(delay) == -1) { + RTC_LOG(LS_ERROR) << "failed to retrieve the playout delay"; + return -1; + } + *delayMS = delay; + return 0; + } + + bool AudioDeviceModuleIOS::BuiltInAECIsAvailable() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInAECIsAvailable(); + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return isAvailable; + } + + int32_t AudioDeviceModuleIOS::EnableBuiltInAEC(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInAEC(enable); + RTC_DLOG(LS_INFO) << "output: " << ok; + return ok; + } + + bool AudioDeviceModuleIOS::BuiltInAGCIsAvailable() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInAGCIsAvailable(); + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return isAvailable; + } + + int32_t AudioDeviceModuleIOS::EnableBuiltInAGC(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInAGC(enable); + RTC_DLOG(LS_INFO) << "output: " << ok; + return ok; + } + + bool AudioDeviceModuleIOS::BuiltInNSIsAvailable() const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + CHECKinitialized__BOOL(); + bool isAvailable = audio_device_->BuiltInNSIsAvailable(); + RTC_DLOG(LS_INFO) << "output: " << isAvailable; + return isAvailable; + } + + int32_t AudioDeviceModuleIOS::EnableBuiltInNS(bool enable) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << "(" << enable << ")"; + CHECKinitialized_(); + int32_t ok = audio_device_->EnableBuiltInNS(enable); + RTC_DLOG(LS_INFO) << "output: " << ok; + return ok; + } + + int32_t AudioDeviceModuleIOS::GetPlayoutUnderrunCount() const { + // Don't log here, as this method can be called very often. + CHECKinitialized_(); + int32_t ok = audio_device_->GetPlayoutUnderrunCount(); + return ok; + } + +#if defined(WEBRTC_IOS) + int AudioDeviceModuleIOS::GetPlayoutAudioParameters( + AudioParameters* params) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + int r = audio_device_->GetPlayoutAudioParameters(params); + RTC_DLOG(LS_INFO) << "output: " << r; + return r; + } + + int AudioDeviceModuleIOS::GetRecordAudioParameters( + AudioParameters* params) const { + RTC_DLOG(LS_INFO) << __FUNCTION__; + int r = audio_device_->GetRecordAudioParameters(params); + RTC_DLOG(LS_INFO) << "output: " << r; + return r; + } +#endif // WEBRTC_IOS +} +} diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/audio_session_observer.h b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_session_observer.h new file mode 100644 index 0000000000..f7c44c8184 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/audio_session_observer.h @@ -0,0 +1,41 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_SESSION_OBSERVER_H_ +#define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_SESSION_OBSERVER_H_ + +#include "rtc_base/thread.h" + +namespace webrtc { + +// Observer interface for listening to AVAudioSession events. +class AudioSessionObserver { + public: + // Called when audio session interruption begins. + virtual void OnInterruptionBegin() = 0; + + // Called when audio session interruption ends. + virtual void OnInterruptionEnd() = 0; + + // Called when audio route changes. + virtual void OnValidRouteChange() = 0; + + // Called when the ability to play or record changes. + virtual void OnCanPlayOrRecordChange(bool can_play_or_record) = 0; + + virtual void OnChangedOutputVolume() = 0; + + protected: + virtual ~AudioSessionObserver() {} +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_SESSION_OBSERVER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.h b/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.h new file mode 100644 index 0000000000..12464ac897 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_HELPERS_H_ +#define SDK_OBJC_NATIVE_SRC_AUDIO_HELPERS_H_ + +#import <Foundation/Foundation.h> +#include <objc/objc.h> + +#include <string> + +namespace webrtc { +namespace ios { + +bool CheckAndLogError(BOOL success, NSError* error); + +NSString* NSStringFromStdString(const std::string& stdString); +std::string StdStringFromNSString(NSString* nsString); + +// Return thread ID as a string. +std::string GetThreadId(); + +// Return thread ID as string suitable for debug logging. +std::string GetThreadInfo(); + +// Returns [NSThread currentThread] description as string. +// Example: <NSThread: 0x170066d80>{number = 1, name = main} +std::string GetCurrentThreadDescription(); + +#if defined(WEBRTC_IOS) +// Returns the current name of the operating system. +std::string GetSystemName(); + +// Returns the current version of the operating system as a string. +std::string GetSystemVersionAsString(); + +// Returns the version of the operating system in double representation. +// Uses a cached value of the system version. +double GetSystemVersion(); + +// Returns the device type. +// Examples: ”iPhone” and ”iPod touch”. +std::string GetDeviceType(); +#endif // defined(WEBRTC_IOS) + +// Returns a more detailed device name. +// Examples: "iPhone 5s (GSM)" and "iPhone 6 Plus". +std::string GetDeviceName(); + +// Returns the name of the process. Does not uniquely identify the process. +std::string GetProcessName(); + +// Returns the identifier of the process (often called process ID). +int GetProcessID(); + +// Returns a string containing the version of the operating system on which the +// process is executing. The string is string is human readable, localized, and +// is appropriate for displaying to the user. +std::string GetOSVersionString(); + +// Returns the number of processing cores available on the device. +int GetProcessorCount(); + +#if defined(WEBRTC_IOS) +// Indicates whether Low Power Mode is enabled on the iOS device. +bool GetLowPowerModeEnabled(); +#endif + +} // namespace ios +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_HELPERS_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.mm b/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.mm new file mode 100644 index 0000000000..cd0469656a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/helpers.mm @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <sys/sysctl.h> +#if defined(WEBRTC_IOS) +#import <UIKit/UIKit.h> +#endif + +#include <memory> + +#include "helpers.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace ios { + +NSString* NSStringFromStdString(const std::string& stdString) { + // std::string may contain null termination character so we construct + // using length. + return [[NSString alloc] initWithBytes:stdString.data() + length:stdString.length() + encoding:NSUTF8StringEncoding]; +} + +std::string StdStringFromNSString(NSString* nsString) { + NSData* charData = [nsString dataUsingEncoding:NSUTF8StringEncoding]; + return std::string(reinterpret_cast<const char*>([charData bytes]), + [charData length]); +} + +bool CheckAndLogError(BOOL success, NSError* error) { + if (!success) { + NSString* msg = + [NSString stringWithFormat:@"Error: %ld, %@, %@", (long)error.code, + error.localizedDescription, + error.localizedFailureReason]; + RTC_LOG(LS_ERROR) << StdStringFromNSString(msg); + return false; + } + return true; +} + +// TODO(henrika): see if it is possible to move to GetThreadName in +// platform_thread.h and base it on pthread methods instead. +std::string GetCurrentThreadDescription() { + NSString* name = [NSString stringWithFormat:@"%@", [NSThread currentThread]]; + return StdStringFromNSString(name); +} + +#if defined(WEBRTC_IOS) +std::string GetSystemName() { + NSString* osName = [[UIDevice currentDevice] systemName]; + return StdStringFromNSString(osName); +} + +std::string GetSystemVersionAsString() { + NSString* osVersion = [[UIDevice currentDevice] systemVersion]; + return StdStringFromNSString(osVersion); +} + +std::string GetDeviceType() { + NSString* deviceModel = [[UIDevice currentDevice] model]; + return StdStringFromNSString(deviceModel); +} + +bool GetLowPowerModeEnabled() { + return [NSProcessInfo processInfo].lowPowerModeEnabled; +} +#endif + +std::string GetDeviceName() { + size_t size; + sysctlbyname("hw.machine", NULL, &size, NULL, 0); + std::unique_ptr<char[]> machine; + machine.reset(new char[size]); + sysctlbyname("hw.machine", machine.get(), &size, NULL, 0); + return std::string(machine.get()); +} + +std::string GetProcessName() { + NSString* processName = [NSProcessInfo processInfo].processName; + return StdStringFromNSString(processName); +} + +int GetProcessID() { + return [NSProcessInfo processInfo].processIdentifier; +} + +std::string GetOSVersionString() { + NSString* osVersion = + [NSProcessInfo processInfo].operatingSystemVersionString; + return StdStringFromNSString(osVersion); +} + +int GetProcessorCount() { + return [NSProcessInfo processInfo].processorCount; +} + +} // namespace ios +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.h b/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.h new file mode 100644 index 0000000000..ed9dd98568 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.h @@ -0,0 +1,141 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_ +#define SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_ + +#include <AudioUnit/AudioUnit.h> + +namespace webrtc { +namespace ios_adm { + +class VoiceProcessingAudioUnitObserver { + public: + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to signal that recorded audio is available. + virtual OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + // Callback function called on a real-time priority I/O thread from the audio + // unit. This method is used to provide audio samples to the audio unit. + virtual OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* io_action_flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) = 0; + + protected: + ~VoiceProcessingAudioUnitObserver() {} +}; + +// Convenience class to abstract away the management of a Voice Processing +// I/O Audio Unit. The Voice Processing I/O unit has the same characteristics +// as the Remote I/O unit (supports full duplex low-latency audio input and +// output) and adds AEC for for two-way duplex communication. It also adds AGC, +// adjustment of voice-processing quality, and muting. Hence, ideal for +// VoIP applications. +class VoiceProcessingAudioUnit { + public: + VoiceProcessingAudioUnit(bool bypass_voice_processing, + VoiceProcessingAudioUnitObserver* observer); + ~VoiceProcessingAudioUnit(); + + // TODO(tkchin): enum for state and state checking. + enum State : int32_t { + // Init() should be called. + kInitRequired, + // Audio unit created but not initialized. + kUninitialized, + // Initialized but not started. Equivalent to stopped. + kInitialized, + // Initialized and started. + kStarted, + }; + + // Number of bytes per audio sample for 16-bit signed integer representation. + static const UInt32 kBytesPerSample; + + // Initializes this class by creating the underlying audio unit instance. + // Creates a Voice-Processing I/O unit and configures it for full-duplex + // audio. The selected stream format is selected to avoid internal resampling + // and to match the 10ms callback rate for WebRTC as well as possible. + // Does not intialize the audio unit. + bool Init(); + + VoiceProcessingAudioUnit::State GetState() const; + + // Initializes the underlying audio unit with the given sample rate. + bool Initialize(Float64 sample_rate); + + // Starts the underlying audio unit. + OSStatus Start(); + + // Stops the underlying audio unit. + bool Stop(); + + // Uninitializes the underlying audio unit. + bool Uninitialize(); + + // Calls render on the underlying audio unit. + OSStatus Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + private: + // The C API used to set callbacks requires static functions. When these are + // called, they will invoke the relevant instance method by casting + // in_ref_con to VoiceProcessingAudioUnit*. + static OSStatus OnGetPlayoutData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + static OSStatus OnDeliverRecordedData(void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Notifies observer that samples are needed for playback. + OSStatus NotifyGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + // Notifies observer that recorded samples are available for render. + OSStatus NotifyDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Returns the predetermined format with a specific sample rate. See + // implementation file for details on format. + AudioStreamBasicDescription GetFormat(Float64 sample_rate) const; + + // Deletes the underlying audio unit. + void DisposeAudioUnit(); + + const bool bypass_voice_processing_; + VoiceProcessingAudioUnitObserver* observer_; + AudioUnit vpio_unit_; + VoiceProcessingAudioUnit::State state_; +}; +} // namespace ios_adm +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_VOICE_PROCESSING_AUDIO_UNIT_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.mm b/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.mm new file mode 100644 index 0000000000..3905b6857a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/audio/voice_processing_audio_unit.mm @@ -0,0 +1,488 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "voice_processing_audio_unit.h" + +#include "rtc_base/checks.h" +#include "system_wrappers/include/metrics.h" + +#import "base/RTCLogging.h" +#import "sdk/objc/components/audio/RTCAudioSessionConfiguration.h" + +#if !defined(NDEBUG) +static void LogStreamDescription(AudioStreamBasicDescription description) { + char formatIdString[5]; + UInt32 formatId = CFSwapInt32HostToBig(description.mFormatID); + bcopy(&formatId, formatIdString, 4); + formatIdString[4] = '\0'; + RTCLog(@"AudioStreamBasicDescription: {\n" + " mSampleRate: %.2f\n" + " formatIDString: %s\n" + " mFormatFlags: 0x%X\n" + " mBytesPerPacket: %u\n" + " mFramesPerPacket: %u\n" + " mBytesPerFrame: %u\n" + " mChannelsPerFrame: %u\n" + " mBitsPerChannel: %u\n" + " mReserved: %u\n}", + description.mSampleRate, formatIdString, + static_cast<unsigned int>(description.mFormatFlags), + static_cast<unsigned int>(description.mBytesPerPacket), + static_cast<unsigned int>(description.mFramesPerPacket), + static_cast<unsigned int>(description.mBytesPerFrame), + static_cast<unsigned int>(description.mChannelsPerFrame), + static_cast<unsigned int>(description.mBitsPerChannel), + static_cast<unsigned int>(description.mReserved)); +} +#endif + +namespace webrtc { +namespace ios_adm { + +// Calls to AudioUnitInitialize() can fail if called back-to-back on different +// ADM instances. A fall-back solution is to allow multiple sequential calls +// with as small delay between each. This factor sets the max number of allowed +// initialization attempts. +static const int kMaxNumberOfAudioUnitInitializeAttempts = 5; +// A VP I/O unit's bus 1 connects to input hardware (microphone). +static const AudioUnitElement kInputBus = 1; +// A VP I/O unit's bus 0 connects to output hardware (speaker). +static const AudioUnitElement kOutputBus = 0; + +// Returns the automatic gain control (AGC) state on the processed microphone +// signal. Should be on by default for Voice Processing audio units. +static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { + RTC_DCHECK(audio_unit); + UInt32 size = sizeof(*enabled); + OSStatus result = AudioUnitGetProperty(audio_unit, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, + kInputBus, + enabled, + &size); + RTCLog(@"VPIO unit AGC: %u", static_cast<unsigned int>(*enabled)); + return result; +} + +VoiceProcessingAudioUnit::VoiceProcessingAudioUnit(bool bypass_voice_processing, + VoiceProcessingAudioUnitObserver* observer) + : bypass_voice_processing_(bypass_voice_processing), + observer_(observer), + vpio_unit_(nullptr), + state_(kInitRequired) { + RTC_DCHECK(observer); +} + +VoiceProcessingAudioUnit::~VoiceProcessingAudioUnit() { + DisposeAudioUnit(); +} + +const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2; + +bool VoiceProcessingAudioUnit::Init() { + RTC_DCHECK_EQ(state_, kInitRequired); + + // Create an audio component description to identify the Voice Processing + // I/O audio unit. + AudioComponentDescription vpio_unit_description; + vpio_unit_description.componentType = kAudioUnitType_Output; + vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO; + vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple; + vpio_unit_description.componentFlags = 0; + vpio_unit_description.componentFlagsMask = 0; + + // Obtain an audio unit instance given the description. + AudioComponent found_vpio_unit_ref = + AudioComponentFindNext(nullptr, &vpio_unit_description); + + // Create a Voice Processing IO audio unit. + OSStatus result = noErr; + result = AudioComponentInstanceNew(found_vpio_unit_ref, &vpio_unit_); + if (result != noErr) { + vpio_unit_ = nullptr; + RTCLogError(@"AudioComponentInstanceNew failed. Error=%ld.", (long)result); + return false; + } + + // Enable input on the input scope of the input element. + UInt32 enable_input = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enable_input, + sizeof(enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + + // Enable output on the output scope of the output element. + UInt32 enable_output = 1; + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Output, kOutputBus, + &enable_output, sizeof(enable_output)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable output on output scope of output element. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback function that provides audio samples to the audio + // unit. + AURenderCallbackStruct render_callback; + render_callback.inputProc = OnGetPlayoutData; + render_callback.inputProcRefCon = this; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, + kOutputBus, &render_callback, sizeof(render_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the render callback on the output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Disable AU buffer allocation for the recorder, we allocate our own. + // TODO(henrika): not sure that it actually saves resource to make this call. + UInt32 flag = 0; + result = AudioUnitSetProperty( + vpio_unit_, kAudioUnitProperty_ShouldAllocateBuffer, + kAudioUnitScope_Output, kInputBus, &flag, sizeof(flag)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to disable buffer allocation on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Specify the callback to be called by the I/O thread to us when input audio + // is available. The recorded samples can then be obtained by calling the + // AudioUnitRender() method. + AURenderCallbackStruct input_callback; + input_callback.inputProc = OnDeliverRecordedData; + input_callback.inputProcRefCon = this; + result = AudioUnitSetProperty(vpio_unit_, + kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Global, kInputBus, + &input_callback, sizeof(input_callback)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to specify the input callback on the input bus. " + "Error=%ld.", + (long)result); + return false; + } + + state_ = kUninitialized; + return true; +} + +VoiceProcessingAudioUnit::State VoiceProcessingAudioUnit::GetState() const { + return state_; +} + +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); + + OSStatus result = noErr; + AudioStreamBasicDescription format = GetFormat(sample_rate); + UInt32 size = sizeof(format); +#if !defined(NDEBUG) + LogStreamDescription(format); +#endif + + // Set the format on the output scope of the input element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kInputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on output scope of input bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Set the format on the input scope of the output element/bus. + result = + AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kOutputBus, &format, size); + if (result != noErr) { + RTCLogError(@"Failed to set format on input scope of output bus. " + "Error=%ld.", + (long)result); + return false; + } + + // Initialize the Voice Processing I/O unit instance. + // Calls to AudioUnitInitialize() can fail if called back-to-back on + // different ADM instances. The error message in this case is -66635 which is + // undocumented. Tests have shown that calling AudioUnitInitialize a second + // time, after a short sleep, avoids this issue. + // See webrtc:5166 for details. + int failed_initalize_attempts = 0; + result = AudioUnitInitialize(vpio_unit_); + while (result != noErr) { + RTCLogError(@"Failed to initialize the Voice Processing I/O unit. " + "Error=%ld.", + (long)result); + ++failed_initalize_attempts; + if (failed_initalize_attempts == kMaxNumberOfAudioUnitInitializeAttempts) { + // Max number of initialization attempts exceeded, hence abort. + RTCLogError(@"Too many initialization attempts."); + return false; + } + RTCLog(@"Pause 100ms and try audio unit initialization again..."); + [NSThread sleepForTimeInterval:0.1f]; + result = AudioUnitInitialize(vpio_unit_); + } + if (result == noErr) { + RTCLog(@"Voice Processing I/O unit is now initialized."); + } + + if (bypass_voice_processing_) { + // Attempt to disable builtin voice processing. + UInt32 toggle = 1; + result = AudioUnitSetProperty(vpio_unit_, + kAUVoiceIOProperty_BypassVoiceProcessing, + kAudioUnitScope_Global, + kInputBus, + &toggle, + sizeof(toggle)); + if (result == noErr) { + RTCLog(@"Successfully bypassed voice processing."); + } else { + RTCLogError(@"Failed to bypass voice processing. Error=%ld.", (long)result); + } + state_ = kInitialized; + return true; + } + + // AGC should be enabled by default for Voice Processing I/O units but it is + // checked below and enabled explicitly if needed. This scheme is used + // to be absolutely sure that the AGC is enabled since we have seen cases + // where only zeros are recorded and a disabled AGC could be one of the + // reasons why it happens. + int agc_was_enabled_by_default = 0; + UInt32 agc_is_enabled = 0; + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (1st attempt). " + "Error=%ld.", + (long)result); + // Example of error code: kAudioUnitErr_NoConnection (-10876). + // All error codes related to audio units are negative and are therefore + // converted into a postive value to match the UMA APIs. + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result); + } else if (agc_is_enabled) { + // Remember that the AGC was enabled by default. Will be used in UMA. + agc_was_enabled_by_default = 1; + } else { + // AGC was initially disabled => try to enable it explicitly. + UInt32 enable_agc = 1; + result = + AudioUnitSetProperty(vpio_unit_, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, kInputBus, &enable_agc, + sizeof(enable_agc)); + if (result != noErr) { + RTCLogError(@"Failed to enable the built-in AGC. " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.SetAGCStateErrorCode", (-1) * result); + } + result = GetAGCState(vpio_unit_, &agc_is_enabled); + if (result != noErr) { + RTCLogError(@"Failed to get AGC state (2nd attempt). " + "Error=%ld.", + (long)result); + RTC_HISTOGRAM_COUNTS_SPARSE_100000( + "WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result); + } + } + + // Track if the built-in AGC was enabled by default (as it should) or not. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault", + agc_was_enabled_by_default); + RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d", + agc_was_enabled_by_default); + // As a final step, add an UMA histogram for tracking the AGC state. + // At this stage, the AGC should be enabled, and if it is not, more work is + // needed to find out the root cause. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled); + RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u", + static_cast<unsigned int>(agc_is_enabled)); + + state_ = kInitialized; + return true; +} + +OSStatus VoiceProcessingAudioUnit::Start() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Starting audio unit."); + + OSStatus result = AudioOutputUnitStart(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to start audio unit. Error=%ld", (long)result); + return result; + } else { + RTCLog(@"Started audio unit"); + } + state_ = kStarted; + return noErr; +} + +bool VoiceProcessingAudioUnit::Stop() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Stopping audio unit."); + + OSStatus result = AudioOutputUnitStop(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to stop audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Stopped audio unit"); + } + + state_ = kInitialized; + return true; +} + +bool VoiceProcessingAudioUnit::Uninitialize() { + RTC_DCHECK_GE(state_, kUninitialized); + RTCLog(@"Unintializing audio unit."); + + OSStatus result = AudioUnitUninitialize(vpio_unit_); + if (result != noErr) { + RTCLogError(@"Failed to uninitialize audio unit. Error=%ld", (long)result); + return false; + } else { + RTCLog(@"Uninitialized audio unit."); + } + + state_ = kUninitialized; + return true; +} + +OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 output_bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK(vpio_unit_) << "Init() not called."; + + OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp, + output_bus_number, num_frames, io_data); + if (result != noErr) { + RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result); + } + return result; +} + +OSStatus VoiceProcessingAudioUnit::OnGetPlayoutData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast<VoiceProcessingAudioUnit*>(in_ref_con); + return audio_unit->NotifyGetPlayoutData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::OnDeliverRecordedData( + void* in_ref_con, + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + VoiceProcessingAudioUnit* audio_unit = + static_cast<VoiceProcessingAudioUnit*>(in_ref_con); + return audio_unit->NotifyDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyGetPlayoutData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnGetPlayoutData(flags, time_stamp, bus_number, num_frames, + io_data); +} + +OSStatus VoiceProcessingAudioUnit::NotifyDeliverRecordedData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + UInt32 bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + return observer_->OnDeliverRecordedData(flags, time_stamp, bus_number, + num_frames, io_data); +} + +AudioStreamBasicDescription VoiceProcessingAudioUnit::GetFormat( + Float64 sample_rate) const { + // Set the application formats for input and output: + // - use same format in both directions + // - avoid resampling in the I/O unit by using the hardware sample rate + // - linear PCM => noncompressed audio data format with one frame per packet + // - no need to specify interleaving since only mono is supported + AudioStreamBasicDescription format; + RTC_DCHECK_EQ(1, kRTCAudioSessionPreferredNumberOfChannels); + format.mSampleRate = sample_rate; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + format.mBytesPerPacket = kBytesPerSample; + format.mFramesPerPacket = 1; // uncompressed. + format.mBytesPerFrame = kBytesPerSample; + format.mChannelsPerFrame = kRTCAudioSessionPreferredNumberOfChannels; + format.mBitsPerChannel = 8 * kBytesPerSample; + return format; +} + +void VoiceProcessingAudioUnit::DisposeAudioUnit() { + if (vpio_unit_) { + switch (state_) { + case kStarted: + Stop(); + [[fallthrough]]; + case kInitialized: + Uninitialize(); + break; + case kUninitialized: + case kInitRequired: + break; + } + + RTCLog(@"Disposing audio unit."); + OSStatus result = AudioComponentInstanceDispose(vpio_unit_); + if (result != noErr) { + RTCLogError(@"AudioComponentInstanceDispose failed. Error=%ld.", + (long)result); + } + vpio_unit_ = nullptr; + } +} + +} // namespace ios_adm +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/network_monitor_observer.h b/third_party/libwebrtc/sdk/objc/native/src/network_monitor_observer.h new file mode 100644 index 0000000000..7c411a1db1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/network_monitor_observer.h @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_NETWORK_MONITOR_OBSERVER_H_ +#define SDK_OBJC_NATIVE_SRC_NETWORK_MONITOR_OBSERVER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "rtc_base/network_constants.h" +#include "rtc_base/string_utils.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// Observer interface for listening to NWPathMonitor updates. +class NetworkMonitorObserver { + public: + // Called when a path update occurs, on network monitor dispatch queue. + // + // `adapter_type_by_name` is a map from interface name (i.e. "pdp_ip0") to + // adapter type, for all available interfaces on the current path. If an + // interface name isn't present it can be assumed to be unavailable. + virtual void OnPathUpdate( + std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp> + adapter_type_by_name) = 0; + + protected: + virtual ~NetworkMonitorObserver() {} +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_SESSION_OBSERVER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.h b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.h new file mode 100644 index 0000000000..fcfe7a6e8b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.h @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ + +#include <memory> + +#import "components/audio/RTCAudioDevice.h" + +#include "modules/audio_device/audio_device_buffer.h" +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/thread.h" + +@class ObjCAudioDeviceDelegate; + +namespace webrtc { + +class FineAudioBuffer; + +namespace objc_adm { + +class ObjCAudioDeviceModule : public AudioDeviceModule { + public: + explicit ObjCAudioDeviceModule(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device); + ~ObjCAudioDeviceModule() override; + + // Retrieve the currently utilized audio layer + int32_t ActiveAudioLayer(AudioLayer* audioLayer) const override; + + // Full-duplex transportation of PCM audio + int32_t RegisterAudioCallback(AudioTransport* audioCallback) override; + + // Main initialization and termination + int32_t Init() override; + int32_t Terminate() override; + bool Initialized() const override; + + // Device enumeration + int16_t PlayoutDevices() override; + int16_t RecordingDevices() override; + int32_t PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + int32_t RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) override; + + // Device selection + int32_t SetPlayoutDevice(uint16_t index) override; + int32_t SetPlayoutDevice(WindowsDeviceType device) override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t SetRecordingDevice(WindowsDeviceType device) override; + + // Audio transport initialization + int32_t PlayoutIsAvailable(bool* available) override; + int32_t InitPlayout() override; + bool PlayoutIsInitialized() const override; + int32_t RecordingIsAvailable(bool* available) override; + int32_t InitRecording() override; + bool RecordingIsInitialized() const override; + + // Audio transport control + int32_t StartPlayout() override; + int32_t StopPlayout() override; + bool Playing() const override; + int32_t StartRecording() override; + int32_t StopRecording() override; + bool Recording() const override; + + // Audio mixer initialization + int32_t InitSpeaker() override; + bool SpeakerIsInitialized() const override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Speaker volume controls + int32_t SpeakerVolumeIsAvailable(bool* available) override; + int32_t SetSpeakerVolume(uint32_t volume) override; + int32_t SpeakerVolume(uint32_t* volume) const override; + int32_t MaxSpeakerVolume(uint32_t* maxVolume) const override; + int32_t MinSpeakerVolume(uint32_t* minVolume) const override; + + // Microphone volume controls + int32_t MicrophoneVolumeIsAvailable(bool* available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; + + // Speaker mute control + int32_t SpeakerMuteIsAvailable(bool* available) override; + int32_t SetSpeakerMute(bool enable) override; + int32_t SpeakerMute(bool* enabled) const override; + + // Microphone mute control + int32_t MicrophoneMuteIsAvailable(bool* available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool* enabled) const override; + + // Stereo support + int32_t StereoPlayoutIsAvailable(bool* available) const override; + int32_t SetStereoPlayout(bool enable) override; + int32_t StereoPlayout(bool* enabled) const override; + int32_t StereoRecordingIsAvailable(bool* available) const override; + int32_t SetStereoRecording(bool enable) override; + int32_t StereoRecording(bool* enabled) const override; + + // Playout delay + int32_t PlayoutDelay(uint16_t* delayMS) const override; + + // Only supported on Android. + bool BuiltInAECIsAvailable() const override; + bool BuiltInAGCIsAvailable() const override; + bool BuiltInNSIsAvailable() const override; + + // Enables the built-in audio effects. Only supported on Android. + int32_t EnableBuiltInAEC(bool enable) override; + int32_t EnableBuiltInAGC(bool enable) override; + int32_t EnableBuiltInNS(bool enable) override; + + // Play underrun count. Only supported on Android. + int32_t GetPlayoutUnderrunCount() const override; + +#if defined(WEBRTC_IOS) + int GetPlayoutAudioParameters(AudioParameters* params) const override; + int GetRecordAudioParameters(AudioParameters* params) const override; +#endif // WEBRTC_IOS + + public: + OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + NSInteger bus_number, + UInt32 num_frames, + const AudioBufferList* io_data, + void* render_context, + RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) render_block); + + OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + NSInteger bus_number, + UInt32 num_frames, + AudioBufferList* io_data); + + // Notifies `ObjCAudioDeviceModule` that at least one of the audio input + // parameters or audio input latency of `RTCAudioDevice` has changed. It necessary to + // update `record_parameters_` with current audio parameter of `RTCAudioDevice` + // via `UpdateAudioParameters` and if parameters are actually change then + // ADB parameters are updated with `UpdateInputAudioDeviceBuffer`. Audio input latency + // stored in `cached_recording_delay_ms_` is also updated with current latency + // of `RTCAudioDevice`. + void HandleAudioInputParametersChange(); + + // Same as `HandleAudioInputParametersChange` but should be called when audio output + // parameters of `RTCAudioDevice` has changed. + void HandleAudioOutputParametersChange(); + + // Notifies `ObjCAudioDeviceModule` about audio input interruption happen due to + // any reason so `ObjCAudioDeviceModule` is can prepare to restart of audio IO. + void HandleAudioInputInterrupted(); + + // Same as `ObjCAudioDeviceModule` but should be called when audio output + // is interrupted. + void HandleAudioOutputInterrupted(); + + private: + // Update our audio parameters if they are different from current device audio parameters + // Returns true when our parameters are update, false - otherwise. + // `ObjCAudioDeviceModule` has audio device buffer (ADB) which has audio parameters + // of playout & recording. The ADB is configured to work with specific sample rate & channel + // count. `ObjCAudioDeviceModule` stores audio parameters which were used to configure ADB in the + // fields `playout_parameters_` and `recording_parameters_`. + // `RTCAudioDevice` protocol has its own audio parameters exposed as individual properties. + // `RTCAudioDevice` audio parameters might change when playout/recording is already in progress, + // for example, when device is switched. `RTCAudioDevice` audio parameters must be kept in sync + // with ADB audio parameters. This method is invoked when `RTCAudioDevice` reports that it's audio + // parameters (`device_params`) are changed and it detects if there any difference with our + // current audio parameters (`params`). Our parameters are updated in case of actual change and + // method returns true. In case of actual change there is follow-up call to either + // `UpdateOutputAudioDeviceBuffer` or `UpdateInputAudioDeviceBuffer` to apply updated + // `playout_parameters_` or `recording_parameters_` to ADB. + + bool UpdateAudioParameters(AudioParameters& params, const AudioParameters& device_params); + + // Update our cached audio latency with device latency. Device latency is reported by + // `RTCAudioDevice` object. Whenever latency is changed, `RTCAudioDevice` is obliged to notify ADM + // about the change via `HandleAudioInputParametersChange` or `HandleAudioOutputParametersChange`. + // Current device IO latency is cached in the atomic field and used from audio IO thread + // to be reported to audio device buffer. It is highly recommended by Apple not to call any + // ObjC methods from audio IO thread, that is why implementation relies on caching latency + // into a field and being notified when latency is changed, which is the case when device + // is switched. + void UpdateAudioDelay(std::atomic<int>& delay_ms, const NSTimeInterval device_latency); + + // Uses current `playout_parameters_` to inform the audio device buffer (ADB) + // about our internal audio parameters. + void UpdateOutputAudioDeviceBuffer(); + + // Uses current `record_parameters_` to inform the audio device buffer (ADB) + // about our internal audio parameters. + void UpdateInputAudioDeviceBuffer(); + + private: + id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device_; + + const std::unique_ptr<TaskQueueFactory> task_queue_factory_; + + // AudioDeviceBuffer is a buffer to consume audio recorded by `RTCAudioDevice` + // and provide audio to be played via `RTCAudioDevice`. + // Audio PCMs could have different sample rate and channels count, but expected + // to be in 16-bit integer interleaved linear PCM format. + // The current parameters ADB configured to work with is stored in field + // `playout_parameters_` for playout and `record_parameters_` for recording. + // These parameters and ADB must kept in sync with `RTCAudioDevice` audio parameters. + std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_; + + // Set to 1 when recording is active and 0 otherwise. + std::atomic<bool> recording_ = false; + + // Set to 1 when playout is active and 0 otherwise. + std::atomic<bool> playing_ = false; + + // Stores cached value of `RTCAudioDevice outputLatency` to be used from + // audio IO thread. Latency is updated on audio output parameters change. + std::atomic<int> cached_playout_delay_ms_ = 0; + + // Same as `cached_playout_delay_ms_` but for audio input + std::atomic<int> cached_recording_delay_ms_ = 0; + + // Thread that is initialized audio device module. + rtc::Thread* thread_; + + // Ensures that methods are called from the same thread as this object is + // initialized on. + SequenceChecker thread_checker_; + + // I/O audio thread checker. + SequenceChecker io_playout_thread_checker_; + SequenceChecker io_record_thread_checker_; + + bool is_initialized_ RTC_GUARDED_BY(thread_checker_) = false; + bool is_playout_initialized_ RTC_GUARDED_BY(thread_checker_) = false; + bool is_recording_initialized_ RTC_GUARDED_BY(thread_checker_) = false; + + // Contains audio parameters (sample rate, #channels, buffer size etc.) for + // the playout and recording sides. + AudioParameters playout_parameters_; + AudioParameters record_parameters_; + + // `FineAudioBuffer` takes an `AudioDeviceBuffer` which delivers audio data + // in chunks of 10ms. `RTCAudioDevice` might deliver recorded data in + // chunks which are not 10ms long. `FineAudioBuffer` implements adaptation + // from undetermined chunk size to 10ms chunks. + std::unique_ptr<FineAudioBuffer> record_fine_audio_buffer_; + + // Same as `record_fine_audio_buffer_` but for audio output. + std::unique_ptr<FineAudioBuffer> playout_fine_audio_buffer_; + + // Temporary storage for recorded data. + rtc::BufferT<int16_t> record_audio_buffer_; + + // Delegate object provided to RTCAudioDevice during initialization + ObjCAudioDeviceDelegate* audio_device_delegate_; +}; + +} // namespace objc_adm + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.mm new file mode 100644 index 0000000000..d629fae20f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device.mm @@ -0,0 +1,711 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "objc_audio_device.h" +#include "objc_audio_device_delegate.h" + +#import "components/audio/RTCAudioDevice.h" +#include "modules/audio_device/fine_audio_buffer.h" + +#include "api/task_queue/default_task_queue_factory.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/time_utils.h" + +namespace { + +webrtc::AudioParameters RecordParameters(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device) { + const double sample_rate = static_cast<int>([audio_device deviceInputSampleRate]); + const size_t channels = static_cast<size_t>([audio_device inputNumberOfChannels]); + const size_t frames_per_buffer = + static_cast<size_t>(sample_rate * [audio_device inputIOBufferDuration] + .5); + return webrtc::AudioParameters(sample_rate, channels, frames_per_buffer); +} + +webrtc::AudioParameters PlayoutParameters(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device) { + const double sample_rate = static_cast<int>([audio_device deviceOutputSampleRate]); + const size_t channels = static_cast<size_t>([audio_device outputNumberOfChannels]); + const size_t frames_per_buffer = + static_cast<size_t>(sample_rate * [audio_device outputIOBufferDuration] + .5); + return webrtc::AudioParameters(sample_rate, channels, frames_per_buffer); +} + +} // namespace + +namespace webrtc { +namespace objc_adm { + +ObjCAudioDeviceModule::ObjCAudioDeviceModule(id<RTC_OBJC_TYPE(RTCAudioDevice)> audio_device) + : audio_device_(audio_device), task_queue_factory_(CreateDefaultTaskQueueFactory()) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK(audio_device_); + thread_checker_.Detach(); + io_playout_thread_checker_.Detach(); + io_record_thread_checker_.Detach(); +} + +ObjCAudioDeviceModule::~ObjCAudioDeviceModule() { + RTC_DLOG_F(LS_VERBOSE) << ""; +} + +int32_t ObjCAudioDeviceModule::RegisterAudioCallback(AudioTransport* audioCallback) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK(audio_device_buffer_); + return audio_device_buffer_->RegisterAudioCallback(audioCallback); +} + +int32_t ObjCAudioDeviceModule::Init() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (Initialized()) { + RTC_LOG_F(LS_INFO) << "Already initialized"; + return 0; + } + io_playout_thread_checker_.Detach(); + io_record_thread_checker_.Detach(); + + thread_ = rtc::Thread::Current(); + audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer(task_queue_factory_.get())); + + if (![audio_device_ isInitialized]) { + if (audio_device_delegate_ == nil) { + audio_device_delegate_ = [[ObjCAudioDeviceDelegate alloc] + initWithAudioDeviceModule:rtc::scoped_refptr<ObjCAudioDeviceModule>(this) + audioDeviceThread:thread_]; + } + + if (![audio_device_ initializeWithDelegate:audio_device_delegate_]) { + RTC_LOG_F(LS_WARNING) << "Failed to initialize audio device"; + [audio_device_delegate_ resetAudioDeviceModule]; + audio_device_delegate_ = nil; + return -1; + } + } + + playout_parameters_.reset([audio_device_delegate_ preferredOutputSampleRate], 1); + UpdateOutputAudioDeviceBuffer(); + + record_parameters_.reset([audio_device_delegate_ preferredInputSampleRate], 1); + UpdateInputAudioDeviceBuffer(); + + is_initialized_ = true; + + RTC_LOG_F(LS_INFO) << "Did initialize"; + return 0; +} + +int32_t ObjCAudioDeviceModule::Terminate() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (!Initialized()) { + RTC_LOG_F(LS_INFO) << "Not initialized"; + return 0; + } + + if ([audio_device_ isInitialized]) { + if (![audio_device_ terminateDevice]) { + RTC_LOG_F(LS_ERROR) << "Failed to terminate audio device"; + return -1; + } + } + + if (audio_device_delegate_ != nil) { + [audio_device_delegate_ resetAudioDeviceModule]; + audio_device_delegate_ = nil; + } + + is_initialized_ = false; + is_playout_initialized_ = false; + is_recording_initialized_ = false; + thread_ = nullptr; + + RTC_LOG_F(LS_INFO) << "Did terminate"; + return 0; +} + +bool ObjCAudioDeviceModule::Initialized() const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + return is_initialized_ && [audio_device_ isInitialized]; +} + +int32_t ObjCAudioDeviceModule::PlayoutIsAvailable(bool* available) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + *available = Initialized(); + return 0; +} + +bool ObjCAudioDeviceModule::PlayoutIsInitialized() const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + return Initialized() && is_playout_initialized_ && [audio_device_ isPlayoutInitialized]; +} + +int32_t ObjCAudioDeviceModule::InitPlayout() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!Initialized()) { + return -1; + } + if (PlayoutIsInitialized()) { + return 0; + } + RTC_DCHECK(!playing_.load()); + + if (![audio_device_ isPlayoutInitialized]) { + if (![audio_device_ initializePlayout]) { + RTC_LOG_F(LS_ERROR) << "Failed to initialize audio device playout"; + return -1; + } + } + + if (UpdateAudioParameters(playout_parameters_, PlayoutParameters(audio_device_))) { + UpdateOutputAudioDeviceBuffer(); + } + + is_playout_initialized_ = true; + RTC_LOG_F(LS_INFO) << "Did initialize playout"; + return 0; +} + +bool ObjCAudioDeviceModule::Playing() const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + return playing_.load() && [audio_device_ isPlaying]; +} + +int32_t ObjCAudioDeviceModule::StartPlayout() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!PlayoutIsInitialized()) { + return -1; + } + if (Playing()) { + return 0; + } + + audio_device_buffer_->StartPlayout(); + if (playout_fine_audio_buffer_) { + playout_fine_audio_buffer_->ResetPlayout(); + } + if (![audio_device_ startPlayout]) { + RTC_LOG_F(LS_ERROR) << "Failed to start audio device playout"; + return -1; + } + playing_.store(true, std::memory_order_release); + RTC_LOG_F(LS_INFO) << "Did start playout"; + return 0; +} + +int32_t ObjCAudioDeviceModule::StopPlayout() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (![audio_device_ stopPlayout]) { + RTC_LOG_F(LS_WARNING) << "Failed to stop playout"; + return -1; + } + + audio_device_buffer_->StopPlayout(); + playing_.store(false, std::memory_order_release); + RTC_LOG_F(LS_INFO) << "Did stop playout"; + return 0; +} + +int32_t ObjCAudioDeviceModule::PlayoutDelay(uint16_t* delayMS) const { + RTC_DCHECK_RUN_ON(&thread_checker_); + *delayMS = static_cast<uint16_t>(rtc::SafeClamp<int>( + cached_playout_delay_ms_.load(), 0, std::numeric_limits<uint16_t>::max())); + return 0; +} + +int32_t ObjCAudioDeviceModule::RecordingIsAvailable(bool* available) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + *available = Initialized(); + return 0; +} + +bool ObjCAudioDeviceModule::RecordingIsInitialized() const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + return Initialized() && is_recording_initialized_ && [audio_device_ isRecordingInitialized]; +} + +int32_t ObjCAudioDeviceModule::InitRecording() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!Initialized()) { + return -1; + } + if (RecordingIsInitialized()) { + return 0; + } + RTC_DCHECK(!recording_.load()); + + if (![audio_device_ isRecordingInitialized]) { + if (![audio_device_ initializeRecording]) { + RTC_LOG_F(LS_ERROR) << "Failed to initialize audio device recording"; + return -1; + } + } + + if (UpdateAudioParameters(record_parameters_, RecordParameters(audio_device_))) { + UpdateInputAudioDeviceBuffer(); + } + + is_recording_initialized_ = true; + RTC_LOG_F(LS_INFO) << "Did initialize recording"; + return 0; +} + +bool ObjCAudioDeviceModule::Recording() const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + return recording_.load() && [audio_device_ isRecording]; +} + +int32_t ObjCAudioDeviceModule::StartRecording() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!RecordingIsInitialized()) { + return -1; + } + if (Recording()) { + return 0; + } + + audio_device_buffer_->StartRecording(); + if (record_fine_audio_buffer_) { + record_fine_audio_buffer_->ResetRecord(); + } + + if (![audio_device_ startRecording]) { + RTC_LOG_F(LS_ERROR) << "Failed to start audio device recording"; + return -1; + } + recording_.store(true, std::memory_order_release); + RTC_LOG_F(LS_INFO) << "Did start recording"; + return 0; +} + +int32_t ObjCAudioDeviceModule::StopRecording() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (![audio_device_ stopRecording]) { + RTC_LOG_F(LS_WARNING) << "Failed to stop recording"; + return -1; + } + audio_device_buffer_->StopRecording(); + recording_.store(false, std::memory_order_release); + RTC_LOG_F(LS_INFO) << "Did stop recording"; + return 0; +} + +#if defined(WEBRTC_IOS) + +int ObjCAudioDeviceModule::GetPlayoutAudioParameters(AudioParameters* params) const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK(playout_parameters_.is_valid()); + RTC_DCHECK_RUN_ON(&thread_checker_); + *params = playout_parameters_; + return 0; +} + +int ObjCAudioDeviceModule::GetRecordAudioParameters(AudioParameters* params) const { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK(record_parameters_.is_valid()); + RTC_DCHECK_RUN_ON(&thread_checker_); + *params = record_parameters_; + return 0; +} + +#endif // WEBRTC_IOS + +void ObjCAudioDeviceModule::UpdateOutputAudioDeviceBuffer() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; + + RTC_DCHECK_GT(playout_parameters_.sample_rate(), 0); + RTC_DCHECK(playout_parameters_.channels() == 1 || playout_parameters_.channels() == 2); + + audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); + audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); + playout_fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_.get())); +} + +void ObjCAudioDeviceModule::UpdateInputAudioDeviceBuffer() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; + + RTC_DCHECK_GT(record_parameters_.sample_rate(), 0); + RTC_DCHECK(record_parameters_.channels() == 1 || record_parameters_.channels() == 2); + + audio_device_buffer_->SetRecordingSampleRate(record_parameters_.sample_rate()); + audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); + record_fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_.get())); +} + +void ObjCAudioDeviceModule::UpdateAudioDelay(std::atomic<int>& delay_ms, + const NSTimeInterval device_latency) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + int latency_ms = static_cast<int>(rtc::kNumMillisecsPerSec * device_latency); + if (latency_ms <= 0) { + return; + } + const int old_latency_ms = delay_ms.exchange(latency_ms); + if (old_latency_ms != latency_ms) { + RTC_LOG_F(LS_INFO) << "Did change audio IO latency from: " << old_latency_ms + << " ms to: " << latency_ms << " ms"; + } +} + +bool ObjCAudioDeviceModule::UpdateAudioParameters(AudioParameters& params, + const AudioParameters& device_params) { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!device_params.is_complete()) { + RTC_LOG_F(LS_INFO) << "Device params are incomplete: " << device_params.ToString(); + return false; + } + if (params.channels() == device_params.channels() && + params.frames_per_buffer() == device_params.frames_per_buffer() && + params.sample_rate() == device_params.sample_rate()) { + RTC_LOG_F(LS_INFO) << "Device params: " << device_params.ToString() + << " are not different from: " << params.ToString(); + return false; + } + + RTC_LOG_F(LS_INFO) << "Audio params will be changed from: " << params.ToString() + << " to: " << device_params.ToString(); + params.reset( + device_params.sample_rate(), device_params.channels(), device_params.frames_per_buffer()); + return true; +} + +OSStatus ObjCAudioDeviceModule::OnDeliverRecordedData( + AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + NSInteger bus_number, + UInt32 num_frames, + const AudioBufferList* io_data, + void* render_context, + RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) render_block) { + RTC_DCHECK_RUN_ON(&io_record_thread_checker_); + OSStatus result = noErr; + // Simply return if recording is not enabled. + if (!recording_.load()) return result; + + if (io_data != nullptr) { + // AudioBuffer already fullfilled with audio data + RTC_DCHECK_EQ(1, io_data->mNumberBuffers); + const AudioBuffer* audio_buffer = &io_data->mBuffers[0]; + RTC_DCHECK(audio_buffer->mNumberChannels == 1 || audio_buffer->mNumberChannels == 2); + + record_fine_audio_buffer_->DeliverRecordedData( + rtc::ArrayView<const int16_t>(static_cast<int16_t*>(audio_buffer->mData), num_frames), + cached_recording_delay_ms_.load()); + return noErr; + } + RTC_DCHECK(render_block != nullptr) << "Either io_data or render_block must be provided"; + + // Set the size of our own audio buffer and clear it first to avoid copying + // in combination with potential reallocations. + // On real iOS devices, the size will only be set once (at first callback). + const int channels_count = record_parameters_.channels(); + record_audio_buffer_.Clear(); + record_audio_buffer_.SetSize(num_frames * channels_count); + + // Allocate AudioBuffers to be used as storage for the received audio. + // The AudioBufferList structure works as a placeholder for the + // AudioBuffer structure, which holds a pointer to the actual data buffer + // in `record_audio_buffer_`. Recorded audio will be rendered into this memory + // at each input callback when calling `render_block`. + AudioBufferList audio_buffer_list; + audio_buffer_list.mNumberBuffers = 1; + AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0]; + audio_buffer->mNumberChannels = channels_count; + audio_buffer->mDataByteSize = + record_audio_buffer_.size() * sizeof(decltype(record_audio_buffer_)::value_type); + audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data()); + + // Obtain the recorded audio samples by initiating a rendering cycle into own buffer. + result = + render_block(flags, time_stamp, bus_number, num_frames, &audio_buffer_list, render_context); + if (result != noErr) { + RTC_LOG_F(LS_ERROR) << "Failed to render audio: " << result; + return result; + } + + // Get a pointer to the recorded audio and send it to the WebRTC ADB. + // Use the FineAudioBuffer instance to convert between native buffer size + // and the 10ms buffer size used by WebRTC. + record_fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, + cached_recording_delay_ms_.load()); + return noErr; +} + +OSStatus ObjCAudioDeviceModule::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, + const AudioTimeStamp* time_stamp, + NSInteger bus_number, + UInt32 num_frames, + AudioBufferList* io_data) { + RTC_DCHECK_RUN_ON(&io_playout_thread_checker_); + // Verify 16-bit, noninterleaved mono or stereo PCM signal format. + RTC_DCHECK_EQ(1, io_data->mNumberBuffers); + AudioBuffer* audio_buffer = &io_data->mBuffers[0]; + RTC_DCHECK(audio_buffer->mNumberChannels == 1 || audio_buffer->mNumberChannels == 2); + RTC_DCHECK_EQ(audio_buffer->mDataByteSize, + sizeof(int16_t) * num_frames * audio_buffer->mNumberChannels); + + // Produce silence and give player a hint about it if playout is not + // activated. + if (!playing_.load()) { + *flags |= kAudioUnitRenderAction_OutputIsSilence; + memset(static_cast<int8_t*>(audio_buffer->mData), 0, audio_buffer->mDataByteSize); + return noErr; + } + + // Read decoded 16-bit PCM samples from WebRTC into the + // `io_data` destination buffer. + playout_fine_audio_buffer_->GetPlayoutData( + rtc::ArrayView<int16_t>(static_cast<int16_t*>(audio_buffer->mData), + num_frames * audio_buffer->mNumberChannels), + cached_playout_delay_ms_.load()); + + return noErr; +} + +void ObjCAudioDeviceModule::HandleAudioInputInterrupted() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + io_record_thread_checker_.Detach(); +} + +void ObjCAudioDeviceModule::HandleAudioOutputInterrupted() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + io_playout_thread_checker_.Detach(); +} + +void ObjCAudioDeviceModule::HandleAudioInputParametersChange() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (UpdateAudioParameters(record_parameters_, RecordParameters(audio_device_))) { + UpdateInputAudioDeviceBuffer(); + } + + UpdateAudioDelay(cached_recording_delay_ms_, [audio_device_ inputLatency]); +} + +void ObjCAudioDeviceModule::HandleAudioOutputParametersChange() { + RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK_RUN_ON(&thread_checker_); + + if (UpdateAudioParameters(playout_parameters_, PlayoutParameters(audio_device_))) { + UpdateOutputAudioDeviceBuffer(); + } + + UpdateAudioDelay(cached_playout_delay_ms_, [audio_device_ outputLatency]); +} + +#pragma mark - Not implemented/Not relevant methods from AudioDeviceModule + +int32_t ObjCAudioDeviceModule::ActiveAudioLayer(AudioLayer* audioLayer) const { + return -1; +} + +int16_t ObjCAudioDeviceModule::PlayoutDevices() { + return 0; +} + +int16_t ObjCAudioDeviceModule::RecordingDevices() { + return 0; +} + +int32_t ObjCAudioDeviceModule::PlayoutDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + return -1; +} + +int32_t ObjCAudioDeviceModule::RecordingDeviceName(uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]) { + return -1; +} + +int32_t ObjCAudioDeviceModule::SetPlayoutDevice(uint16_t index) { + return 0; +} + +int32_t ObjCAudioDeviceModule::SetPlayoutDevice(WindowsDeviceType device) { + return -1; +} + +int32_t ObjCAudioDeviceModule::SetRecordingDevice(uint16_t index) { + return 0; +} + +int32_t ObjCAudioDeviceModule::SetRecordingDevice(WindowsDeviceType device) { + return -1; +} + +int32_t ObjCAudioDeviceModule::InitSpeaker() { + return 0; +} + +bool ObjCAudioDeviceModule::SpeakerIsInitialized() const { + return true; +} + +int32_t ObjCAudioDeviceModule::InitMicrophone() { + return 0; +} + +bool ObjCAudioDeviceModule::MicrophoneIsInitialized() const { + return true; +} + +int32_t ObjCAudioDeviceModule::SpeakerVolumeIsAvailable(bool* available) { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetSpeakerVolume(uint32_t volume) { + return -1; +} + +int32_t ObjCAudioDeviceModule::SpeakerVolume(uint32_t* volume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MaxSpeakerVolume(uint32_t* maxVolume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MinSpeakerVolume(uint32_t* minVolume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::SpeakerMuteIsAvailable(bool* available) { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetSpeakerMute(bool enable) { + return -1; +} + +int32_t ObjCAudioDeviceModule::SpeakerMute(bool* enabled) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MicrophoneMuteIsAvailable(bool* available) { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetMicrophoneMute(bool enable) { + return -1; +} + +int32_t ObjCAudioDeviceModule::MicrophoneMute(bool* enabled) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MicrophoneVolumeIsAvailable(bool* available) { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetMicrophoneVolume(uint32_t volume) { + return -1; +} + +int32_t ObjCAudioDeviceModule::MicrophoneVolume(uint32_t* volume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MaxMicrophoneVolume(uint32_t* maxVolume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::MinMicrophoneVolume(uint32_t* minVolume) const { + return -1; +} + +int32_t ObjCAudioDeviceModule::StereoPlayoutIsAvailable(bool* available) const { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetStereoPlayout(bool enable) { + return -1; +} + +int32_t ObjCAudioDeviceModule::StereoPlayout(bool* enabled) const { + *enabled = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::StereoRecordingIsAvailable(bool* available) const { + *available = false; + return 0; +} + +int32_t ObjCAudioDeviceModule::SetStereoRecording(bool enable) { + return -1; +} + +int32_t ObjCAudioDeviceModule::StereoRecording(bool* enabled) const { + *enabled = false; + return 0; +} + +bool ObjCAudioDeviceModule::BuiltInAECIsAvailable() const { + return false; +} + +int32_t ObjCAudioDeviceModule::EnableBuiltInAEC(bool enable) { + return 0; +} + +bool ObjCAudioDeviceModule::BuiltInAGCIsAvailable() const { + return false; +} + +int32_t ObjCAudioDeviceModule::EnableBuiltInAGC(bool enable) { + return 0; +} + +bool ObjCAudioDeviceModule::BuiltInNSIsAvailable() const { + return false; +} + +int32_t ObjCAudioDeviceModule::EnableBuiltInNS(bool enable) { + return 0; +} + +int32_t ObjCAudioDeviceModule::GetPlayoutUnderrunCount() const { + return -1; +} + +} // namespace objc_adm + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.h b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.h new file mode 100644 index 0000000000..3af079dad9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_DELEGATE_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_DELEGATE_H_ + +#include "api/scoped_refptr.h" +#include "rtc_base/thread.h" + +#import "components/audio/RTCAudioDevice.h" + +namespace webrtc { +namespace objc_adm { +class ObjCAudioDeviceModule; +} // namespace objc_adm +} // namespace webrtc + +@interface ObjCAudioDeviceDelegate : NSObject <RTC_OBJC_TYPE (RTCAudioDeviceDelegate)> + +- (instancetype)initWithAudioDeviceModule: + (rtc::scoped_refptr<webrtc::objc_adm::ObjCAudioDeviceModule>)audioDeviceModule + audioDeviceThread:(rtc::Thread*)thread; + +- (void)resetAudioDeviceModule; + +@end + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_DELEGATE_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.mm new file mode 100644 index 0000000000..156d6326a4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_audio_device_delegate.mm @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AudioUnit/AudioUnit.h> +#import <Foundation/Foundation.h> + +#import "objc_audio_device.h" +#import "objc_audio_device_delegate.h" + +#include "api/make_ref_counted.h" +#include "api/ref_counted_base.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" + +namespace { + +constexpr double kPreferredInputSampleRate = 48000.0; +constexpr double kPreferredOutputSampleRate = 48000.0; + +// WebRTC processes audio in chunks of 10ms. Preferring 20ms audio chunks +// is a compromize between performance and power consumption. +constexpr NSTimeInterval kPeferredInputIOBufferDuration = 0.02; +constexpr NSTimeInterval kPeferredOutputIOBufferDuration = 0.02; + +class AudioDeviceDelegateImpl final : public rtc::RefCountedNonVirtual<AudioDeviceDelegateImpl> { + public: + AudioDeviceDelegateImpl( + rtc::scoped_refptr<webrtc::objc_adm::ObjCAudioDeviceModule> audio_device_module, + rtc::Thread* thread) + : audio_device_module_(audio_device_module), thread_(thread) { + RTC_DCHECK(audio_device_module_); + RTC_DCHECK(thread_); + } + + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device_module() const { + return audio_device_module_.get(); + } + + rtc::Thread* thread() const { return thread_; } + + void reset_audio_device_module() { audio_device_module_ = nullptr; } + + private: + rtc::scoped_refptr<webrtc::objc_adm::ObjCAudioDeviceModule> audio_device_module_; + rtc::Thread* thread_; +}; + +} // namespace + +@implementation ObjCAudioDeviceDelegate { + rtc::scoped_refptr<AudioDeviceDelegateImpl> impl_; +} + +@synthesize getPlayoutData = getPlayoutData_; + +@synthesize deliverRecordedData = deliverRecordedData_; + +@synthesize preferredInputSampleRate = preferredInputSampleRate_; + +@synthesize preferredInputIOBufferDuration = preferredInputIOBufferDuration_; + +@synthesize preferredOutputSampleRate = preferredOutputSampleRate_; + +@synthesize preferredOutputIOBufferDuration = preferredOutputIOBufferDuration_; + +- (instancetype)initWithAudioDeviceModule: + (rtc::scoped_refptr<webrtc::objc_adm::ObjCAudioDeviceModule>)audioDeviceModule + audioDeviceThread:(rtc::Thread*)thread { + RTC_DCHECK_RUN_ON(thread); + if (self = [super init]) { + impl_ = rtc::make_ref_counted<AudioDeviceDelegateImpl>(audioDeviceModule, thread); + preferredInputSampleRate_ = kPreferredInputSampleRate; + preferredInputIOBufferDuration_ = kPeferredInputIOBufferDuration; + preferredOutputSampleRate_ = kPreferredOutputSampleRate; + preferredOutputIOBufferDuration_ = kPeferredOutputIOBufferDuration; + + rtc::scoped_refptr<AudioDeviceDelegateImpl> playout_delegate = impl_; + getPlayoutData_ = ^OSStatus(AudioUnitRenderActionFlags* _Nonnull actionFlags, + const AudioTimeStamp* _Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + AudioBufferList* _Nonnull outputData) { + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device = + playout_delegate->audio_device_module(); + if (audio_device) { + return audio_device->OnGetPlayoutData( + actionFlags, timestamp, inputBusNumber, frameCount, outputData); + } else { + *actionFlags |= kAudioUnitRenderAction_OutputIsSilence; + RTC_LOG(LS_VERBOSE) << "No alive audio device"; + return noErr; + } + }; + + rtc::scoped_refptr<AudioDeviceDelegateImpl> record_delegate = impl_; + deliverRecordedData_ = + ^OSStatus(AudioUnitRenderActionFlags* _Nonnull actionFlags, + const AudioTimeStamp* _Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + const AudioBufferList* _Nullable inputData, + void* renderContext, + RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) _Nullable renderBlock) { + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device = + record_delegate->audio_device_module(); + if (audio_device) { + return audio_device->OnDeliverRecordedData(actionFlags, + timestamp, + inputBusNumber, + frameCount, + inputData, + renderContext, + renderBlock); + } else { + RTC_LOG(LS_VERBOSE) << "No alive audio device"; + return noErr; + } + }; + } + return self; +} + +- (void)notifyAudioInputParametersChange { + RTC_DCHECK_RUN_ON(impl_->thread()); + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device_module = impl_->audio_device_module(); + if (audio_device_module) { + audio_device_module->HandleAudioInputParametersChange(); + } +} + +- (void)notifyAudioOutputParametersChange { + RTC_DCHECK_RUN_ON(impl_->thread()); + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device_module = impl_->audio_device_module(); + if (audio_device_module) { + audio_device_module->HandleAudioOutputParametersChange(); + } +} + +- (void)notifyAudioInputInterrupted { + RTC_DCHECK_RUN_ON(impl_->thread()); + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device_module = impl_->audio_device_module(); + if (audio_device_module) { + audio_device_module->HandleAudioInputInterrupted(); + } +} + +- (void)notifyAudioOutputInterrupted { + RTC_DCHECK_RUN_ON(impl_->thread()); + webrtc::objc_adm::ObjCAudioDeviceModule* audio_device_module = impl_->audio_device_module(); + if (audio_device_module) { + audio_device_module->HandleAudioOutputInterrupted(); + } +} + +- (void)dispatchAsync:(dispatch_block_t)block { + rtc::Thread* thread = impl_->thread(); + RTC_DCHECK(thread); + thread->PostTask([block] { + @autoreleasepool { + block(); + } + }); +} + +- (void)dispatchSync:(dispatch_block_t)block { + rtc::Thread* thread = impl_->thread(); + RTC_DCHECK(thread); + if (thread->IsCurrent()) { + @autoreleasepool { + block(); + } + } else { + thread->BlockingCall([block] { + @autoreleasepool { + block(); + } + }); + } +} + +- (void)resetAudioDeviceModule { + RTC_DCHECK_RUN_ON(impl_->thread()); + impl_->reset_audio_device_module(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.h b/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.h new file mode 100644 index 0000000000..944690c8bc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.h @@ -0,0 +1,56 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_FRAME_BUFFER_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_FRAME_BUFFER_H_ + +#import <CoreVideo/CoreVideo.h> + +#import "base/RTCMacros.h" + +#include "common_video/include/video_frame_buffer.h" + +@protocol RTC_OBJC_TYPE +(RTCVideoFrameBuffer); + +namespace webrtc { + +class ObjCFrameBuffer : public VideoFrameBuffer { + public: + explicit ObjCFrameBuffer(id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>); + ~ObjCFrameBuffer() override; + + Type type() const override; + + int width() const override; + int height() const override; + + rtc::scoped_refptr<I420BufferInterface> ToI420() override; + rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) override; + + id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> wrapped_frame_buffer() const; + + private: + id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> frame_buffer_; + int width_; + int height_; +}; + +id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> ToObjCVideoFrameBuffer( + const rtc::scoped_refptr<VideoFrameBuffer>& buffer); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_FRAME_BUFFER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.mm new file mode 100644 index 0000000000..00e4b4be85 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_frame_buffer.mm @@ -0,0 +1,107 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_frame_buffer.h" + +#include "api/make_ref_counted.h" +#import "base/RTCVideoFrameBuffer.h" +#import "sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer+Private.h" + +namespace webrtc { + +namespace { + +/** ObjCFrameBuffer that conforms to I420BufferInterface by wrapping RTC_OBJC_TYPE(RTCI420Buffer) */ +class ObjCI420FrameBuffer : public I420BufferInterface { + public: + explicit ObjCI420FrameBuffer(id<RTC_OBJC_TYPE(RTCI420Buffer)> frame_buffer) + : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} + ~ObjCI420FrameBuffer() override {} + + int width() const override { return width_; } + + int height() const override { return height_; } + + const uint8_t* DataY() const override { return frame_buffer_.dataY; } + + const uint8_t* DataU() const override { return frame_buffer_.dataU; } + + const uint8_t* DataV() const override { return frame_buffer_.dataV; } + + int StrideY() const override { return frame_buffer_.strideY; } + + int StrideU() const override { return frame_buffer_.strideU; } + + int StrideV() const override { return frame_buffer_.strideV; } + + private: + id<RTC_OBJC_TYPE(RTCI420Buffer)> frame_buffer_; + int width_; + int height_; +}; + +} // namespace + +ObjCFrameBuffer::ObjCFrameBuffer(id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> frame_buffer) + : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} + +ObjCFrameBuffer::~ObjCFrameBuffer() {} + +VideoFrameBuffer::Type ObjCFrameBuffer::type() const { + return Type::kNative; +} + +int ObjCFrameBuffer::width() const { + return width_; +} + +int ObjCFrameBuffer::height() const { + return height_; +} + +rtc::scoped_refptr<I420BufferInterface> ObjCFrameBuffer::ToI420() { + return rtc::make_ref_counted<ObjCI420FrameBuffer>([frame_buffer_ toI420]); +} + +rtc::scoped_refptr<VideoFrameBuffer> ObjCFrameBuffer::CropAndScale(int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) { + if ([frame_buffer_ respondsToSelector:@selector + (cropAndScaleWith:offsetY:cropWidth:cropHeight:scaleWidth:scaleHeight:)]) { + return rtc::make_ref_counted<ObjCFrameBuffer>([frame_buffer_ cropAndScaleWith:offset_x + offsetY:offset_y + cropWidth:crop_width + cropHeight:crop_height + scaleWidth:scaled_width + scaleHeight:scaled_height]); + } + + // Use the default implementation. + return VideoFrameBuffer::CropAndScale( + offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height); +} + +id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> ObjCFrameBuffer::wrapped_frame_buffer() const { + return frame_buffer_; +} + +id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> ToObjCVideoFrameBuffer( + const rtc::scoped_refptr<VideoFrameBuffer>& buffer) { + if (buffer->type() == VideoFrameBuffer::Type::kNative) { + return static_cast<ObjCFrameBuffer*>(buffer.get())->wrapped_frame_buffer(); + } else { + return [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer->ToI420()]; + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.h b/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.h new file mode 100644 index 0000000000..709e9dfbe5 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.h @@ -0,0 +1,67 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_NETWORK_MONITOR_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_NETWORK_MONITOR_H_ + +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/field_trials_view.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "rtc_base/network_monitor.h" +#include "rtc_base/network_monitor_factory.h" +#include "rtc_base/string_utils.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "sdk/objc/components/network/RTCNetworkMonitor+Private.h" +#include "sdk/objc/native/src/network_monitor_observer.h" + +namespace webrtc { + +class ObjCNetworkMonitorFactory : public rtc::NetworkMonitorFactory { + public: + ObjCNetworkMonitorFactory() = default; + ~ObjCNetworkMonitorFactory() override = default; + + rtc::NetworkMonitorInterface* CreateNetworkMonitor( + const FieldTrialsView& field_trials) override; +}; + +class ObjCNetworkMonitor : public rtc::NetworkMonitorInterface, + public NetworkMonitorObserver { + public: + ObjCNetworkMonitor(); + ~ObjCNetworkMonitor() override; + + void Start() override; + void Stop() override; + + InterfaceInfo GetInterfaceInfo(absl::string_view interface_name) override; + + // NetworkMonitorObserver override. + // Fans out updates to observers on the correct thread. + void OnPathUpdate( + std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp> + adapter_type_by_name) override; + + private: + rtc::Thread* thread_ = nullptr; + bool started_ = false; + std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp> + adapter_type_by_name_ RTC_GUARDED_BY(thread_); + rtc::scoped_refptr<PendingTaskSafetyFlag> safety_flag_; + RTCNetworkMonitor* network_monitor_ = nil; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_NETWORK_MONITOR_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.mm new file mode 100644 index 0000000000..535548c64c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_network_monitor.mm @@ -0,0 +1,95 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_network_monitor.h" +#include "absl/strings/string_view.h" + +#include <algorithm> + +#include "rtc_base/logging.h" +#include "rtc_base/string_utils.h" + +namespace webrtc { + +rtc::NetworkMonitorInterface* ObjCNetworkMonitorFactory::CreateNetworkMonitor( + const FieldTrialsView& field_trials) { + return new ObjCNetworkMonitor(); +} + +ObjCNetworkMonitor::ObjCNetworkMonitor() { + safety_flag_ = PendingTaskSafetyFlag::Create(); +} + +ObjCNetworkMonitor::~ObjCNetworkMonitor() { + [network_monitor_ stop]; + network_monitor_ = nil; +} + +void ObjCNetworkMonitor::Start() { + if (started_) { + return; + } + thread_ = rtc::Thread::Current(); + RTC_DCHECK_RUN_ON(thread_); + safety_flag_->SetAlive(); + network_monitor_ = [[RTCNetworkMonitor alloc] initWithObserver:this]; + if (network_monitor_ == nil) { + RTC_LOG(LS_WARNING) << "Failed to create RTCNetworkMonitor; not available on this OS?"; + } + started_ = true; +} + +void ObjCNetworkMonitor::Stop() { + RTC_DCHECK_RUN_ON(thread_); + if (!started_) { + return; + } + safety_flag_->SetNotAlive(); + [network_monitor_ stop]; + network_monitor_ = nil; + started_ = false; +} + +rtc::NetworkMonitorInterface::InterfaceInfo ObjCNetworkMonitor::GetInterfaceInfo( + absl::string_view interface_name) { + RTC_DCHECK_RUN_ON(thread_); + if (adapter_type_by_name_.empty()) { + // If we have no path update, assume everything's available, because it's + // preferable for WebRTC to try all interfaces rather than none at all. + return { + .adapter_type = rtc::ADAPTER_TYPE_UNKNOWN, + .available = true, + }; + } + auto iter = adapter_type_by_name_.find(interface_name); + if (iter == adapter_type_by_name_.end()) { + return { + .adapter_type = rtc::ADAPTER_TYPE_UNKNOWN, + .available = false, + }; + } + + return { + .adapter_type = iter->second, + .available = true, + }; +} + +void ObjCNetworkMonitor::OnPathUpdate( + std::map<std::string, rtc::AdapterType, rtc::AbslStringViewCmp> adapter_type_by_name) { + RTC_DCHECK(network_monitor_ != nil); + thread_->PostTask(SafeTask(safety_flag_, [this, adapter_type_by_name] { + RTC_DCHECK_RUN_ON(thread_); + adapter_type_by_name_ = adapter_type_by_name; + InvokeNetworksChangedCallback(); + })); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.h b/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.h new file mode 100644 index 0000000000..19c997e503 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.h @@ -0,0 +1,40 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_ + +#import "base/RTCMacros.h" + +#include "api/video_codecs/video_decoder_factory.h" +#include "media/base/codec.h" + +@protocol RTC_OBJC_TYPE +(RTCVideoDecoderFactory); + +namespace webrtc { + +class ObjCVideoDecoderFactory : public VideoDecoderFactory { + public: + explicit ObjCVideoDecoderFactory(id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>); + ~ObjCVideoDecoderFactory() override; + + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> wrapped_decoder_factory() const; + + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder(const SdpVideoFormat& format) override; + + private: + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.mm new file mode 100644 index 0000000000..bf5e898a6d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_decoder_factory.mm @@ -0,0 +1,122 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_video_decoder_factory.h" + +#import "base/RTCMacros.h" +#import "base/RTCVideoDecoder.h" +#import "base/RTCVideoDecoderFactory.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_codec/RTCCodecSpecificInfoH264.h" +#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h" +#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h" +#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoDecoder.h" +#import "sdk/objc/helpers/NSString+StdString.h" + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/native/src/objc_frame_buffer.h" + +namespace webrtc { + +namespace { +class ObjCVideoDecoder : public VideoDecoder { + public: + ObjCVideoDecoder(id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder) + : decoder_(decoder), implementation_name_([decoder implementationName].stdString) {} + + bool Configure(const Settings &settings) override { + return + [decoder_ startDecodeWithNumberOfCores:settings.number_of_cores()] == WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Decode(const EncodedImage &input_image, + int64_t render_time_ms = -1) override { + RTC_OBJC_TYPE(RTCEncodedImage) *encodedImage = + [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] initWithNativeEncodedImage:input_image]; + + return [decoder_ decode:encodedImage + missingFrames:false + codecSpecificInfo:nil + renderTimeMs:render_time_ms]; + } + + int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) override { + [decoder_ setCallback:^(RTC_OBJC_TYPE(RTCVideoFrame) * frame) { + const auto buffer = rtc::make_ref_counted<ObjCFrameBuffer>(frame.buffer); + VideoFrame videoFrame = + VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp((uint32_t)(frame.timeStampNs / rtc::kNumNanosecsPerMicrosec)) + .set_timestamp_ms(0) + .set_rotation((VideoRotation)frame.rotation) + .build(); + videoFrame.set_timestamp(frame.timeStamp); + + callback->Decoded(videoFrame); + }]; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return [decoder_ releaseDecoder]; } + + const char *ImplementationName() const override { return implementation_name_.c_str(); } + + private: + id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder_; + const std::string implementation_name_; +}; +} // namespace + +ObjCVideoDecoderFactory::ObjCVideoDecoderFactory( + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> decoder_factory) + : decoder_factory_(decoder_factory) {} + +ObjCVideoDecoderFactory::~ObjCVideoDecoderFactory() {} + +id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> ObjCVideoDecoderFactory::wrapped_decoder_factory() const { + return decoder_factory_; +} + +std::unique_ptr<VideoDecoder> ObjCVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat &format) { + NSString *codecName = [NSString stringWithUTF8String:format.name.c_str()]; + for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * codecInfo in decoder_factory_.supportedCodecs) { + if ([codecName isEqualToString:codecInfo.name]) { + id<RTC_OBJC_TYPE(RTCVideoDecoder)> decoder = [decoder_factory_ createDecoder:codecInfo]; + + if ([decoder isKindOfClass:[RTC_OBJC_TYPE(RTCWrappedNativeVideoDecoder) class]]) { + return [(RTC_OBJC_TYPE(RTCWrappedNativeVideoDecoder) *)decoder releaseWrappedDecoder]; + } else { + return std::unique_ptr<ObjCVideoDecoder>(new ObjCVideoDecoder(decoder)); + } + } + } + + return nullptr; +} + +std::vector<SdpVideoFormat> ObjCVideoDecoderFactory::GetSupportedFormats() const { + std::vector<SdpVideoFormat> supported_formats; + for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in decoder_factory_.supportedCodecs) { + SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat]; + supported_formats.push_back(format); + } + + return supported_formats; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.h b/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.h new file mode 100644 index 0000000000..85a1e5319d --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.h @@ -0,0 +1,43 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_ + +#import <Foundation/Foundation.h> + +#import "base/RTCMacros.h" + +#include "api/video_codecs/video_encoder_factory.h" + +@protocol RTC_OBJC_TYPE +(RTCVideoEncoderFactory); + +namespace webrtc { + +class ObjCVideoEncoderFactory : public VideoEncoderFactory { + public: + explicit ObjCVideoEncoderFactory(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>); + ~ObjCVideoEncoderFactory() override; + + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> wrapped_encoder_factory() const; + + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::vector<SdpVideoFormat> GetImplementations() const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder(const SdpVideoFormat& format) override; + std::unique_ptr<EncoderSelectorInterface> GetEncoderSelector() const override; + + private: + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> encoder_factory_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.mm new file mode 100644 index 0000000000..d4ea79cc88 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_encoder_factory.mm @@ -0,0 +1,209 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_video_encoder_factory.h" + +#include <string> + +#import "base/RTCMacros.h" +#import "base/RTCVideoEncoder.h" +#import "base/RTCVideoEncoderFactory.h" +#import "components/video_codec/RTCCodecSpecificInfoH264+Private.h" +#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h" +#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h" +#import "sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h" +#import "sdk/objc/api/video_codec/RTCVideoCodecConstants.h" +#import "sdk/objc/api/video_codec/RTCWrappedNativeVideoEncoder.h" +#import "sdk/objc/helpers/NSString+StdString.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "sdk/objc/native/src/objc_video_frame.h" + +namespace webrtc { + +namespace { + +class ObjCVideoEncoder : public VideoEncoder { + public: + ObjCVideoEncoder(id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder) + : encoder_(encoder), implementation_name_([encoder implementationName].stdString) {} + + int32_t InitEncode(const VideoCodec *codec_settings, const Settings &encoder_settings) override { + RTC_OBJC_TYPE(RTCVideoEncoderSettings) *settings = + [[RTC_OBJC_TYPE(RTCVideoEncoderSettings) alloc] initWithNativeVideoCodec:codec_settings]; + return [encoder_ startEncodeWithSettings:settings + numberOfCores:encoder_settings.number_of_cores]; + } + + int32_t RegisterEncodeCompleteCallback(EncodedImageCallback *callback) override { + if (callback) { + [encoder_ setCallback:^BOOL(RTC_OBJC_TYPE(RTCEncodedImage) * _Nonnull frame, + id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)> _Nonnull info) { + EncodedImage encodedImage = [frame nativeEncodedImage]; + + // Handle types that can be converted into one of CodecSpecificInfo's hard coded cases. + CodecSpecificInfo codecSpecificInfo; + if ([info isKindOfClass:[RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) class]]) { + codecSpecificInfo = + [(RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) *)info nativeCodecSpecificInfo]; + } + + EncodedImageCallback::Result res = callback->OnEncodedImage(encodedImage, &codecSpecificInfo); + return res.error == EncodedImageCallback::Result::OK; + }]; + } else { + [encoder_ setCallback:nil]; + } + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return [encoder_ releaseEncoder]; } + + int32_t Encode(const VideoFrame &frame, + const std::vector<VideoFrameType> *frame_types) override { + NSMutableArray<NSNumber *> *rtcFrameTypes = [NSMutableArray array]; + for (size_t i = 0; i < frame_types->size(); ++i) { + [rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))]; + } + + return [encoder_ encode:ToObjCVideoFrame(frame) + codecSpecificInfo:nil + frameTypes:rtcFrameTypes]; + } + + void SetRates(const RateControlParameters ¶meters) override { + const uint32_t bitrate = parameters.bitrate.get_sum_kbps(); + const uint32_t framerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5); + [encoder_ setBitrate:bitrate framerate:framerate]; + } + + VideoEncoder::EncoderInfo GetEncoderInfo() const override { + EncoderInfo info; + info.implementation_name = implementation_name_; + + RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *qp_thresholds = [encoder_ scalingSettings]; + info.scaling_settings = qp_thresholds ? ScalingSettings(qp_thresholds.low, qp_thresholds.high) : + ScalingSettings::kOff; + + info.requested_resolution_alignment = encoder_.resolutionAlignment > 0 ?: 1; + info.apply_alignment_to_all_simulcast_layers = encoder_.applyAlignmentToAllSimulcastLayers; + info.supports_native_handle = encoder_.supportsNativeHandle; + info.is_hardware_accelerated = true; + return info; + } + + private: + id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder_; + const std::string implementation_name_; +}; + +class ObjcVideoEncoderSelector : public VideoEncoderFactory::EncoderSelectorInterface { + public: + ObjcVideoEncoderSelector(id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector) { + selector_ = selector; + } + void OnCurrentEncoder(const SdpVideoFormat &format) override { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithNativeSdpVideoFormat:format]; + [selector_ registerCurrentEncoderInfo:info]; + } + absl::optional<SdpVideoFormat> OnEncoderBroken() override { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_ encoderForBrokenEncoder]; + if (info) { + return [info nativeSdpVideoFormat]; + } + return absl::nullopt; + } + absl::optional<SdpVideoFormat> OnAvailableBitrate(const DataRate &rate) override { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_ encoderForBitrate:rate.kbps<NSInteger>()]; + if (info) { + return [info nativeSdpVideoFormat]; + } + return absl::nullopt; + } + + absl::optional<SdpVideoFormat> OnResolutionChange(const RenderResolution &resolution) override { + if ([selector_ respondsToSelector:@selector(encoderForResolutionChangeBySize:)]) { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_ + encoderForResolutionChangeBySize:CGSizeMake(resolution.Width(), resolution.Height())]; + if (info) { + return [info nativeSdpVideoFormat]; + } + } + return absl::nullopt; + } + + private: + id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector_; +}; + +} // namespace + +ObjCVideoEncoderFactory::ObjCVideoEncoderFactory( + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> encoder_factory) + : encoder_factory_(encoder_factory) {} + +ObjCVideoEncoderFactory::~ObjCVideoEncoderFactory() {} + +id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> ObjCVideoEncoderFactory::wrapped_encoder_factory() const { + return encoder_factory_; +} + +std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetSupportedFormats() const { + std::vector<SdpVideoFormat> supported_formats; + for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in [encoder_factory_ supportedCodecs]) { + SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat]; + supported_formats.push_back(format); + } + + return supported_formats; +} + +std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetImplementations() const { + if ([encoder_factory_ respondsToSelector:@selector(implementations)]) { + std::vector<SdpVideoFormat> supported_formats; + for (RTC_OBJC_TYPE(RTCVideoCodecInfo) * supportedCodec in [encoder_factory_ implementations]) { + SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat]; + supported_formats.push_back(format); + } + return supported_formats; + } + return GetSupportedFormats(); +} + +std::unique_ptr<VideoEncoder> ObjCVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat &format) { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithNativeSdpVideoFormat:format]; + id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder = [encoder_factory_ createEncoder:info]; + if ([encoder isKindOfClass:[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) class]]) { + return [(RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) *)encoder releaseWrappedEncoder]; + } else { + return std::unique_ptr<ObjCVideoEncoder>(new ObjCVideoEncoder(encoder)); + } +} + +std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface> + ObjCVideoEncoderFactory::GetEncoderSelector() const { + if ([encoder_factory_ respondsToSelector:@selector(encoderSelector)]) { + id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector = [encoder_factory_ encoderSelector]; + if (selector) { + return absl::make_unique<ObjcVideoEncoderSelector>(selector); + } + } + return nullptr; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.h b/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.h new file mode 100644 index 0000000000..0781d47ee6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.h @@ -0,0 +1,23 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_FRAME_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_FRAME_H_ + +#include "api/video/video_frame.h" +#import "base/RTCVideoFrame.h" + +namespace webrtc { + +RTC_OBJC_TYPE(RTCVideoFrame) * ToObjCVideoFrame(const VideoFrame& frame); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_FRAME_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.mm new file mode 100644 index 0000000000..2e8ce6153e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_frame.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_video_frame.h" + +#include "rtc_base/time_utils.h" +#include "sdk/objc/native/src/objc_frame_buffer.h" + +namespace webrtc { + +RTC_OBJC_TYPE(RTCVideoFrame) * ToObjCVideoFrame(const VideoFrame &frame) { + RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] + initWithBuffer:ToObjCVideoFrameBuffer(frame.video_frame_buffer()) + rotation:RTCVideoRotation(frame.rotation()) + timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; + videoFrame.timeStamp = frame.timestamp(); + + return videoFrame; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.h b/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.h new file mode 100644 index 0000000000..f9c35eae96 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.h @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_ +#define SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_ + +#import <CoreGraphics/CoreGraphics.h> +#import <Foundation/Foundation.h> + +#import "base/RTCMacros.h" + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" + +@protocol RTC_OBJC_TYPE +(RTCVideoRenderer); + +namespace webrtc { + +class ObjCVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> { + public: + ObjCVideoRenderer(id<RTC_OBJC_TYPE(RTCVideoRenderer)> renderer); + void OnFrame(const VideoFrame& nativeVideoFrame) override; + + private: + id<RTC_OBJC_TYPE(RTCVideoRenderer)> renderer_; + CGSize size_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.mm new file mode 100644 index 0000000000..4a9b647ec3 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_renderer.mm @@ -0,0 +1,38 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_video_renderer.h" + +#import "base/RTCMacros.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoRenderer.h" + +#include "sdk/objc/native/src/objc_video_frame.h" + +namespace webrtc { + +ObjCVideoRenderer::ObjCVideoRenderer(id<RTC_OBJC_TYPE(RTCVideoRenderer)> renderer) + : renderer_(renderer), size_(CGSizeZero) {} + +void ObjCVideoRenderer::OnFrame(const VideoFrame& nativeVideoFrame) { + RTC_OBJC_TYPE(RTCVideoFrame)* videoFrame = ToObjCVideoFrame(nativeVideoFrame); + + CGSize current_size = (videoFrame.rotation % 180 == 0) ? + CGSizeMake(videoFrame.width, videoFrame.height) : + CGSizeMake(videoFrame.height, videoFrame.width); + + if (!CGSizeEqualToSize(size_, current_size)) { + size_ = current_size; + [renderer_ setSize:size_]; + } + [renderer_ renderFrame:videoFrame]; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.h b/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.h new file mode 100644 index 0000000000..19a3d6db43 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_CLASSES_VIDEO_OBJC_VIDEO_TRACK_SOURCE_H_ +#define SDK_OBJC_CLASSES_VIDEO_OBJC_VIDEO_TRACK_SOURCE_H_ + +#import "base/RTCVideoCapturer.h" + +#include "base/RTCMacros.h" +#include "media/base/adapted_video_track_source.h" +#include "rtc_base/timestamp_aligner.h" + +RTC_FWD_DECL_OBJC_CLASS(RTC_OBJC_TYPE(RTCVideoFrame)); + +@interface RTCObjCVideoSourceAdapter : NSObject <RTC_OBJC_TYPE (RTCVideoCapturerDelegate)> +@end + +namespace webrtc { + +class ObjCVideoTrackSource : public rtc::AdaptedVideoTrackSource { + public: + ObjCVideoTrackSource(); + explicit ObjCVideoTrackSource(bool is_screencast); + explicit ObjCVideoTrackSource(RTCObjCVideoSourceAdapter* adapter); + + bool is_screencast() const override; + + // Indicates that the encoder should denoise video before encoding it. + // If it is not set, the default configuration is used which is different + // depending on video codec. + absl::optional<bool> needs_denoising() const override; + + SourceState state() const override; + + bool remote() const override; + + void OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame); + + // Called by RTCVideoSource. + void OnOutputFormatRequest(int width, int height, int fps); + + private: + rtc::VideoBroadcaster broadcaster_; + rtc::TimestampAligner timestamp_aligner_; + + RTCObjCVideoSourceAdapter* adapter_; + bool is_screencast_; +}; + +} // namespace webrtc + +#endif // SDK_OBJC_CLASSES_VIDEO_OBJC_VIDEO_TRACK_SOURCE_H_ diff --git a/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.mm b/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.mm new file mode 100644 index 0000000000..7937e90505 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/native/src/objc_video_track_source.mm @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/native/src/objc_video_track_source.h" + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#include "api/video/i420_buffer.h" +#include "sdk/objc/native/src/objc_frame_buffer.h" + +@interface RTCObjCVideoSourceAdapter () +@property(nonatomic) webrtc::ObjCVideoTrackSource *objCVideoTrackSource; +@end + +@implementation RTCObjCVideoSourceAdapter + +@synthesize objCVideoTrackSource = _objCVideoTrackSource; + +- (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer + didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + _objCVideoTrackSource->OnCapturedFrame(frame); +} + +@end + +namespace webrtc { + +ObjCVideoTrackSource::ObjCVideoTrackSource() : ObjCVideoTrackSource(false) {} + +ObjCVideoTrackSource::ObjCVideoTrackSource(bool is_screencast) + : AdaptedVideoTrackSource(/* required resolution alignment */ 2), + is_screencast_(is_screencast) {} + +ObjCVideoTrackSource::ObjCVideoTrackSource(RTCObjCVideoSourceAdapter *adapter) : adapter_(adapter) { + adapter_.objCVideoTrackSource = this; +} + +bool ObjCVideoTrackSource::is_screencast() const { + return is_screencast_; +} + +absl::optional<bool> ObjCVideoTrackSource::needs_denoising() const { + return false; +} + +MediaSourceInterface::SourceState ObjCVideoTrackSource::state() const { + return SourceState::kLive; +} + +bool ObjCVideoTrackSource::remote() const { + return false; +} + +void ObjCVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) { + cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0); + video_adapter()->OnOutputFormatRequest(format); +} + +void ObjCVideoTrackSource::OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame) { + const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec; + const int64_t translated_timestamp_us = + timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros()); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + if (!AdaptFrame(frame.width, + frame.height, + timestamp_us, + &adapted_width, + &adapted_height, + &crop_width, + &crop_height, + &crop_x, + &crop_y)) { + return; + } + + rtc::scoped_refptr<VideoFrameBuffer> buffer; + if (adapted_width == frame.width && adapted_height == frame.height) { + // No adaption - optimized path. + buffer = rtc::make_ref_counted<ObjCFrameBuffer>(frame.buffer); + } else if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { + // Adapted CVPixelBuffer frame. + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer; + buffer = rtc::make_ref_counted<ObjCFrameBuffer>([[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] + initWithPixelBuffer:rtcPixelBuffer.pixelBuffer + adaptedWidth:adapted_width + adaptedHeight:adapted_height + cropWidth:crop_width + cropHeight:crop_height + cropX:crop_x + rtcPixelBuffer.cropX + cropY:crop_y + rtcPixelBuffer.cropY]); + } else { + // Adapted I420 frame. + // TODO(magjed): Optimize this I420 path. + rtc::scoped_refptr<I420Buffer> i420_buffer = I420Buffer::Create(adapted_width, adapted_height); + buffer = rtc::make_ref_counted<ObjCFrameBuffer>(frame.buffer); + i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height); + buffer = i420_buffer; + } + + // Applying rotation is only supported for legacy reasons and performance is + // not critical here. + VideoRotation rotation = static_cast<VideoRotation>(frame.rotation); + if (apply_rotation() && rotation != kVideoRotation_0) { + buffer = I420Buffer::Rotate(*buffer->ToI420(), rotation); + rotation = kVideoRotation_0; + } + + OnFrame(VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(rotation) + .set_timestamp_us(translated_timestamp_us) + .build()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/sdk/objc/unittests/ObjCVideoTrackSource_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/ObjCVideoTrackSource_xctest.mm new file mode 100644 index 0000000000..4c8bf348f4 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/ObjCVideoTrackSource_xctest.mm @@ -0,0 +1,469 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include "sdk/objc/native/src/objc_video_track_source.h" + +#import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "frame_buffer_helpers.h" + +#include "api/scoped_refptr.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/fake_video_renderer.h" +#include "sdk/objc/native/api/video_frame.h" + +typedef void (^VideoSinkCallback)(RTC_OBJC_TYPE(RTCVideoFrame) *); + +namespace { + +class ObjCCallbackVideoSink : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + ObjCCallbackVideoSink(VideoSinkCallback callback) : callback_(callback) {} + + void OnFrame(const webrtc::VideoFrame &frame) override { + callback_(NativeToObjCVideoFrame(frame)); + } + + private: + VideoSinkCallback callback_; +}; + +} // namespace + +@interface ObjCVideoTrackSourceTests : XCTestCase +@end + +@implementation ObjCVideoTrackSourceTests { + rtc::scoped_refptr<webrtc::ObjCVideoTrackSource> _video_source; +} + +- (void)setUp { + _video_source = rtc::make_ref_counted<webrtc::ObjCVideoTrackSource>(); +} + +- (void)tearDown { + _video_source = NULL; +} + +- (void)testOnCapturedFrameAdaptsFrame { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 1); + XCTAssertEqual(video_renderer->width(), 360); + XCTAssertEqual(video_renderer->height(), 640); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameAdaptsFrameWithAlignment { + // Requesting to adapt 1280x720 to 912x514 gives 639x360 without alignment. The 639 causes issues + // with some hardware encoders (e.g. HEVC) so in this test we verify that the alignment is set and + // respected. + + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); + + _video_source->OnOutputFormatRequest(912, 514, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 1); + XCTAssertEqual(video_renderer->width(), 360); + XCTAssertEqual(video_renderer->height(), 640); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameAdaptationResultsInCommonResolutions { + // Some of the most common resolutions used in the wild are 640x360, 480x270 and 320x180. + // Make sure that we properly scale down to exactly these resolutions. + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 1); + XCTAssertEqual(video_renderer->width(), 360); + XCTAssertEqual(video_renderer->height(), 640); + + _video_source->OnOutputFormatRequest(480, 270, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 2); + XCTAssertEqual(video_renderer->width(), 270); + XCTAssertEqual(video_renderer->height(), 480); + + _video_source->OnOutputFormatRequest(320, 180, 30); + _video_source->OnCapturedFrame(frame); + + XCTAssertEqual(video_renderer->num_rendered_frames(), 3); + XCTAssertEqual(video_renderer->width(), 180); + XCTAssertEqual(video_renderer->height(), 320); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameWithoutAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 360, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(frame.width, outputFrame.width); + XCTAssertEqual(frame.height, outputFrame.height); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(buffer.cropX, outputBuffer.cropX); + XCTAssertEqual(buffer.cropY, outputBuffer.cropY); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameCVPixelBufferNeedsAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 0); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 10); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFramePreAdaptedCVPixelBufferNeedsAdaptation { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + // Create a frame that's already adapted down. + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:640 + adaptedHeight:360 + cropWidth:720 + cropHeight:1280 + cropX:0 + cropY:0]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 480); + XCTAssertEqual(outputFrame.height, 270); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 0); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(outputBuffer.cropWidth, 640); + XCTAssertEqual(outputBuffer.cropHeight, 360); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(480, 270, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFramePreCroppedCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:370 + adaptedHeight:640 + cropWidth:370 + cropHeight:640 + cropX:10 + cropY:0]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 14); + XCTAssertEqual(outputBuffer.cropY, 0); + XCTAssertEqual(outputBuffer.cropWidth, 360); + XCTAssertEqual(outputBuffer.cropHeight, 640); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameSmallerPreCroppedCVPixelBufferNeedsCropping { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:300 + adaptedHeight:640 + cropWidth:300 + cropHeight:640 + cropX:40 + cropY:0]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 300); + XCTAssertEqual(outputFrame.height, 534); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; + XCTAssertEqual(outputBuffer.cropX, 40); + XCTAssertEqual(outputBuffer.cropY, 52); + XCTAssertEqual(outputBuffer.cropWidth, 300); + XCTAssertEqual(outputBuffer.cropHeight, 534); + XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; + CVBufferRelease(pixelBufferRef); +} + +- (void)testOnCapturedFrameI420BufferNeedsAdaptation { + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(720, 1280); + RTC_OBJC_TYPE(RTCI420Buffer) *buffer = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTC_OBJC_TYPE(RTCI420Buffer) *outputBuffer = (RTC_OBJC_TYPE(RTCI420Buffer) *)outputFrame.buffer; + + double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testOnCapturedFrameI420BufferNeedsCropping { + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(380, 640); + RTC_OBJC_TYPE(RTCI420Buffer) *buffer = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; + RTC_OBJC_TYPE(RTCVideoFrame) *frame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; + ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { + XCTAssertEqual(outputFrame.width, 360); + XCTAssertEqual(outputFrame.height, 640); + + RTC_OBJC_TYPE(RTCI420Buffer) *outputBuffer = (RTC_OBJC_TYPE(RTCI420Buffer) *)outputFrame.buffer; + + double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); + XCTAssertGreaterThanOrEqual(psnr, 40); + + [callbackExpectation fulfill]; + }); + + const rtc::VideoSinkWants video_sink_wants; + rtc::VideoSourceInterface<webrtc::VideoFrame> *video_source_interface = _video_source.get(); + video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); + + _video_source->OnOutputFormatRequest(640, 360, 30); + _video_source->OnCapturedFrame(frame); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDeviceModule_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDeviceModule_xctest.mm new file mode 100644 index 0000000000..f7439f39f9 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDeviceModule_xctest.mm @@ -0,0 +1,618 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <XCTest/XCTest.h> + +#include <stdlib.h> + +#if defined(WEBRTC_IOS) +#import "sdk/objc/native/api/audio_device_module.h" +#endif + +#include "api/scoped_refptr.h" + +typedef int32_t(^NeedMorePlayDataBlock)(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms); + +typedef int32_t(^RecordedDataIsAvailableBlock)(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel); + + +// This class implements the AudioTransport API and forwards all methods to the appropriate blocks. +class MockAudioTransport : public webrtc::AudioTransport { +public: + MockAudioTransport() {} + ~MockAudioTransport() override {} + + void expectNeedMorePlayData(NeedMorePlayDataBlock block) { + needMorePlayDataBlock = block; + } + + void expectRecordedDataIsAvailable(RecordedDataIsAvailableBlock block) { + recordedDataIsAvailableBlock = block; + } + + int32_t NeedMorePlayData(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override { + return needMorePlayDataBlock(nSamples, + nBytesPerSample, + nChannels, + samplesPerSec, + audioSamples, + nSamplesOut, + elapsed_time_ms, + ntp_time_ms); + } + + int32_t RecordedDataIsAvailable(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) override { + return recordedDataIsAvailableBlock(audioSamples, + nSamples, + nBytesPerSample, + nChannels, + samplesPerSec, + totalDelayMS, + clockDrift, + currentMicLevel, + keyPressed, + newMicLevel); + } + + void PullRenderData(int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override {} + + private: + NeedMorePlayDataBlock needMorePlayDataBlock; + RecordedDataIsAvailableBlock recordedDataIsAvailableBlock; +}; + +// Number of callbacks (input or output) the tests waits for before we set +// an event indicating that the test was OK. +static const NSUInteger kNumCallbacks = 10; +// Max amount of time we wait for an event to be set while counting callbacks. +static const NSTimeInterval kTestTimeOutInSec = 20.0; +// Number of bits per PCM audio sample. +static const NSUInteger kBitsPerSample = 16; +// Number of bytes per PCM audio sample. +static const NSUInteger kBytesPerSample = kBitsPerSample / 8; +// Average number of audio callbacks per second assuming 10ms packet size. +static const NSUInteger kNumCallbacksPerSecond = 100; +// Play out a test file during this time (unit is in seconds). +static const NSUInteger kFilePlayTimeInSec = 15; +// Run the full-duplex test during this time (unit is in seconds). +// Note that first `kNumIgnoreFirstCallbacks` are ignored. +static const NSUInteger kFullDuplexTimeInSec = 10; +// Wait for the callback sequence to stabilize by ignoring this amount of the +// initial callbacks (avoids initial FIFO access). +// Only used in the RunPlayoutAndRecordingInFullDuplex test. +static const NSUInteger kNumIgnoreFirstCallbacks = 50; + +@interface RTCAudioDeviceModuleTests : XCTestCase { + bool _testEnabled; + rtc::scoped_refptr<webrtc::AudioDeviceModule> audioDeviceModule; + MockAudioTransport mock; +} + +@property(nonatomic, assign) webrtc::AudioParameters playoutParameters; +@property(nonatomic, assign) webrtc::AudioParameters recordParameters; + +@end + +@implementation RTCAudioDeviceModuleTests + +@synthesize playoutParameters; +@synthesize recordParameters; + +- (void)setUp { + [super setUp]; +#if defined(WEBRTC_IOS) && TARGET_OS_SIMULATOR + // TODO(peterhanspers): Reenable these tests on simulator. + // See bugs.webrtc.org/7812 + _testEnabled = false; + if (::getenv("WEBRTC_IOS_RUN_AUDIO_TESTS") != nullptr) { + _testEnabled = true; + } +#else + _testEnabled = true; +#endif + + audioDeviceModule = webrtc::CreateAudioDeviceModule(); + XCTAssertEqual(0, audioDeviceModule->Init()); + XCTAssertEqual(0, audioDeviceModule->GetPlayoutAudioParameters(&playoutParameters)); + XCTAssertEqual(0, audioDeviceModule->GetRecordAudioParameters(&recordParameters)); +} + +- (void)tearDown { + XCTAssertEqual(0, audioDeviceModule->Terminate()); + audioDeviceModule = nullptr; + [super tearDown]; +} + +- (void)startPlayout { + XCTAssertFalse(audioDeviceModule->Playing()); + XCTAssertEqual(0, audioDeviceModule->InitPlayout()); + XCTAssertTrue(audioDeviceModule->PlayoutIsInitialized()); + XCTAssertEqual(0, audioDeviceModule->StartPlayout()); + XCTAssertTrue(audioDeviceModule->Playing()); +} + +- (void)stopPlayout { + XCTAssertEqual(0, audioDeviceModule->StopPlayout()); + XCTAssertFalse(audioDeviceModule->Playing()); +} + +- (void)startRecording{ + XCTAssertFalse(audioDeviceModule->Recording()); + XCTAssertEqual(0, audioDeviceModule->InitRecording()); + XCTAssertTrue(audioDeviceModule->RecordingIsInitialized()); + XCTAssertEqual(0, audioDeviceModule->StartRecording()); + XCTAssertTrue(audioDeviceModule->Recording()); +} + +- (void)stopRecording{ + XCTAssertEqual(0, audioDeviceModule->StopRecording()); + XCTAssertFalse(audioDeviceModule->Recording()); +} + +- (NSURL*)fileURLForSampleRate:(int)sampleRate { + XCTAssertTrue(sampleRate == 48000 || sampleRate == 44100 || sampleRate == 16000); + NSString *filename = [NSString stringWithFormat:@"audio_short%d", sampleRate / 1000]; + NSURL *url = [[NSBundle mainBundle] URLForResource:filename withExtension:@"pcm"]; + XCTAssertNotNil(url); + + return url; +} + +#pragma mark - Tests + +- (void)testConstructDestruct { + XCTSkipIf(!_testEnabled); + // Using the test fixture to create and destruct the audio device module. +} + +- (void)testInitTerminate { + XCTSkipIf(!_testEnabled); + // Initialization is part of the test fixture. + XCTAssertTrue(audioDeviceModule->Initialized()); + XCTAssertEqual(0, audioDeviceModule->Terminate()); + XCTAssertFalse(audioDeviceModule->Initialized()); +} + +// Tests that playout can be initiated, started and stopped. No audio callback +// is registered in this test. +- (void)testStartStopPlayout { + XCTSkipIf(!_testEnabled); + [self startPlayout]; + [self stopPlayout]; + [self startPlayout]; + [self stopPlayout]; +} + +// Tests that recording can be initiated, started and stopped. No audio callback +// is registered in this test. +- (void)testStartStopRecording { + XCTSkipIf(!_testEnabled); + [self startRecording]; + [self stopRecording]; + [self startRecording]; + [self stopRecording]; +} +// Verify that calling StopPlayout() will leave us in an uninitialized state +// which will require a new call to InitPlayout(). This test does not call +// StartPlayout() while being uninitialized since doing so will hit a +// RTC_DCHECK. +- (void)testStopPlayoutRequiresInitToRestart { + XCTSkipIf(!_testEnabled); + XCTAssertEqual(0, audioDeviceModule->InitPlayout()); + XCTAssertEqual(0, audioDeviceModule->StartPlayout()); + XCTAssertEqual(0, audioDeviceModule->StopPlayout()); + XCTAssertFalse(audioDeviceModule->PlayoutIsInitialized()); +} + +// Verify that we can create two ADMs and start playing on the second ADM. +// Only the first active instance shall activate an audio session and the +// last active instance shall deactivate the audio session. The test does not +// explicitly verify correct audio session calls but instead focuses on +// ensuring that audio starts for both ADMs. +- (void)testStartPlayoutOnTwoInstances { + XCTSkipIf(!_testEnabled); + // Create and initialize a second/extra ADM instance. The default ADM is + // created by the test harness. + rtc::scoped_refptr<webrtc::AudioDeviceModule> secondAudioDeviceModule = + webrtc::CreateAudioDeviceModule(); + XCTAssertNotEqual(secondAudioDeviceModule.get(), nullptr); + XCTAssertEqual(0, secondAudioDeviceModule->Init()); + + // Start playout for the default ADM but don't wait here. Instead use the + // upcoming second stream for that. We set the same expectation on number + // of callbacks as for the second stream. + mock.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + nSamplesOut = nSamples; + XCTAssertEqual(nSamples, self.playoutParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.playoutParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.playoutParameters.sample_rate()); + XCTAssertNotEqual((void*)NULL, audioSamples); + + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + [self startPlayout]; + + // Initialize playout for the second ADM. If all is OK, the second ADM shall + // reuse the audio session activated when the first ADM started playing. + // This call will also ensure that we avoid a problem related to initializing + // two different audio unit instances back to back (see webrtc:5166 for + // details). + XCTAssertEqual(0, secondAudioDeviceModule->InitPlayout()); + XCTAssertTrue(secondAudioDeviceModule->PlayoutIsInitialized()); + + // Start playout for the second ADM and verify that it starts as intended. + // Passing this test ensures that initialization of the second audio unit + // has been done successfully and that there is no conflict with the already + // playing first ADM. + XCTestExpectation *playoutExpectation = [self expectationWithDescription:@"NeedMorePlayoutData"]; + __block int num_callbacks = 0; + + MockAudioTransport mock2; + mock2.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + nSamplesOut = nSamples; + XCTAssertEqual(nSamples, self.playoutParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.playoutParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.playoutParameters.sample_rate()); + XCTAssertNotEqual((void*)NULL, audioSamples); + if (++num_callbacks == kNumCallbacks) { + [playoutExpectation fulfill]; + } + + return 0; + }); + + XCTAssertEqual(0, secondAudioDeviceModule->RegisterAudioCallback(&mock2)); + XCTAssertEqual(0, secondAudioDeviceModule->StartPlayout()); + XCTAssertTrue(secondAudioDeviceModule->Playing()); + [self waitForExpectationsWithTimeout:kTestTimeOutInSec handler:nil]; + [self stopPlayout]; + XCTAssertEqual(0, secondAudioDeviceModule->StopPlayout()); + XCTAssertFalse(secondAudioDeviceModule->Playing()); + XCTAssertFalse(secondAudioDeviceModule->PlayoutIsInitialized()); + + XCTAssertEqual(0, secondAudioDeviceModule->Terminate()); +} + +// Start playout and verify that the native audio layer starts asking for real +// audio samples to play out using the NeedMorePlayData callback. +- (void)testStartPlayoutVerifyCallbacks { + XCTSkipIf(!_testEnabled); + XCTestExpectation *playoutExpectation = [self expectationWithDescription:@"NeedMorePlayoutData"]; + __block int num_callbacks = 0; + mock.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + nSamplesOut = nSamples; + XCTAssertEqual(nSamples, self.playoutParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.playoutParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.playoutParameters.sample_rate()); + XCTAssertNotEqual((void*)NULL, audioSamples); + if (++num_callbacks == kNumCallbacks) { + [playoutExpectation fulfill]; + } + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + + [self startPlayout]; + [self waitForExpectationsWithTimeout:kTestTimeOutInSec handler:nil]; + [self stopPlayout]; +} + +// Start recording and verify that the native audio layer starts feeding real +// audio samples via the RecordedDataIsAvailable callback. +- (void)testStartRecordingVerifyCallbacks { + XCTSkipIf(!_testEnabled); + XCTestExpectation *recordExpectation = + [self expectationWithDescription:@"RecordedDataIsAvailable"]; + __block int num_callbacks = 0; + + mock.expectRecordedDataIsAvailable(^(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + XCTAssertNotEqual((void*)NULL, audioSamples); + XCTAssertEqual(nSamples, self.recordParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.recordParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.recordParameters.sample_rate()); + XCTAssertEqual(0, clockDrift); + XCTAssertEqual(0u, currentMicLevel); + XCTAssertFalse(keyPressed); + if (++num_callbacks == kNumCallbacks) { + [recordExpectation fulfill]; + } + + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + [self startRecording]; + [self waitForExpectationsWithTimeout:kTestTimeOutInSec handler:nil]; + [self stopRecording]; +} + +// Start playout and recording (full-duplex audio) and verify that audio is +// active in both directions. +- (void)testStartPlayoutAndRecordingVerifyCallbacks { + XCTSkipIf(!_testEnabled); + XCTestExpectation *playoutExpectation = [self expectationWithDescription:@"NeedMorePlayoutData"]; + __block NSUInteger callbackCount = 0; + + XCTestExpectation *recordExpectation = + [self expectationWithDescription:@"RecordedDataIsAvailable"]; + recordExpectation.expectedFulfillmentCount = kNumCallbacks; + + mock.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + nSamplesOut = nSamples; + XCTAssertEqual(nSamples, self.playoutParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.playoutParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.playoutParameters.sample_rate()); + XCTAssertNotEqual((void*)NULL, audioSamples); + if (callbackCount++ >= kNumCallbacks) { + [playoutExpectation fulfill]; + } + + return 0; + }); + + mock.expectRecordedDataIsAvailable(^(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + XCTAssertNotEqual((void*)NULL, audioSamples); + XCTAssertEqual(nSamples, self.recordParameters.frames_per_10ms_buffer()); + XCTAssertEqual(nBytesPerSample, kBytesPerSample); + XCTAssertEqual(nChannels, self.recordParameters.channels()); + XCTAssertEqual((int)samplesPerSec, self.recordParameters.sample_rate()); + XCTAssertEqual(0, clockDrift); + XCTAssertEqual(0u, currentMicLevel); + XCTAssertFalse(keyPressed); + [recordExpectation fulfill]; + + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + [self startPlayout]; + [self startRecording]; + [self waitForExpectationsWithTimeout:kTestTimeOutInSec handler:nil]; + [self stopRecording]; + [self stopPlayout]; +} + +// Start playout and read audio from an external PCM file when the audio layer +// asks for data to play out. Real audio is played out in this test but it does +// not contain any explicit verification that the audio quality is perfect. +- (void)testRunPlayoutWithFileAsSource { + XCTSkipIf(!_testEnabled); + XCTAssertEqual(1u, playoutParameters.channels()); + + // Using XCTestExpectation to count callbacks is very slow. + XCTestExpectation *playoutExpectation = [self expectationWithDescription:@"NeedMorePlayoutData"]; + const int expectedCallbackCount = kFilePlayTimeInSec * kNumCallbacksPerSecond; + __block int callbackCount = 0; + + NSURL *fileURL = [self fileURLForSampleRate:playoutParameters.sample_rate()]; + NSInputStream *inputStream = [[NSInputStream alloc] initWithURL:fileURL]; + + mock.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + [inputStream read:(uint8_t *)audioSamples maxLength:nSamples*nBytesPerSample*nChannels]; + nSamplesOut = nSamples; + if (callbackCount++ == expectedCallbackCount) { + [playoutExpectation fulfill]; + } + + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + [self startPlayout]; + NSTimeInterval waitTimeout = kFilePlayTimeInSec * 2.0; + [self waitForExpectationsWithTimeout:waitTimeout handler:nil]; + [self stopPlayout]; +} + +- (void)testDevices { + XCTSkipIf(!_testEnabled); + // Device enumeration is not supported. Verify fixed values only. + XCTAssertEqual(1, audioDeviceModule->PlayoutDevices()); + XCTAssertEqual(1, audioDeviceModule->RecordingDevices()); +} + +// Start playout and recording and store recorded data in an intermediate FIFO +// buffer from which the playout side then reads its samples in the same order +// as they were stored. Under ideal circumstances, a callback sequence would +// look like: ...+-+-+-+-+-+-+-..., where '+' means 'packet recorded' and '-' +// means 'packet played'. Under such conditions, the FIFO would only contain +// one packet on average. However, under more realistic conditions, the size +// of the FIFO will vary more due to an unbalance between the two sides. +// This test tries to verify that the device maintains a balanced callback- +// sequence by running in loopback for ten seconds while measuring the size +// (max and average) of the FIFO. The size of the FIFO is increased by the +// recording side and decreased by the playout side. +// TODO(henrika): tune the final test parameters after running tests on several +// different devices. +- (void)testRunPlayoutAndRecordingInFullDuplex { + XCTSkipIf(!_testEnabled); + XCTAssertEqual(recordParameters.channels(), playoutParameters.channels()); + XCTAssertEqual(recordParameters.sample_rate(), playoutParameters.sample_rate()); + + XCTestExpectation *playoutExpectation = [self expectationWithDescription:@"NeedMorePlayoutData"]; + __block NSUInteger playoutCallbacks = 0; + NSUInteger expectedPlayoutCallbacks = kFullDuplexTimeInSec * kNumCallbacksPerSecond; + + // FIFO queue and measurements + NSMutableArray *fifoBuffer = [NSMutableArray arrayWithCapacity:20]; + __block NSUInteger fifoMaxSize = 0; + __block NSUInteger fifoTotalWrittenElements = 0; + __block NSUInteger fifoWriteCount = 0; + + mock.expectRecordedDataIsAvailable(^(const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel) { + if (fifoWriteCount++ < kNumIgnoreFirstCallbacks) { + return 0; + } + + NSData *data = [NSData dataWithBytes:audioSamples length:nSamples*nBytesPerSample*nChannels]; + @synchronized(fifoBuffer) { + [fifoBuffer addObject:data]; + fifoMaxSize = MAX(fifoMaxSize, fifoBuffer.count); + fifoTotalWrittenElements += fifoBuffer.count; + } + + return 0; + }); + + mock.expectNeedMorePlayData(^int32_t(const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void *audioSamples, + size_t &nSamplesOut, + int64_t *elapsed_time_ms, + int64_t *ntp_time_ms) { + nSamplesOut = nSamples; + NSData *data; + @synchronized(fifoBuffer) { + data = fifoBuffer.firstObject; + if (data) { + [fifoBuffer removeObjectAtIndex:0]; + } + } + + if (data) { + memcpy(audioSamples, (char*) data.bytes, data.length); + } else { + memset(audioSamples, 0, nSamples*nBytesPerSample*nChannels); + } + + if (playoutCallbacks++ == expectedPlayoutCallbacks) { + [playoutExpectation fulfill]; + } + return 0; + }); + + XCTAssertEqual(0, audioDeviceModule->RegisterAudioCallback(&mock)); + [self startRecording]; + [self startPlayout]; + NSTimeInterval waitTimeout = kFullDuplexTimeInSec * 2.0; + [self waitForExpectationsWithTimeout:waitTimeout handler:nil]; + + size_t fifoAverageSize = + (fifoTotalWrittenElements == 0) + ? 0.0 + : 0.5 + (double)fifoTotalWrittenElements / (fifoWriteCount - kNumIgnoreFirstCallbacks); + + [self stopPlayout]; + [self stopRecording]; + XCTAssertLessThan(fifoAverageSize, 10u); + XCTAssertLessThan(fifoMaxSize, 20u); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDevice_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDevice_xctest.mm new file mode 100644 index 0000000000..eec9e17a17 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioDevice_xctest.mm @@ -0,0 +1,129 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <XCTest/XCTest.h> + +#include <stdlib.h> + +#include "api/task_queue/default_task_queue_factory.h" + +#import "sdk/objc/components/audio/RTCAudioSession+Private.h" +#import "sdk/objc/native/api/audio_device_module.h" +#import "sdk/objc/native/src/audio/audio_device_ios.h" + +@interface RTCAudioDeviceTests : XCTestCase { + bool _testEnabled; + rtc::scoped_refptr<webrtc::AudioDeviceModule> _audioDeviceModule; + std::unique_ptr<webrtc::ios_adm::AudioDeviceIOS> _audio_device; +} + +@property(nonatomic) RTC_OBJC_TYPE(RTCAudioSession) * audioSession; + +@end + +@implementation RTCAudioDeviceTests + +@synthesize audioSession = _audioSession; + +- (void)setUp { + [super setUp]; +#if defined(WEBRTC_IOS) && TARGET_OS_SIMULATOR + // TODO(peterhanspers): Reenable these tests on simulator. + // See bugs.webrtc.org/7812 + _testEnabled = false; + if (::getenv("WEBRTC_IOS_RUN_AUDIO_TESTS") != nullptr) { + _testEnabled = true; + } +#else + _testEnabled = true; +#endif + + _audioDeviceModule = webrtc::CreateAudioDeviceModule(); + _audio_device.reset(new webrtc::ios_adm::AudioDeviceIOS(/*bypass_voice_processing=*/false)); + self.audioSession = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + + NSError *error = nil; + [self.audioSession lockForConfiguration]; + [self.audioSession setCategory:AVAudioSessionCategoryPlayAndRecord withOptions:0 error:&error]; + XCTAssertNil(error); + + [self.audioSession setMode:AVAudioSessionModeVoiceChat error:&error]; + XCTAssertNil(error); + + [self.audioSession setActive:YES error:&error]; + XCTAssertNil(error); + + [self.audioSession unlockForConfiguration]; +} + +- (void)tearDown { + _audio_device->Terminate(); + _audio_device.reset(nullptr); + _audioDeviceModule = nullptr; + [self.audioSession notifyDidEndInterruptionWithShouldResumeSession:NO]; + + [super tearDown]; +} + +// Verifies that the AudioDeviceIOS is_interrupted_ flag is reset correctly +// after an iOS AVAudioSessionInterruptionTypeEnded notification event. +// AudioDeviceIOS listens to RTC_OBJC_TYPE(RTCAudioSession) interrupted notifications by: +// - In AudioDeviceIOS.InitPlayOrRecord registers its audio_session_observer_ +// callback with RTC_OBJC_TYPE(RTCAudioSession)'s delegate list. +// - When RTC_OBJC_TYPE(RTCAudioSession) receives an iOS audio interrupted notification, it +// passes the notification to callbacks in its delegate list which sets +// AudioDeviceIOS's is_interrupted_ flag to true. +// - When AudioDeviceIOS.ShutdownPlayOrRecord is called, its +// audio_session_observer_ callback is removed from RTCAudioSessions's +// delegate list. +// So if RTC_OBJC_TYPE(RTCAudioSession) receives an iOS end audio interruption notification, +// AudioDeviceIOS is not notified as its callback is not in RTC_OBJC_TYPE(RTCAudioSession)'s +// delegate list. This causes AudioDeviceIOS's is_interrupted_ flag to be in +// the wrong (true) state and the audio session will ignore audio changes. +// As RTC_OBJC_TYPE(RTCAudioSession) keeps its own interrupted state, the fix is to initialize +// AudioDeviceIOS's is_interrupted_ flag to RTC_OBJC_TYPE(RTCAudioSession)'s isInterrupted +// flag in AudioDeviceIOS.InitPlayOrRecord. +- (void)testInterruptedAudioSession { + XCTSkipIf(!_testEnabled); + XCTAssertTrue(self.audioSession.isActive); + XCTAssertTrue([self.audioSession.category isEqual:AVAudioSessionCategoryPlayAndRecord] || + [self.audioSession.category isEqual:AVAudioSessionCategoryPlayback]); + XCTAssertEqual(AVAudioSessionModeVoiceChat, self.audioSession.mode); + + std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory = + webrtc::CreateDefaultTaskQueueFactory(); + std::unique_ptr<webrtc::AudioDeviceBuffer> audio_buffer; + audio_buffer.reset(new webrtc::AudioDeviceBuffer(task_queue_factory.get())); + _audio_device->AttachAudioBuffer(audio_buffer.get()); + XCTAssertEqual(webrtc::AudioDeviceGeneric::InitStatus::OK, _audio_device->Init()); + XCTAssertEqual(0, _audio_device->InitPlayout()); + XCTAssertEqual(0, _audio_device->StartPlayout()); + + // Force interruption. + [self.audioSession notifyDidBeginInterruption]; + + // Wait for notification to propagate. + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + XCTAssertTrue(_audio_device->IsInterrupted()); + + // Force it for testing. + _audio_device->StopPlayout(); + + [self.audioSession notifyDidEndInterruptionWithShouldResumeSession:YES]; + // Wait for notification to propagate. + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + XCTAssertTrue(_audio_device->IsInterrupted()); + + _audio_device->Init(); + _audio_device->InitPlayout(); + XCTAssertFalse(_audio_device->IsInterrupted()); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCAudioSessionTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioSessionTest.mm new file mode 100644 index 0000000000..d7cfc9ed04 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCAudioSessionTest.mm @@ -0,0 +1,317 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <OCMock/OCMock.h> +#import <XCTest/XCTest.h> + +#include <vector> + +#include "rtc_base/event.h" +#include "rtc_base/gunit.h" + +#import "components/audio/RTCAudioSession+Private.h" + +#import "components/audio/RTCAudioSession.h" +#import "components/audio/RTCAudioSessionConfiguration.h" + +@interface RTC_OBJC_TYPE (RTCAudioSession) +(UnitTesting) + + @property(nonatomic, + readonly) std::vector<__weak id<RTC_OBJC_TYPE(RTCAudioSessionDelegate)> > delegates; + +- (instancetype)initWithAudioSession:(id)audioSession; + +@end + +@interface MockAVAudioSession : NSObject + +@property (nonatomic, readwrite, assign) float outputVolume; + +@end + +@implementation MockAVAudioSession +@synthesize outputVolume = _outputVolume; +@end + +@interface RTCAudioSessionTestDelegate : NSObject <RTC_OBJC_TYPE (RTCAudioSessionDelegate)> + +@property (nonatomic, readonly) float outputVolume; + +@end + +@implementation RTCAudioSessionTestDelegate + +@synthesize outputVolume = _outputVolume; + +- (instancetype)init { + if (self = [super init]) { + _outputVolume = -1; + } + return self; +} + +- (void)audioSessionDidBeginInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionDidEndInterruption:(RTC_OBJC_TYPE(RTCAudioSession) *)session + shouldResumeSession:(BOOL)shouldResumeSession { +} + +- (void)audioSessionDidChangeRoute:(RTC_OBJC_TYPE(RTCAudioSession) *)session + reason:(AVAudioSessionRouteChangeReason)reason + previousRoute:(AVAudioSessionRouteDescription *)previousRoute { +} + +- (void)audioSessionMediaServerTerminated:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionMediaServerReset:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionShouldConfigure:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSessionShouldUnconfigure:(RTC_OBJC_TYPE(RTCAudioSession) *)session { +} + +- (void)audioSession:(RTC_OBJC_TYPE(RTCAudioSession) *)audioSession + didChangeOutputVolume:(float)outputVolume { + _outputVolume = outputVolume; +} + +@end + +// A delegate that adds itself to the audio session on init and removes itself +// in its dealloc. +@interface RTCTestRemoveOnDeallocDelegate : RTCAudioSessionTestDelegate +@end + +@implementation RTCTestRemoveOnDeallocDelegate + +- (instancetype)init { + if (self = [super init]) { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session addDelegate:self]; + } + return self; +} + +- (void)dealloc { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + [session removeDelegate:self]; +} + +@end + +@interface RTCAudioSessionTest : XCTestCase + +@end + +@implementation RTCAudioSessionTest + +- (void)testAddAndRemoveDelegates { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + NSMutableArray *delegates = [NSMutableArray array]; + const size_t count = 5; + for (size_t i = 0; i < count; ++i) { + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + [delegates addObject:delegate]; + EXPECT_EQ(i + 1, session.delegates.size()); + } + [delegates enumerateObjectsUsingBlock:^(RTCAudioSessionTestDelegate *obj, + NSUInteger idx, + BOOL *stop) { + [session removeDelegate:obj]; + }]; + EXPECT_EQ(0u, session.delegates.size()); +} + +- (void)testPushDelegate { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + NSMutableArray *delegates = [NSMutableArray array]; + const size_t count = 2; + for (size_t i = 0; i < count; ++i) { + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + [delegates addObject:delegate]; + } + // Test that it gets added to the front of the list. + RTCAudioSessionTestDelegate *pushedDelegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session pushDelegate:pushedDelegate]; + EXPECT_TRUE(pushedDelegate == session.delegates[0]); + + // Test that it stays at the front of the list. + for (size_t i = 0; i < count; ++i) { + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + [delegates addObject:delegate]; + } + EXPECT_TRUE(pushedDelegate == session.delegates[0]); + + // Test that the next one goes to the front too. + pushedDelegate = [[RTCAudioSessionTestDelegate alloc] init]; + [session pushDelegate:pushedDelegate]; + EXPECT_TRUE(pushedDelegate == session.delegates[0]); +} + +// Tests that delegates added to the audio session properly zero out. This is +// checking an implementation detail (that vectors of __weak work as expected). +- (void)testZeroingWeakDelegate { + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + @autoreleasepool { + // Add a delegate to the session. There should be one delegate at this + // point. + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + EXPECT_EQ(1u, session.delegates.size()); + EXPECT_TRUE(session.delegates[0]); + } + // The previously created delegate should've de-alloced, leaving a nil ptr. + EXPECT_FALSE(session.delegates[0]); + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + // On adding a new delegate, nil ptrs should've been cleared. + EXPECT_EQ(1u, session.delegates.size()); + EXPECT_TRUE(session.delegates[0]); +} + +// Tests that we don't crash when removing delegates in dealloc. +// Added as a regression test. +- (void)testRemoveDelegateOnDealloc { + @autoreleasepool { + RTCTestRemoveOnDeallocDelegate *delegate = + [[RTCTestRemoveOnDeallocDelegate alloc] init]; + EXPECT_TRUE(delegate); + } + RTC_OBJC_TYPE(RTCAudioSession) *session = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + EXPECT_EQ(0u, session.delegates.size()); +} + +- (void)testAudioSessionActivation { + RTC_OBJC_TYPE(RTCAudioSession) *audioSession = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; + EXPECT_EQ(0, audioSession.activationCount); + [audioSession audioSessionDidActivate:[AVAudioSession sharedInstance]]; + EXPECT_EQ(1, audioSession.activationCount); + [audioSession audioSessionDidDeactivate:[AVAudioSession sharedInstance]]; + EXPECT_EQ(0, audioSession.activationCount); +} + +// TODO(b/298960678): Fix crash when running the test on simulators. +- (void)DISABLED_testConfigureWebRTCSession { + NSError *error = nil; + + void (^setActiveBlock)(NSInvocation *invocation) = ^(NSInvocation *invocation) { + __autoreleasing NSError **retError; + [invocation getArgument:&retError atIndex:4]; + *retError = [NSError errorWithDomain:@"AVAudioSession" + code:AVAudioSessionErrorCodeCannotInterruptOthers + userInfo:nil]; + BOOL failure = NO; + [invocation setReturnValue:&failure]; + }; + + id mockAVAudioSession = OCMPartialMock([AVAudioSession sharedInstance]); + OCMStub([[mockAVAudioSession ignoringNonObjectArgs] setActive:YES + withOptions:0 + error:([OCMArg anyObjectRef])]) + .andDo(setActiveBlock); + + id mockAudioSession = OCMPartialMock([RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]); + OCMStub([mockAudioSession session]).andReturn(mockAVAudioSession); + + RTC_OBJC_TYPE(RTCAudioSession) *audioSession = mockAudioSession; + EXPECT_EQ(0, audioSession.activationCount); + [audioSession lockForConfiguration]; + // configureWebRTCSession is forced to fail in the above mock interface, + // so activationCount should remain 0 + OCMExpect([[mockAVAudioSession ignoringNonObjectArgs] setActive:YES + withOptions:0 + error:([OCMArg anyObjectRef])]) + .andDo(setActiveBlock); + OCMExpect([mockAudioSession session]).andReturn(mockAVAudioSession); + EXPECT_FALSE([audioSession configureWebRTCSession:&error]); + EXPECT_EQ(0, audioSession.activationCount); + + id session = audioSession.session; + EXPECT_EQ(session, mockAVAudioSession); + EXPECT_EQ(NO, [mockAVAudioSession setActive:YES withOptions:0 error:&error]); + [audioSession unlockForConfiguration]; + + // The -Wunused-value is a workaround for https://bugs.llvm.org/show_bug.cgi?id=45245 + _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wunused-value\""); + OCMVerify([mockAudioSession session]); + OCMVerify([[mockAVAudioSession ignoringNonObjectArgs] setActive:YES withOptions:0 error:&error]); + OCMVerify([[mockAVAudioSession ignoringNonObjectArgs] setActive:NO withOptions:0 error:&error]); + _Pragma("clang diagnostic pop"); + + [mockAVAudioSession stopMocking]; + [mockAudioSession stopMocking]; +} + +// TODO(b/298960678): Fix crash when running the test on simulators. +- (void)DISABLED_testConfigureWebRTCSessionWithoutLocking { + NSError *error = nil; + + id mockAVAudioSession = OCMPartialMock([AVAudioSession sharedInstance]); + id mockAudioSession = OCMPartialMock([RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]); + OCMStub([mockAudioSession session]).andReturn(mockAVAudioSession); + + RTC_OBJC_TYPE(RTCAudioSession) *audioSession = mockAudioSession; + + std::unique_ptr<rtc::Thread> thread = rtc::Thread::Create(); + EXPECT_TRUE(thread); + EXPECT_TRUE(thread->Start()); + + rtc::Event waitLock; + rtc::Event waitCleanup; + constexpr webrtc::TimeDelta timeout = webrtc::TimeDelta::Seconds(5); + thread->PostTask([audioSession, &waitLock, &waitCleanup, timeout] { + [audioSession lockForConfiguration]; + waitLock.Set(); + waitCleanup.Wait(timeout); + [audioSession unlockForConfiguration]; + }); + + waitLock.Wait(timeout); + [audioSession setCategory:AVAudioSessionCategoryPlayAndRecord withOptions:0 error:&error]; + EXPECT_TRUE(error != nil); + EXPECT_EQ(error.domain, kRTCAudioSessionErrorDomain); + EXPECT_EQ(error.code, kRTCAudioSessionErrorLockRequired); + waitCleanup.Set(); + thread->Stop(); + + [mockAVAudioSession stopMocking]; + [mockAudioSession stopMocking]; +} + +- (void)testAudioVolumeDidNotify { + MockAVAudioSession *mockAVAudioSession = [[MockAVAudioSession alloc] init]; + RTC_OBJC_TYPE(RTCAudioSession) *session = + [[RTC_OBJC_TYPE(RTCAudioSession) alloc] initWithAudioSession:mockAVAudioSession]; + RTCAudioSessionTestDelegate *delegate = + [[RTCAudioSessionTestDelegate alloc] init]; + [session addDelegate:delegate]; + + float expectedVolume = 0.75; + mockAVAudioSession.outputVolume = expectedVolume; + + EXPECT_EQ(expectedVolume, delegate.outputVolume); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm new file mode 100644 index 0000000000..cf759c5243 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm @@ -0,0 +1,461 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +#import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "frame_buffer_helpers.h" + +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "third_party/libyuv/include/libyuv.h" + +namespace { + +struct ToI420WithCropAndScaleSetting { + int inputWidth; + int inputHeight; + int offsetX; + int offsetY; + int cropWidth; + int cropHeight; + int scaleWidth; + int scaleHeight; +}; + +constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = { + ToI420WithCropAndScaleSetting{ + .inputWidth = 640, + .inputHeight = 360, + .offsetX = 0, + .offsetY = 0, + .cropWidth = 640, + .cropHeight = 360, + .scaleWidth = 320, + .scaleHeight = 180, + }, + ToI420WithCropAndScaleSetting{ + .inputWidth = 640, + .inputHeight = 360, + .offsetX = 160, + .offsetY = 90, + .cropWidth = 160, + .cropHeight = 90, + .scaleWidth = 320, + .scaleHeight = 180, + }, +}; + +} // namespace + +@interface RTCCVPixelBufferTests : XCTestCase +@end + +@implementation RTCCVPixelBufferTests { +} + +- (void)testRequiresCroppingNoCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + XCTAssertFalse([buffer requiresCropping]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresCroppingWithCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + RTC_OBJC_TYPE(RTCCVPixelBuffer) *croppedBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:720 + adaptedHeight:1280 + cropWidth:360 + cropHeight:640 + cropX:100 + cropY:100]; + + XCTAssertTrue([croppedBuffer requiresCropping]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingNoScale { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertFalse([buffer requiresScalingToWidth:720 height:1280]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingWithScale { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertTrue([buffer requiresScalingToWidth:360 height:640]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testRequiresScalingWithScaleAndMatchingCrop { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef + adaptedWidth:720 + adaptedHeight:1280 + cropWidth:360 + cropHeight:640 + cropX:100 + cropY:100]; + XCTAssertFalse([buffer requiresScalingToWidth:360 height:640]); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testBufferSize_NV12 { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 576000); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testBufferSize_RGB { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 720, 1280, kCVPixelFormatType_32BGRA, NULL, &pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 0); + + CVBufferRelease(pixelBufferRef); +} + +- (void)testCropAndScale_NV12 { + [self cropAndScaleTestWithNV12]; +} + +- (void)testCropAndScaleNoOp_NV12 { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputSize:CGSizeMake(720, 1280)]; +} + +- (void)testCropAndScale_NV12FullToVideo { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; +} + +- (void)testCropAndScaleZeroSizeFrame_NV12 { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputSize:CGSizeMake(0, 0)]; +} + +- (void)testCropAndScaleToSmallFormat_NV12 { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputSize:CGSizeMake(148, 320)]; +} + +- (void)testCropAndScaleToOddFormat_NV12 { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputSize:CGSizeMake(361, 640)]; +} + +- (void)testCropAndScale_32BGRA { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32BGRA]; +} + +- (void)testCropAndScale_32ARGB { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB]; +} + +- (void)testCropAndScaleWithSmallCropInfo_32ARGB { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:2 cropY:3]; +} + +- (void)testCropAndScaleWithLargeCropInfo_32ARGB { + [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:200 cropY:300]; +} + +- (void)testToI420_NV12 { + [self toI420WithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; +} + +- (void)testToI420_32BGRA { + [self toI420WithPixelFormat:kCVPixelFormatType_32BGRA]; +} + +- (void)testToI420_32ARGB { + [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB]; +} + +- (void)testToI420WithCropAndScale_NV12 { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + setting:setting]; + } +} + +- (void)testToI420WithCropAndScale_32BGRA { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting]; + } +} + +- (void)testToI420WithCropAndScale_32ARGB { + for (const auto &setting : kToI420WithCropAndScaleSettings) { + [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting]; + } +} + +- (void)testScaleBufferTest { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); + + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(1920, 1080); + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + XCTAssertEqual(buffer.width, 1920); + XCTAssertEqual(buffer.height, 1080); + XCTAssertEqual(buffer.cropX, 0); + XCTAssertEqual(buffer.cropY, 0); + XCTAssertEqual(buffer.cropWidth, 1920); + XCTAssertEqual(buffer.cropHeight, 1080); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320 + offsetY:180 + cropWidth:1280 + cropHeight:720 + scaleWidth:960 + scaleHeight:540]; + + XCTAssertEqual(buffer2.width, 960); + XCTAssertEqual(buffer2.height, 540); + XCTAssertEqual(buffer2.cropX, 320); + XCTAssertEqual(buffer2.cropY, 180); + XCTAssertEqual(buffer2.cropWidth, 1280); + XCTAssertEqual(buffer2.cropHeight, 720); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 = + (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240 + offsetY:135 + cropWidth:480 + cropHeight:270 + scaleWidth:320 + scaleHeight:180]; + + XCTAssertEqual(buffer3.width, 320); + XCTAssertEqual(buffer3.height, 180); + XCTAssertEqual(buffer3.cropX, 640); + XCTAssertEqual(buffer3.cropY, 360); + XCTAssertEqual(buffer3.cropWidth, 640); + XCTAssertEqual(buffer3.cropHeight, 360); + + CVBufferRelease(pixelBufferRef); +} + +#pragma mark - Shared test code + +- (void)cropAndScaleTestWithNV12 { + [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]; +} + +- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat { + [self cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat + outputFormat:(OSType)outputFormat + outputSize:CGSizeMake(360, 640)]; +} + +- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat + outputFormat:(OSType)outputFormat + outputSize:(CGSize)outputSize { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 720, 1280, inputFormat, NULL, &pixelBufferRef); + + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(720, 1280); + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + XCTAssertEqual(buffer.width, 720); + XCTAssertEqual(buffer.height, 1280); + + CVPixelBufferRef outputPixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, outputSize.width, outputSize.height, outputFormat, NULL, &outputPixelBufferRef); + + std::vector<uint8_t> frameScaleBuffer; + if ([buffer requiresScalingToWidth:outputSize.width height:outputSize.height]) { + int size = + [buffer bufferSizeForCroppingAndScalingToWidth:outputSize.width height:outputSize.height]; + frameScaleBuffer.resize(size); + } else { + frameScaleBuffer.clear(); + } + frameScaleBuffer.shrink_to_fit(); + + [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:frameScaleBuffer.data()]; + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef]; + XCTAssertEqual(scaledBuffer.width, outputSize.width); + XCTAssertEqual(scaledBuffer.height, outputSize.height); + + if (outputSize.width > 0 && outputSize.height > 0) { + RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420]; + RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420]; + double psnr = + I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + } + + CVBufferRelease(pixelBufferRef); +} + +- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat { + [self cropAndScaleTestWithRGBPixelFormat:pixelFormat cropX:0 cropY:0]; +} + +- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat cropX:(int)cropX cropY:(int)cropY { + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 720, 1280, pixelFormat, NULL, &pixelBufferRef); + + DrawGradientInRGBPixelBuffer(pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] + initWithPixelBuffer:pixelBufferRef + adaptedWidth:CVPixelBufferGetWidth(pixelBufferRef) + adaptedHeight:CVPixelBufferGetHeight(pixelBufferRef) + cropWidth:CVPixelBufferGetWidth(pixelBufferRef) - cropX + cropHeight:CVPixelBufferGetHeight(pixelBufferRef) - cropY + cropX:cropX + cropY:cropY]; + + XCTAssertEqual(buffer.width, 720); + XCTAssertEqual(buffer.height, 1280); + + CVPixelBufferRef outputPixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &outputPixelBufferRef); + [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:NULL]; + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef]; + XCTAssertEqual(scaledBuffer.width, 360); + XCTAssertEqual(scaledBuffer.height, 640); + + RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420]; + RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420]; + double psnr = + I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]); + XCTAssertEqual(psnr, webrtc::kPerfectPSNR); + + CVBufferRelease(pixelBufferRef); +} + +- (void)toI420WithPixelFormat:(OSType)pixelFormat { + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(360, 640); + + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &pixelBufferRef); + + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [buffer toI420]; + + double psnr = I420PSNR(*i420Buffer, *[fromCVPixelBuffer nativeI420Buffer]); + double target = webrtc::kPerfectPSNR; + if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + // libyuv's I420ToRGB functions seem to lose some quality. + target = 19.0; + } + XCTAssertGreaterThanOrEqual(psnr, target); + + CVBufferRelease(pixelBufferRef); +} + +- (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat + setting:(const ToI420WithCropAndScaleSetting &)setting { + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = + CreateI420Gradient(setting.inputWidth, setting.inputHeight); + + CVPixelBufferRef pixelBufferRef = NULL; + CVPixelBufferCreate( + NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef); + + CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef); + + RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer = + [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; + RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer = + (RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX + offsetY:setting.offsetY + cropWidth:setting.cropWidth + cropHeight:setting.cropHeight + scaleWidth:setting.scaleWidth + scaleHeight:setting.scaleHeight]; + RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> scaledBuffer = + [buffer cropAndScaleWith:setting.offsetX + offsetY:setting.offsetY + cropWidth:setting.cropWidth + cropHeight:setting.cropHeight + scaleWidth:setting.scaleWidth + scaleHeight:setting.scaleHeight]; + XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]); + + RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420]; + + double psnr = + I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]); + double target = webrtc::kPerfectPSNR; + if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) { + // libyuv's I420ToRGB functions seem to lose some quality. + target = 19.0; + } + XCTAssertGreaterThanOrEqual(psnr, target); + + CVBufferRelease(pixelBufferRef); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCCallbackLogger_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCCallbackLogger_xctest.m new file mode 100644 index 0000000000..1b6fb1c07b --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCCallbackLogger_xctest.m @@ -0,0 +1,244 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "api/logging/RTCCallbackLogger.h" + +#import <XCTest/XCTest.h> + +@interface RTCCallbackLoggerTests : XCTestCase + +@property(nonatomic, strong) RTC_OBJC_TYPE(RTCCallbackLogger) * logger; + +@end + +@implementation RTCCallbackLoggerTests + +@synthesize logger; + +- (void)setUp { + self.logger = [[RTC_OBJC_TYPE(RTCCallbackLogger) alloc] init]; +} + +- (void)tearDown { + self.logger = nil; +} + +- (void)testDefaultSeverityLevel { + XCTAssertEqual(self.logger.severity, RTCLoggingSeverityInfo); +} + +- (void)testCallbackGetsCalledForAppropriateLevel { + self.logger.severity = RTCLoggingSeverityWarning; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"callbackWarning"]; + + [self.logger start:^(NSString *message) { + XCTAssertTrue([message hasSuffix:@"Horrible error\n"]); + [callbackExpectation fulfill]; + }]; + + RTCLogError("Horrible error"); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testCallbackWithSeverityGetsCalledForAppropriateLevel { + self.logger.severity = RTCLoggingSeverityWarning; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"callbackWarning"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity severity) { + XCTAssertTrue([message hasSuffix:@"Horrible error\n"]); + XCTAssertEqual(severity, RTCLoggingSeverityError); + [callbackExpectation fulfill]; + }]; + + RTCLogError("Horrible error"); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testCallbackDoesNotGetCalledForOtherLevels { + self.logger.severity = RTCLoggingSeverityError; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"callbackError"]; + + [self.logger start:^(NSString *message) { + XCTAssertTrue([message hasSuffix:@"Horrible error\n"]); + [callbackExpectation fulfill]; + }]; + + RTCLogInfo("Just some info"); + RTCLogWarning("Warning warning"); + RTCLogError("Horrible error"); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testCallbackWithSeverityDoesNotGetCalledForOtherLevels { + self.logger.severity = RTCLoggingSeverityError; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"callbackError"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity severity) { + XCTAssertTrue([message hasSuffix:@"Horrible error\n"]); + XCTAssertEqual(severity, RTCLoggingSeverityError); + [callbackExpectation fulfill]; + }]; + + RTCLogInfo("Just some info"); + RTCLogWarning("Warning warning"); + RTCLogError("Horrible error"); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +- (void)testCallbackDoesNotgetCalledForSeverityNone { + self.logger.severity = RTCLoggingSeverityNone; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"unexpectedCallback"]; + + [self.logger start:^(NSString *message) { + [callbackExpectation fulfill]; + XCTAssertTrue(false); + }]; + + RTCLogInfo("Just some info"); + RTCLogWarning("Warning warning"); + RTCLogError("Horrible error"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testCallbackWithSeverityDoesNotgetCalledForSeverityNone { + self.logger.severity = RTCLoggingSeverityNone; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"unexpectedCallback"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity severity) { + [callbackExpectation fulfill]; + XCTAssertTrue(false); + }]; + + RTCLogInfo("Just some info"); + RTCLogWarning("Warning warning"); + RTCLogError("Horrible error"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testStartingWithNilCallbackDoesNotCrash { + [self.logger start:nil]; + + RTCLogError("Horrible error"); +} + +- (void)testStartingWithNilCallbackWithSeverityDoesNotCrash { + [self.logger startWithMessageAndSeverityHandler:nil]; + + RTCLogError("Horrible error"); +} + +- (void)testStopCallbackLogger { + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"stopped"]; + + [self.logger start:^(NSString *message) { + [callbackExpectation fulfill]; + }]; + + [self.logger stop]; + + RTCLogInfo("Just some info"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testStopCallbackWithSeverityLogger { + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"stopped"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity loggingServerity) { + [callbackExpectation fulfill]; + }]; + + [self.logger stop]; + + RTCLogInfo("Just some info"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testDestroyingCallbackLogger { + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"destroyed"]; + + [self.logger start:^(NSString *message) { + [callbackExpectation fulfill]; + }]; + + self.logger = nil; + + RTCLogInfo("Just some info"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testDestroyingCallbackWithSeverityLogger { + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"destroyed"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity loggingServerity) { + [callbackExpectation fulfill]; + }]; + + self.logger = nil; + + RTCLogInfo("Just some info"); + + XCTWaiter *waiter = [[XCTWaiter alloc] init]; + XCTWaiterResult result = [waiter waitForExpectations:@[ callbackExpectation ] timeout:1.0]; + XCTAssertEqual(result, XCTWaiterResultTimedOut); +} + +- (void)testCallbackWithSeverityLoggerCannotStartTwice { + self.logger.severity = RTCLoggingSeverityWarning; + + XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"callbackWarning"]; + + [self.logger + startWithMessageAndSeverityHandler:^(NSString *message, RTCLoggingSeverity loggingServerity) { + XCTAssertTrue([message hasSuffix:@"Horrible error\n"]); + XCTAssertEqual(loggingServerity, RTCLoggingSeverityError); + [callbackExpectation fulfill]; + }]; + + [self.logger start:^(NSString *message) { + [callbackExpectation fulfill]; + XCTAssertTrue(false); + }]; + + RTCLogError("Horrible error"); + + [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCCameraVideoCapturerTests.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCCameraVideoCapturerTests.mm new file mode 100644 index 0000000000..6a117a3546 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCCameraVideoCapturerTests.mm @@ -0,0 +1,560 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <OCMock/OCMock.h> +#import <XCTest/XCTest.h> + +#if TARGET_OS_IPHONE +#import <UIKit/UIKit.h> +#endif + +#import "base/RTCVideoFrame.h" +#import "components/capturer/RTCCameraVideoCapturer.h" +#import "helpers/AVCaptureSession+DevicePosition.h" +#import "helpers/RTCDispatcher.h" +#import "helpers/scoped_cftyperef.h" + +#define WAIT(timeoutMs) \ + do { \ + id expectation = [[XCTestExpectation alloc] initWithDescription:@"Dummy"]; \ + XCTWaiterResult res = [XCTWaiter waitForExpectations:@[ expectation ] \ + timeout:timeoutMs / 1000.0]; \ + XCTAssertEqual(XCTWaiterResultTimedOut, res); \ + } while (false); + +#if TARGET_OS_IPHONE +// Helper method. +CMSampleBufferRef createTestSampleBufferRef() { + + // This image is already in the testing bundle. + UIImage *image = [UIImage imageNamed:@"Default.png"]; + CGSize size = image.size; + CGImageRef imageRef = [image CGImage]; + + CVPixelBufferRef pixelBuffer = nullptr; + CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height, kCVPixelFormatType_32ARGB, nil, + &pixelBuffer); + + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + // We don't care about bitsPerComponent and bytesPerRow so arbitrary value of 8 for both. + CGContextRef context = CGBitmapContextCreate(nil, size.width, size.height, 8, 8 * size.width, + rgbColorSpace, kCGImageAlphaPremultipliedFirst); + + CGContextDrawImage( + context, CGRectMake(0, 0, CGImageGetWidth(imageRef), CGImageGetHeight(imageRef)), imageRef); + + CGColorSpaceRelease(rgbColorSpace); + CGContextRelease(context); + + // We don't really care about the timing. + CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid}; + CMVideoFormatDescriptionRef description = nullptr; + CMVideoFormatDescriptionCreateForImageBuffer(NULL, pixelBuffer, &description); + + CMSampleBufferRef sampleBuffer = nullptr; + CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, YES, NULL, NULL, description, + &timing, &sampleBuffer); + CFRelease(pixelBuffer); + + return sampleBuffer; + +} +#endif +@interface RTC_OBJC_TYPE (RTCCameraVideoCapturer) +(Tests)<AVCaptureVideoDataOutputSampleBufferDelegate> - + (instancetype)initWithDelegate + : (__weak id<RTC_OBJC_TYPE(RTCVideoCapturerDelegate)>)delegate captureSession + : (AVCaptureSession *)captureSession; +@end + +@interface RTCCameraVideoCapturerTests : XCTestCase +@property(nonatomic, strong) id delegateMock; +@property(nonatomic, strong) id deviceMock; +@property(nonatomic, strong) id captureConnectionMock; +@property(nonatomic, strong) RTC_OBJC_TYPE(RTCCameraVideoCapturer) * capturer; +@end + +@implementation RTCCameraVideoCapturerTests +@synthesize delegateMock = _delegateMock; +@synthesize deviceMock = _deviceMock; +@synthesize captureConnectionMock = _captureConnectionMock; +@synthesize capturer = _capturer; + +- (void)setUp { + self.delegateMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoCapturerDelegate))); + self.captureConnectionMock = OCMClassMock([AVCaptureConnection class]); + self.capturer = + [[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:self.delegateMock]; + self.deviceMock = [RTCCameraVideoCapturerTests createDeviceMock]; +} + +- (void)tearDown { + [self.delegateMock stopMocking]; + [self.deviceMock stopMocking]; + self.delegateMock = nil; + self.deviceMock = nil; + self.capturer = nil; +} + +#pragma mark - utils + ++ (id)createDeviceMock { + return OCMClassMock([AVCaptureDevice class]); +} + +#pragma mark - test cases + +- (void)testSetupSession { + AVCaptureSession *session = self.capturer.captureSession; + XCTAssertTrue(session != nil); + +#if TARGET_OS_IPHONE + XCTAssertEqual(session.sessionPreset, AVCaptureSessionPresetInputPriority); + XCTAssertEqual(session.usesApplicationAudioSession, NO); +#endif + XCTAssertEqual(session.outputs.count, 1u); +} + +- (void)testSetupSessionOutput { + AVCaptureVideoDataOutput *videoOutput = self.capturer.captureSession.outputs[0]; + XCTAssertEqual(videoOutput.alwaysDiscardsLateVideoFrames, NO); + XCTAssertEqual(videoOutput.sampleBufferDelegate, self.capturer); +} + +- (void)testSupportedFormatsForDevice { + // given + id validFormat1 = OCMClassMock([AVCaptureDeviceFormat class]); + CMVideoFormatDescriptionRef format; + + // We don't care about width and heigth so arbitrary 123 and 456 values. + int width = 123; + int height = 456; + CMVideoFormatDescriptionCreate(nil, kCVPixelFormatType_420YpCbCr8PlanarFullRange, width, height, + nil, &format); + OCMStub([validFormat1 formatDescription]).andReturn(format); + + id validFormat2 = OCMClassMock([AVCaptureDeviceFormat class]); + CMVideoFormatDescriptionCreate(nil, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, width, + height, nil, &format); + OCMStub([validFormat2 formatDescription]).andReturn(format); + + id invalidFormat = OCMClassMock([AVCaptureDeviceFormat class]); + CMVideoFormatDescriptionCreate(nil, kCVPixelFormatType_422YpCbCr8_yuvs, width, height, nil, + &format); + OCMStub([invalidFormat formatDescription]).andReturn(format); + + NSArray *formats = @[ validFormat1, validFormat2, invalidFormat ]; + OCMStub([self.deviceMock formats]).andReturn(formats); + + // when + NSArray *supportedFormats = + [RTC_OBJC_TYPE(RTCCameraVideoCapturer) supportedFormatsForDevice:self.deviceMock]; + + // then + XCTAssertEqual(supportedFormats.count, 3u); + XCTAssertTrue([supportedFormats containsObject:validFormat1]); + XCTAssertTrue([supportedFormats containsObject:validFormat2]); + XCTAssertTrue([supportedFormats containsObject:invalidFormat]); + + // cleanup + [validFormat1 stopMocking]; + [validFormat2 stopMocking]; + [invalidFormat stopMocking]; + validFormat1 = nil; + validFormat2 = nil; + invalidFormat = nil; +} + +- (void)testDelegateCallbackNotCalledWhenInvalidBuffer { + // given + CMSampleBufferRef sampleBuffer = nullptr; + [[self.delegateMock reject] capturer:[OCMArg any] didCaptureVideoFrame:[OCMArg any]]; + + // when + [self.capturer captureOutput:self.capturer.captureSession.outputs[0] + didOutputSampleBuffer:sampleBuffer + fromConnection:self.captureConnectionMock]; + + // then + [self.delegateMock verify]; +} + +- (void)testDelegateCallbackWithValidBufferAndOrientationUpdate { +#if TARGET_OS_IPHONE + XCTExpectFailure(@"Setting orientation on UIDevice is not supported"); + [UIDevice.currentDevice setValue:@(UIDeviceOrientationPortraitUpsideDown) forKey:@"orientation"]; + CMSampleBufferRef sampleBuffer = createTestSampleBufferRef(); + + // then + [[self.delegateMock expect] capturer:self.capturer + didCaptureVideoFrame:[OCMArg checkWithBlock:^BOOL(RTC_OBJC_TYPE(RTCVideoFrame) * + expectedFrame) { + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_270); + return YES; + }]]; + + // when + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + [center postNotificationName:UIDeviceOrientationDidChangeNotification object:nil]; + + // We need to wait for the dispatch to finish. + WAIT(1000); + + [self.capturer captureOutput:self.capturer.captureSession.outputs[0] + didOutputSampleBuffer:sampleBuffer + fromConnection:self.captureConnectionMock]; + + [self.delegateMock verify]; + CFRelease(sampleBuffer); +#endif +} + +// The XCTest framework considers functions that don't take arguments tests. This is a helper. +- (void)testRotationCamera:(AVCaptureDevicePosition)camera + withOrientation:(UIDeviceOrientation)deviceOrientation { +#if TARGET_OS_IPHONE + // Mock the AVCaptureConnection as we will get the camera position from the connection's + // input ports. + AVCaptureDeviceInput *inputPortMock = OCMClassMock([AVCaptureDeviceInput class]); + AVCaptureInputPort *captureInputPort = OCMClassMock([AVCaptureInputPort class]); + NSArray *inputPortsArrayMock = @[captureInputPort]; + AVCaptureDevice *captureDeviceMock = OCMClassMock([AVCaptureDevice class]); + OCMStub(((AVCaptureConnection *)self.captureConnectionMock).inputPorts). + andReturn(inputPortsArrayMock); + OCMStub(captureInputPort.input).andReturn(inputPortMock); + OCMStub(inputPortMock.device).andReturn(captureDeviceMock); + OCMStub(captureDeviceMock.position).andReturn(camera); + + XCTExpectFailure(@"Setting orientation on UIDevice is not supported"); + [UIDevice.currentDevice setValue:@(deviceOrientation) forKey:@"orientation"]; + + CMSampleBufferRef sampleBuffer = createTestSampleBufferRef(); + + [[self.delegateMock expect] capturer:self.capturer + didCaptureVideoFrame:[OCMArg checkWithBlock:^BOOL(RTC_OBJC_TYPE(RTCVideoFrame) * + expectedFrame) { + if (camera == AVCaptureDevicePositionFront) { + if (deviceOrientation == UIDeviceOrientationLandscapeLeft) { + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_180); + } else if (deviceOrientation == UIDeviceOrientationLandscapeRight) { + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_0); + } + } else if (camera == AVCaptureDevicePositionBack) { + if (deviceOrientation == UIDeviceOrientationLandscapeLeft) { + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_0); + } else if (deviceOrientation == UIDeviceOrientationLandscapeRight) { + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_180); + } + } + return YES; + }]]; + + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + [center postNotificationName:UIDeviceOrientationDidChangeNotification object:nil]; + + // We need to wait for the dispatch to finish. + WAIT(1000); + + [self.capturer captureOutput:self.capturer.captureSession.outputs[0] + didOutputSampleBuffer:sampleBuffer + fromConnection:self.captureConnectionMock]; + + [self.delegateMock verify]; + + CFRelease(sampleBuffer); +#endif +} + +- (void)testRotationCameraBackLandscapeLeft { + [self testRotationCamera:AVCaptureDevicePositionBack + withOrientation:UIDeviceOrientationLandscapeLeft]; +} + +- (void)testRotationCameraFrontLandscapeLeft { + [self testRotationCamera:AVCaptureDevicePositionFront + withOrientation:UIDeviceOrientationLandscapeLeft]; +} + +- (void)testRotationCameraBackLandscapeRight { + [self testRotationCamera:AVCaptureDevicePositionBack + withOrientation:UIDeviceOrientationLandscapeRight]; +} + +- (void)testRotationCameraFrontLandscapeRight { + [self testRotationCamera:AVCaptureDevicePositionFront + withOrientation:UIDeviceOrientationLandscapeRight]; +} + +- (void)setExif:(CMSampleBufferRef)sampleBuffer { + rtc::ScopedCFTypeRef<CFMutableDictionaryRef> exif(CFDictionaryCreateMutable( + kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); + CFDictionarySetValue(exif.get(), CFSTR("LensModel"), CFSTR("iPhone SE back camera 4.15mm f/2.2")); + CMSetAttachment(sampleBuffer, CFSTR("{Exif}"), exif.get(), kCMAttachmentMode_ShouldPropagate); +} + +- (void)testRotationFrame { +#if TARGET_OS_IPHONE + // Mock the AVCaptureConnection as we will get the camera position from the connection's + // input ports. + AVCaptureDeviceInput *inputPortMock = OCMClassMock([AVCaptureDeviceInput class]); + AVCaptureInputPort *captureInputPort = OCMClassMock([AVCaptureInputPort class]); + NSArray *inputPortsArrayMock = @[captureInputPort]; + AVCaptureDevice *captureDeviceMock = OCMClassMock([AVCaptureDevice class]); + OCMStub(((AVCaptureConnection *)self.captureConnectionMock).inputPorts). + andReturn(inputPortsArrayMock); + OCMStub(captureInputPort.input).andReturn(inputPortMock); + OCMStub(inputPortMock.device).andReturn(captureDeviceMock); + OCMStub(captureDeviceMock.position).andReturn(AVCaptureDevicePositionFront); + + XCTExpectFailure(@"Setting orientation on UIDevice is not supported"); + [UIDevice.currentDevice setValue:@(UIDeviceOrientationLandscapeLeft) forKey:@"orientation"]; + + CMSampleBufferRef sampleBuffer = createTestSampleBufferRef(); + + [[self.delegateMock expect] capturer:self.capturer + didCaptureVideoFrame:[OCMArg checkWithBlock:^BOOL(RTC_OBJC_TYPE(RTCVideoFrame) * + expectedFrame) { + // Front camera and landscape left should return 180. But the frame's exif + // we add below says its from the back camera, so rotation should be 0. + XCTAssertEqual(expectedFrame.rotation, RTCVideoRotation_0); + return YES; + }]]; + + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + [center postNotificationName:UIDeviceOrientationDidChangeNotification object:nil]; + + // We need to wait for the dispatch to finish. + WAIT(1000); + + [self setExif:sampleBuffer]; + + [self.capturer captureOutput:self.capturer.captureSession.outputs[0] + didOutputSampleBuffer:sampleBuffer + fromConnection:self.captureConnectionMock]; + + [self.delegateMock verify]; + CFRelease(sampleBuffer); +#endif +} + +- (void)testImageExif { +#if TARGET_OS_IPHONE + CMSampleBufferRef sampleBuffer = createTestSampleBufferRef(); + [self setExif:sampleBuffer]; + + AVCaptureDevicePosition cameraPosition = [AVCaptureSession + devicePositionForSampleBuffer:sampleBuffer]; + XCTAssertEqual(cameraPosition, AVCaptureDevicePositionBack); +#endif +} + +@end + +@interface RTCCameraVideoCapturerTestsWithMockedCaptureSession : XCTestCase +@property(nonatomic, strong) id delegateMock; +@property(nonatomic, strong) id deviceMock; +@property(nonatomic, strong) id captureSessionMock; +@property(nonatomic, strong) RTC_OBJC_TYPE(RTCCameraVideoCapturer) * capturer; +@end + +@implementation RTCCameraVideoCapturerTestsWithMockedCaptureSession +@synthesize delegateMock = _delegateMock; +@synthesize deviceMock = _deviceMock; +@synthesize captureSessionMock = _captureSessionMock; +@synthesize capturer = _capturer; + +- (void)setUp { + self.captureSessionMock = OCMStrictClassMock([AVCaptureSession class]); + OCMStub([self.captureSessionMock setSessionPreset:[OCMArg any]]); + OCMStub([self.captureSessionMock setUsesApplicationAudioSession:NO]); + OCMStub([self.captureSessionMock canAddOutput:[OCMArg any]]).andReturn(YES); + OCMStub([self.captureSessionMock addOutput:[OCMArg any]]); + OCMStub([self.captureSessionMock beginConfiguration]); + OCMStub([self.captureSessionMock commitConfiguration]); + self.delegateMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoCapturerDelegate))); + self.capturer = + [[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:self.delegateMock + captureSession:self.captureSessionMock]; + self.deviceMock = [RTCCameraVideoCapturerTests createDeviceMock]; +} + +- (void)tearDown { + [self.delegateMock stopMocking]; + [self.deviceMock stopMocking]; + self.delegateMock = nil; + self.deviceMock = nil; + self.capturer = nil; + self.captureSessionMock = nil; +} + +#pragma mark - test cases + +- (void)testStartingAndStoppingCapture { + id expectedDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + id captureDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + OCMStub([captureDeviceInputMock deviceInputWithDevice:self.deviceMock error:[OCMArg setTo:nil]]) + .andReturn(expectedDeviceInputMock); + + OCMStub([self.deviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES); + OCMStub([self.deviceMock unlockForConfiguration]); + OCMStub([_captureSessionMock canAddInput:expectedDeviceInputMock]).andReturn(YES); + OCMStub([_captureSessionMock inputs]).andReturn(@[ expectedDeviceInputMock ]); + OCMStub([_captureSessionMock removeInput:expectedDeviceInputMock]); + + // Set expectation that the capture session should be started with correct device. + OCMExpect([_captureSessionMock addInput:expectedDeviceInputMock]); + OCMExpect([_captureSessionMock startRunning]); + OCMExpect([_captureSessionMock stopRunning]); + + id format = OCMClassMock([AVCaptureDeviceFormat class]); + [self.capturer startCaptureWithDevice:self.deviceMock format:format fps:30]; + [self.capturer stopCapture]; + + // Start capture code is dispatched async. + OCMVerifyAllWithDelay(_captureSessionMock, 15); +} + +- (void)testStartCaptureFailingToLockForConfiguration { + // The captureSessionMock is a strict mock, so this test will crash if the startCapture + // method does not return when failing to lock for configuration. + OCMExpect([self.deviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(NO); + + id format = OCMClassMock([AVCaptureDeviceFormat class]); + [self.capturer startCaptureWithDevice:self.deviceMock format:format fps:30]; + + // Start capture code is dispatched async. + OCMVerifyAllWithDelay(self.deviceMock, 15); +} + +- (void)testStartingAndStoppingCaptureWithCallbacks { + id expectedDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + id captureDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + OCMStub([captureDeviceInputMock deviceInputWithDevice:self.deviceMock error:[OCMArg setTo:nil]]) + .andReturn(expectedDeviceInputMock); + + OCMStub([self.deviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES); + OCMStub([self.deviceMock unlockForConfiguration]); + OCMStub([_captureSessionMock canAddInput:expectedDeviceInputMock]).andReturn(YES); + OCMStub([_captureSessionMock inputs]).andReturn(@[ expectedDeviceInputMock ]); + OCMStub([_captureSessionMock removeInput:expectedDeviceInputMock]); + + // Set expectation that the capture session should be started with correct device. + OCMExpect([_captureSessionMock addInput:expectedDeviceInputMock]); + OCMExpect([_captureSessionMock startRunning]); + OCMExpect([_captureSessionMock stopRunning]); + + dispatch_semaphore_t completedStopSemaphore = dispatch_semaphore_create(0); + + __block BOOL completedStart = NO; + id format = OCMClassMock([AVCaptureDeviceFormat class]); + [self.capturer startCaptureWithDevice:self.deviceMock + format:format + fps:30 + completionHandler:^(NSError *error) { + XCTAssertEqual(error, nil); + completedStart = YES; + }]; + + __block BOOL completedStop = NO; + [self.capturer stopCaptureWithCompletionHandler:^{ + completedStop = YES; + dispatch_semaphore_signal(completedStopSemaphore); + }]; + + dispatch_semaphore_wait(completedStopSemaphore, + dispatch_time(DISPATCH_TIME_NOW, 15.0 * NSEC_PER_SEC)); + OCMVerifyAllWithDelay(_captureSessionMock, 15); + XCTAssertTrue(completedStart); + XCTAssertTrue(completedStop); +} + +- (void)testStartCaptureFailingToLockForConfigurationWithCallback { + id expectedDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + id captureDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + OCMStub([captureDeviceInputMock deviceInputWithDevice:self.deviceMock error:[OCMArg setTo:nil]]) + .andReturn(expectedDeviceInputMock); + + id errorMock = OCMClassMock([NSError class]); + + OCMStub([self.deviceMock lockForConfiguration:[OCMArg setTo:errorMock]]).andReturn(NO); + OCMStub([_captureSessionMock canAddInput:expectedDeviceInputMock]).andReturn(YES); + OCMStub([self.deviceMock unlockForConfiguration]); + + OCMExpect([_captureSessionMock addInput:expectedDeviceInputMock]); + + dispatch_semaphore_t completedStartSemaphore = dispatch_semaphore_create(0); + __block NSError *callbackError = nil; + + id format = OCMClassMock([AVCaptureDeviceFormat class]); + [self.capturer startCaptureWithDevice:self.deviceMock + format:format + fps:30 + completionHandler:^(NSError *error) { + callbackError = error; + dispatch_semaphore_signal(completedStartSemaphore); + }]; + + long ret = dispatch_semaphore_wait(completedStartSemaphore, + dispatch_time(DISPATCH_TIME_NOW, 15.0 * NSEC_PER_SEC)); + XCTAssertEqual(ret, 0); + XCTAssertEqual(callbackError, errorMock); +} + +// TODO(crbug.com/webrtc/14829): Test is disabled on iOS < 16 and broken on iOS 16. +- (void)DISABLED_testStartCaptureSetsOutputDimensionsInvalidPixelFormat { + id expectedDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + id captureDeviceInputMock = OCMClassMock([AVCaptureDeviceInput class]); + OCMStub([captureDeviceInputMock deviceInputWithDevice:_deviceMock error:[OCMArg setTo:nil]]) + .andReturn(expectedDeviceInputMock); + + OCMStub([_deviceMock lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES); + OCMStub([_deviceMock unlockForConfiguration]); + OCMStub([_captureSessionMock canAddInput:expectedDeviceInputMock]).andReturn(YES); + OCMStub([_captureSessionMock addInput:expectedDeviceInputMock]); + OCMStub([_captureSessionMock inputs]).andReturn(@[ expectedDeviceInputMock ]); + OCMStub([_captureSessionMock removeInput:expectedDeviceInputMock]); + OCMStub([_captureSessionMock startRunning]); + OCMStub([_captureSessionMock stopRunning]); + + id deviceFormatMock = OCMClassMock([AVCaptureDeviceFormat class]); + CMVideoFormatDescriptionRef formatDescription; + + int width = 110; + int height = 220; + FourCharCode pixelFormat = 0x18000000; + CMVideoFormatDescriptionCreate(nil, pixelFormat, width, height, nil, &formatDescription); + OCMStub([deviceFormatMock formatDescription]).andReturn(formatDescription); + + [_capturer startCaptureWithDevice:_deviceMock format:deviceFormatMock fps:30]; + + XCTestExpectation *expectation = [self expectationWithDescription:@"StopCompletion"]; + [_capturer stopCaptureWithCompletionHandler:^(void) { + [expectation fulfill]; + }]; + + [self waitForExpectationsWithTimeout:15 handler:nil]; + + OCMVerify([_captureSessionMock + addOutput:[OCMArg checkWithBlock:^BOOL(AVCaptureVideoDataOutput *output) { + if (@available(iOS 16, *)) { + XCTAssertEqual(width, [output.videoSettings[(id)kCVPixelBufferWidthKey] intValue]); + XCTAssertEqual(height, [output.videoSettings[(id)kCVPixelBufferHeightKey] intValue]); + } else { + XCTAssertEqual(0, [output.videoSettings[(id)kCVPixelBufferWidthKey] intValue]); + XCTAssertEqual(0, [output.videoSettings[(id)kCVPixelBufferHeightKey] intValue]); + } + XCTAssertEqual( + (FourCharCode)kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + [output.videoSettings[(id)kCVPixelBufferPixelFormatTypeKey] unsignedIntValue]); + return YES; + }]]); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCCertificateTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCCertificateTest.mm new file mode 100644 index 0000000000..bc1347336c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCCertificateTest.mm @@ -0,0 +1,73 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <vector> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCConfiguration+Private.h" +#import "api/peerconnection/RTCConfiguration.h" +#import "api/peerconnection/RTCIceServer.h" +#import "api/peerconnection/RTCMediaConstraints.h" +#import "api/peerconnection/RTCPeerConnection.h" +#import "api/peerconnection/RTCPeerConnectionFactory.h" +#import "helpers/NSString+StdString.h" + +@interface RTCCertificateTest : XCTestCase +@end + +@implementation RTCCertificateTest + +- (void)testCertificateIsUsedInConfig { + RTC_OBJC_TYPE(RTCConfiguration) *originalConfig = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + + NSArray *urlStrings = @[ @"stun:stun1.example.net" ]; + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:urlStrings]; + originalConfig.iceServers = @[ server ]; + + // Generate a new certificate. + RTC_OBJC_TYPE(RTCCertificate) *originalCertificate = [RTC_OBJC_TYPE(RTCCertificate) + generateCertificateWithParams:@{@"expires" : @100000, @"name" : @"RSASSA-PKCS1-v1_5"}]; + + // Store certificate in configuration. + originalConfig.certificate = originalCertificate; + + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + + // Create PeerConnection with this certificate. + RTC_OBJC_TYPE(RTCPeerConnection) *peerConnection = + [factory peerConnectionWithConfiguration:originalConfig constraints:contraints delegate:nil]; + + // Retrieve certificate from the configuration. + RTC_OBJC_TYPE(RTCConfiguration) *retrievedConfig = peerConnection.configuration; + + // Extract PEM strings from original certificate. + std::string originalPrivateKeyField = [[originalCertificate private_key] UTF8String]; + std::string originalCertificateField = [[originalCertificate certificate] UTF8String]; + + // Extract PEM strings from certificate retrieved from configuration. + RTC_OBJC_TYPE(RTCCertificate) *retrievedCertificate = retrievedConfig.certificate; + std::string retrievedPrivateKeyField = [[retrievedCertificate private_key] UTF8String]; + std::string retrievedCertificateField = [[retrievedCertificate certificate] UTF8String]; + + // Check that the original certificate and retrieved certificate match. + EXPECT_EQ(originalPrivateKeyField, retrievedPrivateKeyField); + EXPECT_EQ(retrievedCertificateField, retrievedCertificateField); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCConfigurationTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCConfigurationTest.mm new file mode 100644 index 0000000000..18cc97191e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCConfigurationTest.mm @@ -0,0 +1,162 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <vector> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCConfiguration+Private.h" +#import "api/peerconnection/RTCConfiguration.h" +#import "api/peerconnection/RTCIceServer.h" +#import "helpers/NSString+StdString.h" + +@interface RTCConfigurationTest : XCTestCase +@end + +@implementation RTCConfigurationTest + +- (void)testConversionToNativeConfiguration { + NSArray *urlStrings = @[ @"stun:stun1.example.net" ]; + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:urlStrings]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.iceServers = @[ server ]; + config.iceTransportPolicy = RTCIceTransportPolicyRelay; + config.bundlePolicy = RTCBundlePolicyMaxBundle; + config.rtcpMuxPolicy = RTCRtcpMuxPolicyNegotiate; + config.tcpCandidatePolicy = RTCTcpCandidatePolicyDisabled; + config.candidateNetworkPolicy = RTCCandidateNetworkPolicyLowCost; + const int maxPackets = 60; + const int timeout = 1; + const int interval = 2; + config.audioJitterBufferMaxPackets = maxPackets; + config.audioJitterBufferFastAccelerate = YES; + config.iceConnectionReceivingTimeout = timeout; + config.iceBackupCandidatePairPingInterval = interval; + config.continualGatheringPolicy = + RTCContinualGatheringPolicyGatherContinually; + config.shouldPruneTurnPorts = YES; + config.cryptoOptions = + [[RTC_OBJC_TYPE(RTCCryptoOptions) alloc] initWithSrtpEnableGcmCryptoSuites:YES + srtpEnableAes128Sha1_32CryptoCipher:YES + srtpEnableEncryptedRtpHeaderExtensions:YES + sframeRequireFrameEncryption:YES]; + config.rtcpAudioReportIntervalMs = 2500; + config.rtcpVideoReportIntervalMs = 3750; + + std::unique_ptr<webrtc::PeerConnectionInterface::RTCConfiguration> + nativeConfig([config createNativeConfiguration]); + EXPECT_TRUE(nativeConfig.get()); + EXPECT_EQ(1u, nativeConfig->servers.size()); + webrtc::PeerConnectionInterface::IceServer nativeServer = + nativeConfig->servers.front(); + EXPECT_EQ(1u, nativeServer.urls.size()); + EXPECT_EQ("stun:stun1.example.net", nativeServer.urls.front()); + + EXPECT_EQ(webrtc::PeerConnectionInterface::kRelay, nativeConfig->type); + EXPECT_EQ(webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle, + nativeConfig->bundle_policy); + EXPECT_EQ(webrtc::PeerConnectionInterface::kRtcpMuxPolicyNegotiate, + nativeConfig->rtcp_mux_policy); + EXPECT_EQ(webrtc::PeerConnectionInterface::kTcpCandidatePolicyDisabled, + nativeConfig->tcp_candidate_policy); + EXPECT_EQ(webrtc::PeerConnectionInterface::kCandidateNetworkPolicyLowCost, + nativeConfig->candidate_network_policy); + EXPECT_EQ(maxPackets, nativeConfig->audio_jitter_buffer_max_packets); + EXPECT_EQ(true, nativeConfig->audio_jitter_buffer_fast_accelerate); + EXPECT_EQ(timeout, nativeConfig->ice_connection_receiving_timeout); + EXPECT_EQ(interval, nativeConfig->ice_backup_candidate_pair_ping_interval); + EXPECT_EQ(webrtc::PeerConnectionInterface::GATHER_CONTINUALLY, + nativeConfig->continual_gathering_policy); + EXPECT_EQ(true, nativeConfig->prune_turn_ports); + EXPECT_EQ(true, nativeConfig->crypto_options->srtp.enable_gcm_crypto_suites); + EXPECT_EQ(true, nativeConfig->crypto_options->srtp.enable_aes128_sha1_32_crypto_cipher); + EXPECT_EQ(true, nativeConfig->crypto_options->srtp.enable_encrypted_rtp_header_extensions); + EXPECT_EQ(true, nativeConfig->crypto_options->sframe.require_frame_encryption); + EXPECT_EQ(2500, nativeConfig->audio_rtcp_report_interval_ms()); + EXPECT_EQ(3750, nativeConfig->video_rtcp_report_interval_ms()); +} + +- (void)testNativeConversionToConfiguration { + NSArray *urlStrings = @[ @"stun:stun1.example.net" ]; + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:urlStrings]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.iceServers = @[ server ]; + config.iceTransportPolicy = RTCIceTransportPolicyRelay; + config.bundlePolicy = RTCBundlePolicyMaxBundle; + config.rtcpMuxPolicy = RTCRtcpMuxPolicyNegotiate; + config.tcpCandidatePolicy = RTCTcpCandidatePolicyDisabled; + config.candidateNetworkPolicy = RTCCandidateNetworkPolicyLowCost; + const int maxPackets = 60; + const int timeout = 1; + const int interval = 2; + config.audioJitterBufferMaxPackets = maxPackets; + config.audioJitterBufferFastAccelerate = YES; + config.iceConnectionReceivingTimeout = timeout; + config.iceBackupCandidatePairPingInterval = interval; + config.continualGatheringPolicy = + RTCContinualGatheringPolicyGatherContinually; + config.shouldPruneTurnPorts = YES; + config.cryptoOptions = + [[RTC_OBJC_TYPE(RTCCryptoOptions) alloc] initWithSrtpEnableGcmCryptoSuites:YES + srtpEnableAes128Sha1_32CryptoCipher:NO + srtpEnableEncryptedRtpHeaderExtensions:NO + sframeRequireFrameEncryption:NO]; + config.rtcpAudioReportIntervalMs = 1500; + config.rtcpVideoReportIntervalMs = 2150; + + webrtc::PeerConnectionInterface::RTCConfiguration *nativeConfig = + [config createNativeConfiguration]; + RTC_OBJC_TYPE(RTCConfiguration) *newConfig = + [[RTC_OBJC_TYPE(RTCConfiguration) alloc] initWithNativeConfiguration:*nativeConfig]; + EXPECT_EQ([config.iceServers count], newConfig.iceServers.count); + RTC_OBJC_TYPE(RTCIceServer) *newServer = newConfig.iceServers[0]; + RTC_OBJC_TYPE(RTCIceServer) *origServer = config.iceServers[0]; + EXPECT_EQ(origServer.urlStrings.count, server.urlStrings.count); + std::string origUrl = origServer.urlStrings.firstObject.UTF8String; + std::string url = newServer.urlStrings.firstObject.UTF8String; + EXPECT_EQ(origUrl, url); + + EXPECT_EQ(config.iceTransportPolicy, newConfig.iceTransportPolicy); + EXPECT_EQ(config.bundlePolicy, newConfig.bundlePolicy); + EXPECT_EQ(config.rtcpMuxPolicy, newConfig.rtcpMuxPolicy); + EXPECT_EQ(config.tcpCandidatePolicy, newConfig.tcpCandidatePolicy); + EXPECT_EQ(config.candidateNetworkPolicy, newConfig.candidateNetworkPolicy); + EXPECT_EQ(config.audioJitterBufferMaxPackets, newConfig.audioJitterBufferMaxPackets); + EXPECT_EQ(config.audioJitterBufferFastAccelerate, newConfig.audioJitterBufferFastAccelerate); + EXPECT_EQ(config.iceConnectionReceivingTimeout, newConfig.iceConnectionReceivingTimeout); + EXPECT_EQ(config.iceBackupCandidatePairPingInterval, + newConfig.iceBackupCandidatePairPingInterval); + EXPECT_EQ(config.continualGatheringPolicy, newConfig.continualGatheringPolicy); + EXPECT_EQ(config.shouldPruneTurnPorts, newConfig.shouldPruneTurnPorts); + EXPECT_EQ(config.cryptoOptions.srtpEnableGcmCryptoSuites, + newConfig.cryptoOptions.srtpEnableGcmCryptoSuites); + EXPECT_EQ(config.cryptoOptions.srtpEnableAes128Sha1_32CryptoCipher, + newConfig.cryptoOptions.srtpEnableAes128Sha1_32CryptoCipher); + EXPECT_EQ(config.cryptoOptions.srtpEnableEncryptedRtpHeaderExtensions, + newConfig.cryptoOptions.srtpEnableEncryptedRtpHeaderExtensions); + EXPECT_EQ(config.cryptoOptions.sframeRequireFrameEncryption, + newConfig.cryptoOptions.sframeRequireFrameEncryption); + EXPECT_EQ(config.rtcpAudioReportIntervalMs, newConfig.rtcpAudioReportIntervalMs); + EXPECT_EQ(config.rtcpVideoReportIntervalMs, newConfig.rtcpVideoReportIntervalMs); +} + +- (void)testDefaultValues { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + EXPECT_EQ(config.cryptoOptions, nil); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCDataChannelConfigurationTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCDataChannelConfigurationTest.mm new file mode 100644 index 0000000000..ccebd74198 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCDataChannelConfigurationTest.mm @@ -0,0 +1,51 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCDataChannelConfiguration+Private.h" +#import "api/peerconnection/RTCDataChannelConfiguration.h" +#import "helpers/NSString+StdString.h" + +@interface RTCDataChannelConfigurationTest : XCTestCase +@end + +@implementation RTCDataChannelConfigurationTest + +- (void)testConversionToNativeDataChannelInit { + BOOL isOrdered = NO; + int maxPacketLifeTime = 5; + int maxRetransmits = 4; + BOOL isNegotiated = YES; + int channelId = 4; + NSString *protocol = @"protocol"; + + RTC_OBJC_TYPE(RTCDataChannelConfiguration) *dataChannelConfig = + [[RTC_OBJC_TYPE(RTCDataChannelConfiguration) alloc] init]; + dataChannelConfig.isOrdered = isOrdered; + dataChannelConfig.maxPacketLifeTime = maxPacketLifeTime; + dataChannelConfig.maxRetransmits = maxRetransmits; + dataChannelConfig.isNegotiated = isNegotiated; + dataChannelConfig.channelId = channelId; + dataChannelConfig.protocol = protocol; + + webrtc::DataChannelInit nativeInit = dataChannelConfig.nativeDataChannelInit; + EXPECT_EQ(isOrdered, nativeInit.ordered); + EXPECT_EQ(maxPacketLifeTime, nativeInit.maxRetransmitTime); + EXPECT_EQ(maxRetransmits, nativeInit.maxRetransmits); + EXPECT_EQ(isNegotiated, nativeInit.negotiated); + EXPECT_EQ(channelId, nativeInit.id); + EXPECT_EQ(protocol.stdString, nativeInit.protocol); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCDoNotPutCPlusPlusInFrameworkHeaders_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCDoNotPutCPlusPlusInFrameworkHeaders_xctest.m new file mode 100644 index 0000000000..02bef9bfb7 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCDoNotPutCPlusPlusInFrameworkHeaders_xctest.m @@ -0,0 +1,30 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <XCTest/XCTest.h> + +#import <Foundation/Foundation.h> + +#import <WebRTC/WebRTC.h> + +@interface RTCDoNotPutCPlusPlusInFrameworkHeaders : XCTestCase +@end + +@implementation RTCDoNotPutCPlusPlusInFrameworkHeaders + +- (void)testNoCPlusPlusInFrameworkHeaders { + NSString *fullPath = [NSString stringWithFormat:@"%s", __FILE__]; + NSString *extension = fullPath.pathExtension; + + XCTAssertEqualObjects( + @"m", extension, @"Do not rename %@. It should end with .m.", fullPath.lastPathComponent); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCEncodedImage_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCEncodedImage_xctest.mm new file mode 100644 index 0000000000..84804fee87 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCEncodedImage_xctest.mm @@ -0,0 +1,55 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "api/peerconnection/RTCEncodedImage+Private.h" + +#import <XCTest/XCTest.h> + +@interface RTCEncodedImageTests : XCTestCase +@end + +@implementation RTCEncodedImageTests + +- (void)testInitializedWithNativeEncodedImage { + const auto encoded_data = webrtc::EncodedImageBuffer::Create(); + webrtc::EncodedImage encoded_image; + encoded_image.SetEncodedData(encoded_data); + + RTC_OBJC_TYPE(RTCEncodedImage) *encodedImage = + [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] initWithNativeEncodedImage:encoded_image]; + + XCTAssertEqual([encodedImage nativeEncodedImage].GetEncodedData(), encoded_data); +} + +- (void)testInitWithNSData { + NSData *bufferData = [NSData data]; + RTC_OBJC_TYPE(RTCEncodedImage) *encodedImage = [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] init]; + encodedImage.buffer = bufferData; + + webrtc::EncodedImage result_encoded_image = [encodedImage nativeEncodedImage]; + XCTAssertTrue(result_encoded_image.GetEncodedData() != nullptr); + XCTAssertEqual(result_encoded_image.GetEncodedData()->data(), bufferData.bytes); +} + +- (void)testRetainsNativeEncodedImage { + RTC_OBJC_TYPE(RTCEncodedImage) * encodedImage; + { + const auto encoded_data = webrtc::EncodedImageBuffer::Create(); + webrtc::EncodedImage encoded_image; + encoded_image.SetEncodedData(encoded_data); + encodedImage = + [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] initWithNativeEncodedImage:encoded_image]; + } + webrtc::EncodedImage result_encoded_image = [encodedImage nativeEncodedImage]; + XCTAssertTrue(result_encoded_image.GetEncodedData() != nullptr); + XCTAssertTrue(result_encoded_image.GetEncodedData()->data() != nullptr); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCFileVideoCapturer_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCFileVideoCapturer_xctest.mm new file mode 100644 index 0000000000..2407c88c1a --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCFileVideoCapturer_xctest.mm @@ -0,0 +1,114 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "components/capturer/RTCFileVideoCapturer.h" + +#import <XCTest/XCTest.h> + +#include "rtc_base/gunit.h" + +NSString *const kTestFileName = @"foreman.mp4"; +static const int kTestTimeoutMs = 5 * 1000; // 5secs. + +@interface MockCapturerDelegate : NSObject <RTC_OBJC_TYPE (RTCVideoCapturerDelegate)> + +@property(nonatomic, assign) NSInteger capturedFramesCount; + +@end + +@implementation MockCapturerDelegate +@synthesize capturedFramesCount = _capturedFramesCount; + +- (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) *)capturer + didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame { + self.capturedFramesCount++; +} + +@end + +NS_CLASS_AVAILABLE_IOS(10) +@interface RTCFileVideoCapturerTests : XCTestCase + +@property(nonatomic, strong) RTC_OBJC_TYPE(RTCFileVideoCapturer) * capturer; +@property(nonatomic, strong) MockCapturerDelegate *mockDelegate; + +@end + +@implementation RTCFileVideoCapturerTests +@synthesize capturer = _capturer; +@synthesize mockDelegate = _mockDelegate; + +- (void)setUp { + self.mockDelegate = [[MockCapturerDelegate alloc] init]; + self.capturer = [[RTC_OBJC_TYPE(RTCFileVideoCapturer) alloc] initWithDelegate:self.mockDelegate]; +} + +- (void)tearDown { + self.capturer = nil; + self.mockDelegate = nil; +} + +- (void)testCaptureWhenFileNotInBundle { + __block BOOL errorOccured = NO; + + RTCFileVideoCapturerErrorBlock errorBlock = ^void(NSError *error) { + errorOccured = YES; + }; + + [self.capturer startCapturingFromFileNamed:@"not_in_bundle.mov" onError:errorBlock]; + ASSERT_TRUE_WAIT(errorOccured, kTestTimeoutMs); +} + +- (void)testSecondStartCaptureCallFails { + __block BOOL secondError = NO; + + RTCFileVideoCapturerErrorBlock firstErrorBlock = ^void(NSError *error) { + // This block should never be called. + NSLog(@"Error: %@", [error userInfo]); + ASSERT_TRUE(false); + }; + + RTCFileVideoCapturerErrorBlock secondErrorBlock = ^void(NSError *error) { + secondError = YES; + }; + + [self.capturer startCapturingFromFileNamed:kTestFileName onError:firstErrorBlock]; + [self.capturer startCapturingFromFileNamed:kTestFileName onError:secondErrorBlock]; + + ASSERT_TRUE_WAIT(secondError, kTestTimeoutMs); +} + +- (void)testStartStopCapturer { +#if defined(__IPHONE_11_0) && (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_11_0) + if (@available(iOS 10, *)) { + [self.capturer startCapturingFromFileNamed:kTestFileName onError:nil]; + + __block BOOL done = NO; + __block NSInteger capturedFrames = -1; + NSInteger capturedFramesAfterStop = -1; + + // We're dispatching the `stopCapture` with delay to ensure the capturer has + // had the chance to capture several frames. + dispatch_time_t captureDelay = dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC); // 2secs. + dispatch_after(captureDelay, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + capturedFrames = self.mockDelegate.capturedFramesCount; + [self.capturer stopCapture]; + done = YES; + }); + WAIT(done, kTestTimeoutMs); + + capturedFramesAfterStop = self.mockDelegate.capturedFramesCount; + ASSERT_TRUE(capturedFrames != -1); + ASSERT_EQ(capturedFrames, capturedFramesAfterStop); + } +#endif +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCH264ProfileLevelId_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCH264ProfileLevelId_xctest.m new file mode 100644 index 0000000000..ec9dc41796 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCH264ProfileLevelId_xctest.m @@ -0,0 +1,48 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "components/video_codec/RTCH264ProfileLevelId.h" + +#import <XCTest/XCTest.h> + +@interface RTCH264ProfileLevelIdTests : XCTestCase + +@end + +static NSString *level31ConstrainedHigh = @"640c1f"; +static NSString *level31ConstrainedBaseline = @"42e01f"; + +@implementation RTCH264ProfileLevelIdTests + +- (void)testInitWithString { + RTC_OBJC_TYPE(RTCH264ProfileLevelId) *profileLevelId = + [[RTC_OBJC_TYPE(RTCH264ProfileLevelId) alloc] initWithHexString:level31ConstrainedHigh]; + XCTAssertEqual(profileLevelId.profile, RTCH264ProfileConstrainedHigh); + XCTAssertEqual(profileLevelId.level, RTCH264Level3_1); + + profileLevelId = + [[RTC_OBJC_TYPE(RTCH264ProfileLevelId) alloc] initWithHexString:level31ConstrainedBaseline]; + XCTAssertEqual(profileLevelId.profile, RTCH264ProfileConstrainedBaseline); + XCTAssertEqual(profileLevelId.level, RTCH264Level3_1); +} + +- (void)testInitWithProfileAndLevel { + RTC_OBJC_TYPE(RTCH264ProfileLevelId) *profileLevelId = + [[RTC_OBJC_TYPE(RTCH264ProfileLevelId) alloc] initWithProfile:RTCH264ProfileConstrainedHigh + level:RTCH264Level3_1]; + XCTAssertEqualObjects(profileLevelId.hexString, level31ConstrainedHigh); + + profileLevelId = [[RTC_OBJC_TYPE(RTCH264ProfileLevelId) alloc] + initWithProfile:RTCH264ProfileConstrainedBaseline + level:RTCH264Level3_1]; + XCTAssertEqualObjects(profileLevelId.hexString, level31ConstrainedBaseline); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCIceCandidateTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCIceCandidateTest.mm new file mode 100644 index 0000000000..d781488286 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCIceCandidateTest.mm @@ -0,0 +1,61 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <memory> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCIceCandidate+Private.h" +#import "api/peerconnection/RTCIceCandidate.h" +#import "helpers/NSString+StdString.h" + +@interface RTCIceCandidateTest : XCTestCase +@end + +@implementation RTCIceCandidateTest + +- (void)testCandidate { + NSString *sdp = @"candidate:4025901590 1 udp 2122265343 " + "fdff:2642:12a6:fe38:c001:beda:fcf9:51aa " + "59052 typ host generation 0"; + + RTC_OBJC_TYPE(RTCIceCandidate) *candidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithSdp:sdp sdpMLineIndex:0 sdpMid:@"audio"]; + + std::unique_ptr<webrtc::IceCandidateInterface> nativeCandidate = + candidate.nativeCandidate; + EXPECT_EQ("audio", nativeCandidate->sdp_mid()); + EXPECT_EQ(0, nativeCandidate->sdp_mline_index()); + + std::string sdpString; + nativeCandidate->ToString(&sdpString); + EXPECT_EQ(sdp.stdString, sdpString); +} + +- (void)testInitFromNativeCandidate { + std::string sdp("candidate:4025901590 1 udp 2122265343 " + "fdff:2642:12a6:fe38:c001:beda:fcf9:51aa " + "59052 typ host generation 0"); + std::unique_ptr<webrtc::IceCandidateInterface> nativeCandidate( + webrtc::CreateIceCandidate("audio", 0, sdp, nullptr)); + + RTC_OBJC_TYPE(RTCIceCandidate) *iceCandidate = + [[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithNativeCandidate:nativeCandidate.get()]; + EXPECT_NE(nativeCandidate.get(), iceCandidate.nativeCandidate.get()); + EXPECT_TRUE([@"audio" isEqualToString:iceCandidate.sdpMid]); + EXPECT_EQ(0, iceCandidate.sdpMLineIndex); + + EXPECT_EQ(sdp, iceCandidate.sdp.stdString); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCIceServerTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCIceServerTest.mm new file mode 100644 index 0000000000..772653c4dc --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCIceServerTest.mm @@ -0,0 +1,136 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <vector> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCIceServer+Private.h" +#import "api/peerconnection/RTCIceServer.h" +#import "helpers/NSString+StdString.h" + +@interface RTCIceServerTest : XCTestCase +@end + +@implementation RTCIceServerTest + +- (void)testOneURLServer { + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:@[ @"stun:stun1.example.net" ]]; + + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(1u, iceStruct.urls.size()); + EXPECT_EQ("stun:stun1.example.net", iceStruct.urls.front()); + EXPECT_EQ("", iceStruct.username); + EXPECT_EQ("", iceStruct.password); +} + +- (void)testTwoURLServer { + RTC_OBJC_TYPE(RTCIceServer) *server = [[RTC_OBJC_TYPE(RTCIceServer) alloc] + initWithURLStrings:@[ @"turn1:turn1.example.net", @"turn2:turn2.example.net" ]]; + + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(2u, iceStruct.urls.size()); + EXPECT_EQ("turn1:turn1.example.net", iceStruct.urls.front()); + EXPECT_EQ("turn2:turn2.example.net", iceStruct.urls.back()); + EXPECT_EQ("", iceStruct.username); + EXPECT_EQ("", iceStruct.password); +} + +- (void)testPasswordCredential { + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:@[ @"turn1:turn1.example.net" ] + username:@"username" + credential:@"credential"]; + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(1u, iceStruct.urls.size()); + EXPECT_EQ("turn1:turn1.example.net", iceStruct.urls.front()); + EXPECT_EQ("username", iceStruct.username); + EXPECT_EQ("credential", iceStruct.password); +} + +- (void)testHostname { + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:@[ @"turn1:turn1.example.net" ] + username:@"username" + credential:@"credential" + tlsCertPolicy:RTCTlsCertPolicySecure + hostname:@"hostname"]; + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(1u, iceStruct.urls.size()); + EXPECT_EQ("turn1:turn1.example.net", iceStruct.urls.front()); + EXPECT_EQ("username", iceStruct.username); + EXPECT_EQ("credential", iceStruct.password); + EXPECT_EQ("hostname", iceStruct.hostname); +} + +- (void)testTlsAlpnProtocols { + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:@[ @"turn1:turn1.example.net" ] + username:@"username" + credential:@"credential" + tlsCertPolicy:RTCTlsCertPolicySecure + hostname:@"hostname" + tlsAlpnProtocols:@[ @"proto1", @"proto2" ]]; + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(1u, iceStruct.urls.size()); + EXPECT_EQ("turn1:turn1.example.net", iceStruct.urls.front()); + EXPECT_EQ("username", iceStruct.username); + EXPECT_EQ("credential", iceStruct.password); + EXPECT_EQ("hostname", iceStruct.hostname); + EXPECT_EQ(2u, iceStruct.tls_alpn_protocols.size()); +} + +- (void)testTlsEllipticCurves { + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:@[ @"turn1:turn1.example.net" ] + username:@"username" + credential:@"credential" + tlsCertPolicy:RTCTlsCertPolicySecure + hostname:@"hostname" + tlsAlpnProtocols:@[ @"proto1", @"proto2" ] + tlsEllipticCurves:@[ @"curve1", @"curve2" ]]; + webrtc::PeerConnectionInterface::IceServer iceStruct = server.nativeServer; + EXPECT_EQ(1u, iceStruct.urls.size()); + EXPECT_EQ("turn1:turn1.example.net", iceStruct.urls.front()); + EXPECT_EQ("username", iceStruct.username); + EXPECT_EQ("credential", iceStruct.password); + EXPECT_EQ("hostname", iceStruct.hostname); + EXPECT_EQ(2u, iceStruct.tls_alpn_protocols.size()); + EXPECT_EQ(2u, iceStruct.tls_elliptic_curves.size()); +} + +- (void)testInitFromNativeServer { + webrtc::PeerConnectionInterface::IceServer nativeServer; + nativeServer.username = "username"; + nativeServer.password = "password"; + nativeServer.urls.push_back("stun:stun.example.net"); + nativeServer.hostname = "hostname"; + nativeServer.tls_alpn_protocols.push_back("proto1"); + nativeServer.tls_alpn_protocols.push_back("proto2"); + nativeServer.tls_elliptic_curves.push_back("curve1"); + nativeServer.tls_elliptic_curves.push_back("curve2"); + + RTC_OBJC_TYPE(RTCIceServer) *iceServer = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithNativeServer:nativeServer]; + EXPECT_EQ(1u, iceServer.urlStrings.count); + EXPECT_EQ("stun:stun.example.net", + [NSString stdStringForString:iceServer.urlStrings.firstObject]); + EXPECT_EQ("username", [NSString stdStringForString:iceServer.username]); + EXPECT_EQ("password", [NSString stdStringForString:iceServer.credential]); + EXPECT_EQ("hostname", [NSString stdStringForString:iceServer.hostname]); + EXPECT_EQ(2u, iceServer.tlsAlpnProtocols.count); + EXPECT_EQ(2u, iceServer.tlsEllipticCurves.count); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCMTLVideoView_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCMTLVideoView_xctest.m new file mode 100644 index 0000000000..587a6b588f --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCMTLVideoView_xctest.m @@ -0,0 +1,299 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <XCTest/XCTest.h> + +#import <Foundation/Foundation.h> +#import <MetalKit/MetalKit.h> +#import <OCMock/OCMock.h> + +#import "components/renderer/metal/RTCMTLVideoView.h" + +#import "api/video_frame_buffer/RTCNativeI420Buffer.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/renderer/metal/RTCMTLNV12Renderer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +static size_t kBufferWidth = 200; +static size_t kBufferHeight = 200; + +// Extension of RTC_OBJC_TYPE(RTCMTLVideoView) for testing purposes. +@interface RTC_OBJC_TYPE (RTCMTLVideoView) +(Testing) + + @property(nonatomic, readonly) MTKView *metalView; + ++ (BOOL)isMetalAvailable; ++ (UIView *)createMetalView:(CGRect)frame; ++ (id<RTCMTLRenderer>)createNV12Renderer; ++ (id<RTCMTLRenderer>)createI420Renderer; +- (void)drawInMTKView:(id)view; +@end + +@interface RTCMTLVideoViewTests : XCTestCase +@property(nonatomic, strong) id classMock; +@property(nonatomic, strong) id rendererNV12Mock; +@property(nonatomic, strong) id rendererI420Mock; +@property(nonatomic, strong) id frameMock; +@end + +@implementation RTCMTLVideoViewTests + +@synthesize classMock = _classMock; +@synthesize rendererNV12Mock = _rendererNV12Mock; +@synthesize rendererI420Mock = _rendererI420Mock; +@synthesize frameMock = _frameMock; + +- (void)setUp { + self.classMock = OCMClassMock([RTC_OBJC_TYPE(RTCMTLVideoView) class]); + [self startMockingNilView]; +} + +- (void)tearDown { + [self.classMock stopMocking]; + [self.rendererI420Mock stopMocking]; + [self.rendererNV12Mock stopMocking]; + [self.frameMock stopMocking]; + self.classMock = nil; + self.rendererI420Mock = nil; + self.rendererNV12Mock = nil; + self.frameMock = nil; +} + +- (id)frameMockWithCVPixelBuffer:(BOOL)hasCVPixelBuffer { + id frameMock = OCMClassMock([RTC_OBJC_TYPE(RTCVideoFrame) class]); + if (hasCVPixelBuffer) { + CVPixelBufferRef pixelBufferRef; + CVPixelBufferCreate(kCFAllocatorDefault, + kBufferWidth, + kBufferHeight, + kCVPixelFormatType_420YpCbCr8Planar, + nil, + &pixelBufferRef); + OCMStub([frameMock buffer]) + .andReturn([[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]); + } else { + OCMStub([frameMock buffer]) + .andReturn([[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithWidth:kBufferWidth + height:kBufferHeight]); + } + OCMStub([((RTC_OBJC_TYPE(RTCVideoFrame) *)frameMock) width]).andReturn(kBufferWidth); + OCMStub([((RTC_OBJC_TYPE(RTCVideoFrame) *)frameMock) height]).andReturn(kBufferHeight); + OCMStub([frameMock timeStampNs]).andReturn(arc4random_uniform(INT_MAX)); + return frameMock; +} + +- (id)rendererMockWithSuccessfulSetup:(BOOL)success { + id rendererMock = OCMClassMock([RTCMTLRenderer class]); + OCMStub([rendererMock addRenderingDestination:[OCMArg any]]).andReturn(success); + return rendererMock; +} + +- (void)startMockingNilView { + // Use OCMock 2 syntax here until OCMock is upgraded to 3.4 + [[[self.classMock stub] andReturn:nil] createMetalView:CGRectZero]; +} + +#pragma mark - Test cases + +- (void)testInitAssertsIfMetalUnavailabe { + // given + OCMStub([self.classMock isMetalAvailable]).andReturn(NO); + + // when + BOOL asserts = NO; + @try { + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectZero]; + (void)realView; + } @catch (NSException *ex) { + asserts = YES; + } + + XCTAssertTrue(asserts); +} + +- (void)testRTCVideoRenderNilFrameCallback { + // given + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + self.frameMock = OCMClassMock([RTC_OBJC_TYPE(RTCVideoFrame) class]); + + [[self.frameMock reject] buffer]; + [[self.classMock reject] createNV12Renderer]; + [[self.classMock reject] createI420Renderer]; + + // when + [realView renderFrame:nil]; + [realView drawInMTKView:realView.metalView]; + + // then + [self.frameMock verify]; + [self.classMock verify]; +} + +- (void)testRTCVideoRenderFrameCallbackI420 { + // given + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + self.rendererI420Mock = [self rendererMockWithSuccessfulSetup:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:NO]; + + OCMExpect([self.rendererI420Mock drawFrame:self.frameMock]); + OCMExpect([self.classMock createI420Renderer]).andReturn(self.rendererI420Mock); + [[self.classMock reject] createNV12Renderer]; + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + + // when + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + // then + [self.rendererI420Mock verify]; + [self.classMock verify]; +} + +- (void)testRTCVideoRenderFrameCallbackNV12 { + // given + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock); + [[self.classMock reject] createI420Renderer]; + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + + // when + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + // then + [self.rendererNV12Mock verify]; + [self.classMock verify]; +} + +- (void)testRTCVideoRenderWorksAfterReconstruction { + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock); + [[self.classMock reject] createI420Renderer]; + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + [self.rendererNV12Mock verify]; + [self.classMock verify]; + + // Recreate view. + realView = [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + // View hould reinit renderer. + OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock); + + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + [self.rendererNV12Mock verify]; + [self.classMock verify]; +} + +- (void)testDontRedrawOldFrame { + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock); + [[self.classMock reject] createI420Renderer]; + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + [self.rendererNV12Mock verify]; + [self.classMock verify]; + + [[self.rendererNV12Mock reject] drawFrame:[OCMArg any]]; + + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + [self.rendererNV12Mock verify]; +} + +- (void)testDoDrawNewFrame { + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock); + [[self.classMock reject] createI420Renderer]; + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + [self.rendererNV12Mock verify]; + [self.classMock verify]; + + // Get new frame. + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; + OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); + + [realView renderFrame:self.frameMock]; + [realView drawInMTKView:realView.metalView]; + + [self.rendererNV12Mock verify]; +} + +- (void)testReportsSizeChangesToDelegate { + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + + id delegateMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoViewDelegate))); + CGSize size = CGSizeMake(640, 480); + OCMExpect([delegateMock videoView:[OCMArg any] didChangeVideoSize:size]); + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = + [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] initWithFrame:CGRectMake(0, 0, 640, 480)]; + realView.delegate = delegateMock; + [realView setSize:size]; + + // Delegate method is invoked with a dispatch_async. + OCMVerifyAllWithDelay(delegateMock, 1); +} + +// TODO(b/298960678): Fix test expectations. +- (void)DISABLED_testSetContentMode { + OCMStub([self.classMock isMetalAvailable]).andReturn(YES); + id metalKitView = OCMClassMock([MTKView class]); + [[[[self.classMock stub] ignoringNonObjectArgs] andReturn:metalKitView] + createMetalView:CGRectZero]; + OCMExpect([metalKitView setContentMode:UIViewContentModeScaleAspectFill]); + + RTC_OBJC_TYPE(RTCMTLVideoView) *realView = [[RTC_OBJC_TYPE(RTCMTLVideoView) alloc] init]; + [realView setVideoContentMode:UIViewContentModeScaleAspectFill]; + + OCMVerifyAll(metalKitView); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCMediaConstraintsTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCMediaConstraintsTest.mm new file mode 100644 index 0000000000..6ed7859ba1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCMediaConstraintsTest.mm @@ -0,0 +1,58 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <memory> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCMediaConstraints+Private.h" +#import "api/peerconnection/RTCMediaConstraints.h" +#import "helpers/NSString+StdString.h" + +@interface RTCMediaConstraintsTests : XCTestCase +@end + +@implementation RTCMediaConstraintsTests + +- (void)testMediaConstraints { + NSDictionary *mandatory = @{@"key1": @"value1", @"key2": @"value2"}; + NSDictionary *optional = @{@"key3": @"value3", @"key4": @"value4"}; + + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:mandatory + optionalConstraints:optional]; + std::unique_ptr<webrtc::MediaConstraints> nativeConstraints = + [constraints nativeConstraints]; + + webrtc::MediaConstraints::Constraints nativeMandatory = nativeConstraints->GetMandatory(); + [self expectConstraints:mandatory inNativeConstraints:nativeMandatory]; + + webrtc::MediaConstraints::Constraints nativeOptional = nativeConstraints->GetOptional(); + [self expectConstraints:optional inNativeConstraints:nativeOptional]; +} + +- (void)expectConstraints:(NSDictionary *)constraints + inNativeConstraints:(webrtc::MediaConstraints::Constraints)nativeConstraints { + EXPECT_EQ(constraints.count, nativeConstraints.size()); + + for (NSString *key in constraints) { + NSString *value = [constraints objectForKey:key]; + + std::string nativeValue; + bool found = nativeConstraints.FindFirst(key.stdString, &nativeValue); + EXPECT_TRUE(found); + EXPECT_EQ(value.stdString, nativeValue); + } +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCNV12TextureCache_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCNV12TextureCache_xctest.m new file mode 100644 index 0000000000..7bdc538f67 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCNV12TextureCache_xctest.m @@ -0,0 +1,55 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <CoreVideo/CoreVideo.h> +#import <Foundation/Foundation.h> +#import <GLKit/GLKit.h> +#import <XCTest/XCTest.h> + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/renderer/opengl/RTCNV12TextureCache.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" + +@interface RTCNV12TextureCacheTests : XCTestCase +@end + +@implementation RTCNV12TextureCacheTests { + EAGLContext *_glContext; + RTCNV12TextureCache *_nv12TextureCache; +} + +- (void)setUp { + [super setUp]; + _glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; + if (!_glContext) { + _glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; + } + _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; +} + +- (void)tearDown { + _nv12TextureCache = nil; + _glContext = nil; + [super tearDown]; +} + +- (void)testNV12TextureCacheDoesNotCrashOnEmptyFrame { + CVPixelBufferRef nullPixelBuffer = NULL; + RTC_OBJC_TYPE(RTCCVPixelBuffer) *badFrameBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:nullPixelBuffer]; + RTC_OBJC_TYPE(RTCVideoFrame) *badFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:badFrameBuffer + rotation:RTCVideoRotation_0 + timeStampNs:0]; + [_nv12TextureCache uploadFrameToTextures:badFrame]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm new file mode 100644 index 0000000000..5ba5a52a53 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactoryBuilderTest.mm @@ -0,0 +1,72 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> +#ifdef __cplusplus +extern "C" { +#endif +#import <OCMock/OCMock.h> +#ifdef __cplusplus +} +#endif +#import "api/peerconnection/RTCPeerConnectionFactory+Native.h" +#import "api/peerconnection/RTCPeerConnectionFactoryBuilder+DefaultComponents.h" +#import "api/peerconnection/RTCPeerConnectionFactoryBuilder.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" + +#include "rtc_base/gunit.h" +#include "rtc_base/system/unused.h" + +@interface RTCPeerConnectionFactoryBuilderTests : XCTestCase +@end + +@implementation RTCPeerConnectionFactoryBuilderTests + +- (void)testBuilder { + id factoryMock = OCMStrictClassMock([RTC_OBJC_TYPE(RTCPeerConnectionFactory) class]); + OCMExpect([factoryMock alloc]).andReturn(factoryMock); + RTC_UNUSED([[[[factoryMock expect] andReturn:factoryMock] ignoringNonObjectArgs] + initWithNativeAudioEncoderFactory:nullptr + nativeAudioDecoderFactory:nullptr + nativeVideoEncoderFactory:nullptr + nativeVideoDecoderFactory:nullptr + audioDeviceModule:nullptr + audioProcessingModule:nullptr]); + RTCPeerConnectionFactoryBuilder* builder = [[RTCPeerConnectionFactoryBuilder alloc] init]; + RTC_OBJC_TYPE(RTCPeerConnectionFactory)* peerConnectionFactory = + [builder createPeerConnectionFactory]; + EXPECT_TRUE(peerConnectionFactory != nil); + OCMVerifyAll(factoryMock); +} + +- (void)testDefaultComponentsBuilder { + id factoryMock = OCMStrictClassMock([RTC_OBJC_TYPE(RTCPeerConnectionFactory) class]); + OCMExpect([factoryMock alloc]).andReturn(factoryMock); + RTC_UNUSED([[[[factoryMock expect] andReturn:factoryMock] ignoringNonObjectArgs] + initWithNativeAudioEncoderFactory:nullptr + nativeAudioDecoderFactory:nullptr + nativeVideoEncoderFactory:nullptr + nativeVideoDecoderFactory:nullptr + audioDeviceModule:nullptr + audioProcessingModule:nullptr]); + RTCPeerConnectionFactoryBuilder* builder = [RTCPeerConnectionFactoryBuilder defaultBuilder]; + RTC_OBJC_TYPE(RTCPeerConnectionFactory)* peerConnectionFactory = + [builder createPeerConnectionFactory]; + EXPECT_TRUE(peerConnectionFactory != nil); + OCMVerifyAll(factoryMock); +} +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactory_xctest.m b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactory_xctest.m new file mode 100644 index 0000000000..56c74971b6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionFactory_xctest.m @@ -0,0 +1,380 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "api/peerconnection/RTCAudioSource.h" +#import "api/peerconnection/RTCConfiguration.h" +#import "api/peerconnection/RTCDataChannel.h" +#import "api/peerconnection/RTCDataChannelConfiguration.h" +#import "api/peerconnection/RTCMediaConstraints.h" +#import "api/peerconnection/RTCMediaStreamTrack.h" +#import "api/peerconnection/RTCPeerConnection.h" +#import "api/peerconnection/RTCPeerConnectionFactory.h" +#import "api/peerconnection/RTCRtpReceiver.h" +#import "api/peerconnection/RTCRtpSender.h" +#import "api/peerconnection/RTCRtpTransceiver.h" +#import "api/peerconnection/RTCSessionDescription.h" +#import "api/peerconnection/RTCVideoSource.h" +#import "rtc_base/system/unused.h" + +#import <XCTest/XCTest.h> + +@interface RTCPeerConnectionFactoryTests : XCTestCase +@end + +@implementation RTCPeerConnectionFactoryTests + +- (void)testPeerConnectionLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCPeerConnection) * peerConnection; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + peerConnection = + [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + [peerConnection close]; + factory = nil; + } + peerConnection = nil; + } + + XCTAssertTrue(true, @"Expect test does not crash"); +} + +- (void)testMediaStreamLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCMediaStream) * mediaStream; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + mediaStream = [factory mediaStreamWithStreamId:@"mediaStream"]; + factory = nil; + } + mediaStream = nil; + RTC_UNUSED(mediaStream); + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testDataChannelLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCDataChannelConfiguration) *dataChannelConfig = + [[RTC_OBJC_TYPE(RTCDataChannelConfiguration) alloc] init]; + + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCPeerConnection) * peerConnection; + RTC_OBJC_TYPE(RTCDataChannel) * dataChannel; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + peerConnection = + [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + dataChannel = + [peerConnection dataChannelForLabel:@"test_channel" configuration:dataChannelConfig]; + XCTAssertNotNil(dataChannel); + [peerConnection close]; + peerConnection = nil; + factory = nil; + } + dataChannel = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testRTCRtpTransceiverLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCRtpTransceiverInit) *init = + [[RTC_OBJC_TYPE(RTCRtpTransceiverInit) alloc] init]; + + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCPeerConnection) * peerConnection; + RTC_OBJC_TYPE(RTCRtpTransceiver) * tranceiver; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + peerConnection = + [factory peerConnectionWithConfiguration:config constraints:contraints delegate:nil]; + tranceiver = [peerConnection addTransceiverOfType:RTCRtpMediaTypeAudio init:init]; + XCTAssertNotNil(tranceiver); + [peerConnection close]; + peerConnection = nil; + factory = nil; + } + tranceiver = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testRTCRtpSenderLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsPlanB; + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCPeerConnection) * peerConnection; + RTC_OBJC_TYPE(RTCRtpSender) * sender; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + peerConnection = + [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + sender = [peerConnection senderWithKind:kRTCMediaStreamTrackKindVideo streamId:@"stream"]; + XCTAssertNotNil(sender); + [peerConnection close]; + peerConnection = nil; + factory = nil; + } + sender = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testRTCRtpReceiverLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsPlanB; + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCPeerConnection) * pc1; + RTC_OBJC_TYPE(RTCPeerConnection) * pc2; + + NSArray<RTC_OBJC_TYPE(RTCRtpReceiver) *> *receivers1; + NSArray<RTC_OBJC_TYPE(RTCRtpReceiver) *> *receivers2; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + pc1 = [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + [pc1 senderWithKind:kRTCMediaStreamTrackKindAudio streamId:@"stream"]; + + pc2 = [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + [pc2 senderWithKind:kRTCMediaStreamTrackKindAudio streamId:@"stream"]; + + NSTimeInterval negotiationTimeout = 15; + XCTAssertTrue([self negotiatePeerConnection:pc1 + withPeerConnection:pc2 + negotiationTimeout:negotiationTimeout]); + + XCTAssertEqual(pc1.signalingState, RTCSignalingStateStable); + XCTAssertEqual(pc2.signalingState, RTCSignalingStateStable); + + receivers1 = pc1.receivers; + receivers2 = pc2.receivers; + XCTAssertTrue(receivers1.count > 0); + XCTAssertTrue(receivers2.count > 0); + [pc1 close]; + [pc2 close]; + pc1 = nil; + pc2 = nil; + factory = nil; + } + receivers1 = nil; + receivers2 = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testAudioSourceLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCAudioSource) * audioSource; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + audioSource = [factory audioSourceWithConstraints:nil]; + XCTAssertNotNil(audioSource); + factory = nil; + } + audioSource = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testVideoSourceLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCVideoSource) * videoSource; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + videoSource = [factory videoSource]; + XCTAssertNotNil(videoSource); + factory = nil; + } + videoSource = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testAudioTrackLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCAudioTrack) * audioTrack; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + audioTrack = [factory audioTrackWithTrackId:@"audioTrack"]; + XCTAssertNotNil(audioTrack); + factory = nil; + } + audioTrack = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testVideoTrackLifetime { + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + RTC_OBJC_TYPE(RTCVideoTrack) * videoTrack; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + videoTrack = [factory videoTrackWithSource:[factory videoSource] trackId:@"videoTrack"]; + XCTAssertNotNil(videoTrack); + factory = nil; + } + videoTrack = nil; + } + + XCTAssertTrue(true, "Expect test does not crash"); +} + +- (void)testRollback { + @autoreleasepool { + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + RTC_OBJC_TYPE(RTCMediaConstraints) *constraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{ + kRTCMediaConstraintsOfferToReceiveAudio : kRTCMediaConstraintsValueTrue + } + optionalConstraints:nil]; + + __block RTC_OBJC_TYPE(RTCPeerConnectionFactory) * factory; + __block RTC_OBJC_TYPE(RTCPeerConnection) * pc1; + RTC_OBJC_TYPE(RTCSessionDescription) *rollback = + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithType:RTCSdpTypeRollback sdp:@""]; + + @autoreleasepool { + factory = [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + pc1 = [factory peerConnectionWithConfiguration:config constraints:constraints delegate:nil]; + dispatch_semaphore_t negotiatedSem = dispatch_semaphore_create(0); + [pc1 offerForConstraints:constraints + completionHandler:^(RTC_OBJC_TYPE(RTCSessionDescription) * offer, NSError * error) { + XCTAssertNil(error); + XCTAssertNotNil(offer); + + __weak RTC_OBJC_TYPE(RTCPeerConnection) *weakPC1 = pc1; + [pc1 setLocalDescription:offer + completionHandler:^(NSError *error) { + XCTAssertNil(error); + [weakPC1 setLocalDescription:rollback + completionHandler:^(NSError *error) { + XCTAssertNil(error); + }]; + }]; + NSTimeInterval negotiationTimeout = 15; + dispatch_semaphore_wait( + negotiatedSem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(negotiationTimeout * NSEC_PER_SEC))); + + XCTAssertEqual(pc1.signalingState, RTCSignalingStateStable); + + [pc1 close]; + pc1 = nil; + factory = nil; + }]; + } + + XCTAssertTrue(true, "Expect test does not crash"); + } +} + +- (bool)negotiatePeerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)pc1 + withPeerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)pc2 + negotiationTimeout:(NSTimeInterval)timeout { + __weak RTC_OBJC_TYPE(RTCPeerConnection) *weakPC1 = pc1; + __weak RTC_OBJC_TYPE(RTCPeerConnection) *weakPC2 = pc2; + RTC_OBJC_TYPE(RTCMediaConstraints) *sdpConstraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{ + kRTCMediaConstraintsOfferToReceiveAudio : kRTCMediaConstraintsValueTrue + } + optionalConstraints:nil]; + + dispatch_semaphore_t negotiatedSem = dispatch_semaphore_create(0); + [weakPC1 offerForConstraints:sdpConstraints + completionHandler:^(RTC_OBJC_TYPE(RTCSessionDescription) * offer, NSError * error) { + XCTAssertNil(error); + XCTAssertNotNil(offer); + [weakPC1 + setLocalDescription:offer + completionHandler:^(NSError *error) { + XCTAssertNil(error); + [weakPC2 + setRemoteDescription:offer + completionHandler:^(NSError *error) { + XCTAssertNil(error); + [weakPC2 + answerForConstraints:sdpConstraints + completionHandler:^( + RTC_OBJC_TYPE(RTCSessionDescription) * answer, + NSError * error) { + XCTAssertNil(error); + XCTAssertNotNil(answer); + [weakPC2 + setLocalDescription:answer + completionHandler:^(NSError *error) { + XCTAssertNil(error); + [weakPC1 + setRemoteDescription:answer + completionHandler:^(NSError *error) { + XCTAssertNil(error); + dispatch_semaphore_signal(negotiatedSem); + }]; + }]; + }]; + }]; + }]; + }]; + + return 0 == + dispatch_semaphore_wait(negotiatedSem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(timeout * NSEC_PER_SEC))); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionTest.mm new file mode 100644 index 0000000000..9ca8403559 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCPeerConnectionTest.mm @@ -0,0 +1,204 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <memory> +#include <vector> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCConfiguration+Private.h" +#import "api/peerconnection/RTCConfiguration.h" +#import "api/peerconnection/RTCCryptoOptions.h" +#import "api/peerconnection/RTCIceCandidate.h" +#import "api/peerconnection/RTCIceServer.h" +#import "api/peerconnection/RTCMediaConstraints.h" +#import "api/peerconnection/RTCPeerConnection.h" +#import "api/peerconnection/RTCPeerConnectionFactory+Native.h" +#import "api/peerconnection/RTCPeerConnectionFactory.h" +#import "api/peerconnection/RTCSessionDescription.h" +#import "helpers/NSString+StdString.h" + +@interface RTCPeerConnectionTests : XCTestCase +@end + +@implementation RTCPeerConnectionTests + +- (void)testConfigurationGetter { + NSArray *urlStrings = @[ @"stun:stun1.example.net" ]; + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:urlStrings]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + config.iceServers = @[ server ]; + config.iceTransportPolicy = RTCIceTransportPolicyRelay; + config.bundlePolicy = RTCBundlePolicyMaxBundle; + config.rtcpMuxPolicy = RTCRtcpMuxPolicyNegotiate; + config.tcpCandidatePolicy = RTCTcpCandidatePolicyDisabled; + config.candidateNetworkPolicy = RTCCandidateNetworkPolicyLowCost; + const int maxPackets = 60; + const int timeout = 1500; + const int interval = 2000; + config.audioJitterBufferMaxPackets = maxPackets; + config.audioJitterBufferFastAccelerate = YES; + config.iceConnectionReceivingTimeout = timeout; + config.iceBackupCandidatePairPingInterval = interval; + config.continualGatheringPolicy = + RTCContinualGatheringPolicyGatherContinually; + config.shouldPruneTurnPorts = YES; + config.activeResetSrtpParams = YES; + config.cryptoOptions = + [[RTC_OBJC_TYPE(RTCCryptoOptions) alloc] initWithSrtpEnableGcmCryptoSuites:YES + srtpEnableAes128Sha1_32CryptoCipher:YES + srtpEnableEncryptedRtpHeaderExtensions:NO + sframeRequireFrameEncryption:NO]; + + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + + RTC_OBJC_TYPE(RTCConfiguration) * newConfig; + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnection) *peerConnection = + [factory peerConnectionWithConfiguration:config constraints:contraints delegate:nil]; + newConfig = peerConnection.configuration; + + EXPECT_TRUE([peerConnection setBweMinBitrateBps:[NSNumber numberWithInt:100000] + currentBitrateBps:[NSNumber numberWithInt:5000000] + maxBitrateBps:[NSNumber numberWithInt:500000000]]); + EXPECT_FALSE([peerConnection setBweMinBitrateBps:[NSNumber numberWithInt:2] + currentBitrateBps:[NSNumber numberWithInt:1] + maxBitrateBps:nil]); + } + + EXPECT_EQ([config.iceServers count], [newConfig.iceServers count]); + RTC_OBJC_TYPE(RTCIceServer) *newServer = newConfig.iceServers[0]; + RTC_OBJC_TYPE(RTCIceServer) *origServer = config.iceServers[0]; + std::string origUrl = origServer.urlStrings.firstObject.UTF8String; + std::string url = newServer.urlStrings.firstObject.UTF8String; + EXPECT_EQ(origUrl, url); + + EXPECT_EQ(config.iceTransportPolicy, newConfig.iceTransportPolicy); + EXPECT_EQ(config.bundlePolicy, newConfig.bundlePolicy); + EXPECT_EQ(config.rtcpMuxPolicy, newConfig.rtcpMuxPolicy); + EXPECT_EQ(config.tcpCandidatePolicy, newConfig.tcpCandidatePolicy); + EXPECT_EQ(config.candidateNetworkPolicy, newConfig.candidateNetworkPolicy); + EXPECT_EQ(config.audioJitterBufferMaxPackets, newConfig.audioJitterBufferMaxPackets); + EXPECT_EQ(config.audioJitterBufferFastAccelerate, newConfig.audioJitterBufferFastAccelerate); + EXPECT_EQ(config.iceConnectionReceivingTimeout, newConfig.iceConnectionReceivingTimeout); + EXPECT_EQ(config.iceBackupCandidatePairPingInterval, + newConfig.iceBackupCandidatePairPingInterval); + EXPECT_EQ(config.continualGatheringPolicy, newConfig.continualGatheringPolicy); + EXPECT_EQ(config.shouldPruneTurnPorts, newConfig.shouldPruneTurnPorts); + EXPECT_EQ(config.activeResetSrtpParams, newConfig.activeResetSrtpParams); + EXPECT_EQ(config.cryptoOptions.srtpEnableGcmCryptoSuites, + newConfig.cryptoOptions.srtpEnableGcmCryptoSuites); + EXPECT_EQ(config.cryptoOptions.srtpEnableAes128Sha1_32CryptoCipher, + newConfig.cryptoOptions.srtpEnableAes128Sha1_32CryptoCipher); + EXPECT_EQ(config.cryptoOptions.srtpEnableEncryptedRtpHeaderExtensions, + newConfig.cryptoOptions.srtpEnableEncryptedRtpHeaderExtensions); + EXPECT_EQ(config.cryptoOptions.sframeRequireFrameEncryption, + newConfig.cryptoOptions.sframeRequireFrameEncryption); +} + +- (void)testWithDependencies { + NSArray *urlStrings = @[ @"stun:stun1.example.net" ]; + RTC_OBJC_TYPE(RTCIceServer) *server = + [[RTC_OBJC_TYPE(RTCIceServer) alloc] initWithURLStrings:urlStrings]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + config.iceServers = @[ server ]; + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + + std::unique_ptr<webrtc::PeerConnectionDependencies> pc_dependencies = + std::make_unique<webrtc::PeerConnectionDependencies>(nullptr); + @autoreleasepool { + RTC_OBJC_TYPE(RTCPeerConnection) *peerConnection = + [factory peerConnectionWithDependencies:config + constraints:contraints + dependencies:std::move(pc_dependencies) + delegate:nil]; + ASSERT_NE(peerConnection, nil); + } +} + +- (void)testWithInvalidSDP { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCPeerConnection) *peerConnection = + [factory peerConnectionWithConfiguration:config constraints:contraints delegate:nil]; + + dispatch_semaphore_t negotiatedSem = dispatch_semaphore_create(0); + [peerConnection setRemoteDescription:[[RTC_OBJC_TYPE(RTCSessionDescription) alloc] + initWithType:RTCSdpTypeOffer + sdp:@"invalid"] + completionHandler:^(NSError *error) { + ASSERT_NE(error, nil); + if (error != nil) { + dispatch_semaphore_signal(negotiatedSem); + } + }]; + + NSTimeInterval timeout = 5; + ASSERT_EQ( + 0, + dispatch_semaphore_wait(negotiatedSem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(timeout * NSEC_PER_SEC)))); + [peerConnection close]; +} + +- (void)testWithInvalidIceCandidate { + RTC_OBJC_TYPE(RTCPeerConnectionFactory) *factory = + [[RTC_OBJC_TYPE(RTCPeerConnectionFactory) alloc] init]; + + RTC_OBJC_TYPE(RTCConfiguration) *config = [[RTC_OBJC_TYPE(RTCConfiguration) alloc] init]; + config.sdpSemantics = RTCSdpSemanticsUnifiedPlan; + RTC_OBJC_TYPE(RTCMediaConstraints) *contraints = + [[RTC_OBJC_TYPE(RTCMediaConstraints) alloc] initWithMandatoryConstraints:@{} + optionalConstraints:nil]; + RTC_OBJC_TYPE(RTCPeerConnection) *peerConnection = + [factory peerConnectionWithConfiguration:config constraints:contraints delegate:nil]; + + dispatch_semaphore_t negotiatedSem = dispatch_semaphore_create(0); + [peerConnection addIceCandidate:[[RTC_OBJC_TYPE(RTCIceCandidate) alloc] initWithSdp:@"invalid" + sdpMLineIndex:-1 + sdpMid:nil] + completionHandler:^(NSError *error) { + ASSERT_NE(error, nil); + if (error != nil) { + dispatch_semaphore_signal(negotiatedSem); + } + }]; + + NSTimeInterval timeout = 5; + ASSERT_EQ( + 0, + dispatch_semaphore_wait(negotiatedSem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(timeout * NSEC_PER_SEC)))); + [peerConnection close]; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCSessionDescriptionTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCSessionDescriptionTest.mm new file mode 100644 index 0000000000..70c82f78ce --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCSessionDescriptionTest.mm @@ -0,0 +1,122 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCSessionDescription+Private.h" +#import "api/peerconnection/RTCSessionDescription.h" +#import "helpers/NSString+StdString.h" + +@interface RTCSessionDescriptionTests : XCTestCase +@end + +@implementation RTCSessionDescriptionTests + +/** + * Test conversion of an Objective-C RTC_OBJC_TYPE(RTCSessionDescription) to a native + * SessionDescriptionInterface (based on the types and SDP strings being equal). + */ +- (void)testSessionDescriptionConversion { + RTC_OBJC_TYPE(RTCSessionDescription) *description = + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithType:RTCSdpTypeAnswer sdp:[self sdp]]; + + std::unique_ptr<webrtc::SessionDescriptionInterface> nativeDescription = + description.nativeDescription; + + EXPECT_EQ(RTCSdpTypeAnswer, + [RTC_OBJC_TYPE(RTCSessionDescription) typeForStdString:nativeDescription->type()]); + + std::string sdp; + nativeDescription->ToString(&sdp); + EXPECT_EQ([self sdp].stdString, sdp); +} + +- (void)testInitFromNativeSessionDescription { + webrtc::SessionDescriptionInterface *nativeDescription; + + nativeDescription = webrtc::CreateSessionDescription( + webrtc::SessionDescriptionInterface::kAnswer, + [self sdp].stdString, + nullptr); + + RTC_OBJC_TYPE(RTCSessionDescription) *description = + [[RTC_OBJC_TYPE(RTCSessionDescription) alloc] initWithNativeDescription:nativeDescription]; + EXPECT_EQ(webrtc::SessionDescriptionInterface::kAnswer, + [RTC_OBJC_TYPE(RTCSessionDescription) stdStringForType:description.type]); + EXPECT_TRUE([[self sdp] isEqualToString:description.sdp]); +} + +- (NSString *)sdp { + return @"v=0\r\n" + "o=- 5319989746393411314 2 IN IP4 127.0.0.1\r\n" + "s=-\r\n" + "t=0 0\r\n" + "a=group:BUNDLE audio video\r\n" + "a=msid-semantic: WMS ARDAMS\r\n" + "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 9 0 8 126\r\n" + "c=IN IP4 0.0.0.0\r\n" + "a=rtcp:9 IN IP4 0.0.0.0\r\n" + "a=ice-ufrag:f3o+0HG7l9nwIWFY\r\n" + "a=ice-pwd:VDctmJNCptR2TB7+meDpw7w5\r\n" + "a=fingerprint:sha-256 A9:D5:8D:A8:69:22:39:60:92:AD:94:1A:22:2D:5E:" + "A5:4A:A9:18:C2:35:5D:46:5E:59:BD:1C:AF:38:9F:E6:E1\r\n" + "a=setup:active\r\n" + "a=mid:audio\r\n" + "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" + "a=extmap:3 http://www.webrtc.org/experiments/rtp-hdrext/" + "abs-send-time\r\n" + "a=sendrecv\r\n" + "a=rtcp-mux\r\n" + "a=rtpmap:111 opus/48000/2\r\n" + "a=fmtp:111 minptime=10;useinbandfec=1\r\n" + "a=rtpmap:103 ISAC/16000\r\n" + "a=rtpmap:9 G722/8000\r\n" + "a=rtpmap:0 PCMU/8000\r\n" + "a=rtpmap:8 PCMA/8000\r\n" + "a=rtpmap:126 telephone-event/8000\r\n" + "a=maxptime:60\r\n" + "a=ssrc:1504474588 cname:V+FdIC5AJpxLhdYQ\r\n" + "a=ssrc:1504474588 msid:ARDAMS ARDAMSa0\r\n" + "m=video 9 UDP/TLS/RTP/SAVPF 100 116 117 96\r\n" + "c=IN IP4 0.0.0.0\r\n" + "a=rtcp:9 IN IP4 0.0.0.0\r\n" + "a=ice-ufrag:f3o+0HG7l9nwIWFY\r\n" + "a=ice-pwd:VDctmJNCptR2TB7+meDpw7w5\r\n" + "a=fingerprint:sha-256 A9:D5:8D:A8:69:22:39:60:92:AD:94:1A:22:2D:5E:" + "A5:4A:A9:18:C2:35:5D:46:5E:59:BD:1C:AF:38:9F:E6:E1\r\n" + "a=setup:active\r\n" + "a=mid:video\r\n" + "a=extmap:2 urn:ietf:params:rtp-hdrext:toffset\r\n" + "a=extmap:3 http://www.webrtc.org/experiments/rtp-hdrext/" + "abs-send-time\r\n" + "a=extmap:4 urn:3gpp:video-orientation\r\n" + "a=sendrecv\r\n" + "a=rtcp-mux\r\n" + "a=rtpmap:100 VP8/90000\r\n" + "a=rtcp-fb:100 ccm fir\r\n" + "a=rtcp-fb:100 goog-lntf\r\n" + "a=rtcp-fb:100 nack\r\n" + "a=rtcp-fb:100 nack pli\r\n" + "a=rtcp-fb:100 goog-remb\r\n" + "a=rtpmap:116 red/90000\r\n" + "a=rtpmap:117 ulpfec/90000\r\n" + "a=rtpmap:96 rtx/90000\r\n" + "a=fmtp:96 apt=100\r\n" + "a=ssrc-group:FID 498297514 1644357692\r\n" + "a=ssrc:498297514 cname:V+FdIC5AJpxLhdYQ\r\n" + "a=ssrc:498297514 msid:ARDAMS ARDAMSv0\r\n" + "a=ssrc:1644357692 cname:V+FdIC5AJpxLhdYQ\r\n" + "a=ssrc:1644357692 msid:ARDAMS ARDAMSv0\r\n"; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/RTCTracingTest.mm b/third_party/libwebrtc/sdk/objc/unittests/RTCTracingTest.mm new file mode 100644 index 0000000000..ff93047bdf --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/RTCTracingTest.mm @@ -0,0 +1,41 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <XCTest/XCTest.h> + +#include <vector> + +#include "rtc_base/gunit.h" + +#import "api/peerconnection/RTCTracing.h" +#import "helpers/NSString+StdString.h" + +@interface RTCTracingTests : XCTestCase +@end + +@implementation RTCTracingTests + +- (NSString *)documentsFilePathForFileName:(NSString *)fileName { + NSParameterAssert(fileName.length); + NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); + NSString *documentsDirPath = paths.firstObject; + NSString *filePath = + [documentsDirPath stringByAppendingPathComponent:fileName]; + return filePath; +} + +- (void)testTracingTestNoInitialization { + NSString *filePath = [self documentsFilePathForFileName:@"webrtc-trace.txt"]; + EXPECT_EQ(NO, RTCStartInternalCapture(filePath)); + RTCStopInternalCapture(); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/audio_short16.pcm b/third_party/libwebrtc/sdk/objc/unittests/audio_short16.pcm Binary files differnew file mode 100644 index 0000000000..15a0f1811c --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/audio_short16.pcm diff --git a/third_party/libwebrtc/sdk/objc/unittests/audio_short44.pcm b/third_party/libwebrtc/sdk/objc/unittests/audio_short44.pcm Binary files differnew file mode 100644 index 0000000000..011cdce959 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/audio_short44.pcm diff --git a/third_party/libwebrtc/sdk/objc/unittests/audio_short48.pcm b/third_party/libwebrtc/sdk/objc/unittests/audio_short48.pcm Binary files differnew file mode 100644 index 0000000000..06fd8261cd --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/audio_short48.pcm diff --git a/third_party/libwebrtc/sdk/objc/unittests/avformatmappertests.mm b/third_party/libwebrtc/sdk/objc/unittests/avformatmappertests.mm new file mode 100644 index 0000000000..35e95a8c22 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/avformatmappertests.mm @@ -0,0 +1,243 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <OCMock/OCMock.h> + +#include "rtc_base/gunit.h" + +// Width and height don't play any role so lets use predefined values throughout +// the tests. +static const int kFormatWidth = 789; +static const int kFormatHeight = 987; + +// Hardcoded framrate to be used throughout the tests. +static const int kFramerate = 30; + +// Same width and height is used so it's ok to expect same cricket::VideoFormat +static cricket::VideoFormat expectedFormat = + cricket::VideoFormat(kFormatWidth, + kFormatHeight, + cricket::VideoFormat::FpsToInterval(kFramerate), + cricket::FOURCC_NV12); + +// Mock class for AVCaptureDeviceFormat. +// Custom implementation needed because OCMock cannot handle the +// CMVideoDescriptionRef mocking. +@interface AVCaptureDeviceFormatMock : NSObject + +@property (nonatomic, assign) CMVideoFormatDescriptionRef format; +@property (nonatomic, strong) OCMockObject *rangeMock; + +- (instancetype)initWithMediaSubtype:(FourCharCode)subtype + minFps:(float)minFps + maxFps:(float)maxFps; ++ (instancetype)validFormat; ++ (instancetype)invalidFpsFormat; ++ (instancetype)invalidMediaSubtypeFormat; + +@end + +@implementation AVCaptureDeviceFormatMock + +@synthesize format = _format; +@synthesize rangeMock = _rangeMock; + +- (instancetype)initWithMediaSubtype:(FourCharCode)subtype + minFps:(float)minFps + maxFps:(float)maxFps { + if (self = [super init]) { + CMVideoFormatDescriptionCreate(nil, subtype, kFormatWidth, kFormatHeight, + nil, &_format); + // We can use OCMock for the range. + _rangeMock = [OCMockObject mockForClass:[AVFrameRateRange class]]; + [[[_rangeMock stub] andReturnValue:@(minFps)] minFrameRate]; + [[[_rangeMock stub] andReturnValue:@(maxFps)] maxFrameRate]; + } + + return self; +} + ++ (instancetype)validFormat { + AVCaptureDeviceFormatMock *instance = [[AVCaptureDeviceFormatMock alloc] + initWithMediaSubtype:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + minFps:0.0 + maxFps:30.0]; + return instance; +} + ++ (instancetype)invalidFpsFormat { + AVCaptureDeviceFormatMock *instance = [[AVCaptureDeviceFormatMock alloc] + initWithMediaSubtype:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + minFps:0.0 + maxFps:22.0]; + return instance; +} + ++ (instancetype)invalidMediaSubtypeFormat { + AVCaptureDeviceFormatMock *instance = [[AVCaptureDeviceFormatMock alloc] + initWithMediaSubtype:kCVPixelFormatType_420YpCbCr8Planar + minFps:0.0 + maxFps:60.0]; + return instance; +} + +- (void)dealloc { + if (_format != nil) { + CFRelease(_format); + _format = nil; + } +} + +// Redefinition of AVCaptureDevice methods we want to mock. +- (CMVideoFormatDescriptionRef)formatDescription { + return self.format; +} + +- (NSArray *)videoSupportedFrameRateRanges { + return @[ self.rangeMock ]; +} + +@end + +TEST(AVFormatMapperTest, SuportedCricketFormatsWithInvalidFramerateFormats) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + + // Valid media subtype, invalid framerate + AVCaptureDeviceFormatMock* mock = + [AVCaptureDeviceFormatMock invalidFpsFormat]; + OCMStub([mockDevice formats]).andReturn(@[ mock ]); + + // when + std::set<cricket::VideoFormat> result = + webrtc::GetSupportedVideoFormatsForDevice(mockDevice); + + // then + EXPECT_TRUE(result.empty()); +} + +TEST(AVFormatMapperTest, SuportedCricketFormatsWithInvalidFormats) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + + // Invalid media subtype, valid framerate + AVCaptureDeviceFormatMock* mock = + [AVCaptureDeviceFormatMock invalidMediaSubtypeFormat]; + OCMStub([mockDevice formats]).andReturn(@[ mock ]); + + // when + std::set<cricket::VideoFormat> result = + webrtc::GetSupportedVideoFormatsForDevice(mockDevice); + + // then + EXPECT_TRUE(result.empty()); +} + +TEST(AVFormatMapperTest, SuportedCricketFormats) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + + // valid media subtype, valid framerate + AVCaptureDeviceFormatMock* mock = [AVCaptureDeviceFormatMock validFormat]; + OCMStub([mockDevice formats]).andReturn(@[ mock ]); + + // when + std::set<cricket::VideoFormat> result = + webrtc::GetSupportedVideoFormatsForDevice(mockDevice); + + // then + EXPECT_EQ(1u, result.size()); + // make sure the set has the expected format + EXPECT_EQ(expectedFormat, *result.begin()); +} + +TEST(AVFormatMapperTest, MediaSubtypePreference) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + + // valid media subtype, valid framerate + AVCaptureDeviceFormatMock* mockOne = [[AVCaptureDeviceFormatMock alloc] + initWithMediaSubtype:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + minFps:0.0 + maxFps:30.0]; + // valid media subtype, valid framerate. + // This media subtype should be the preffered one. + AVCaptureDeviceFormatMock* mockTwo = [[AVCaptureDeviceFormatMock alloc] + initWithMediaSubtype:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + minFps:0.0 + maxFps:30.0]; + OCMStub([mockDevice lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES); + OCMStub([mockDevice unlockForConfiguration]); + NSArray* array = @[ mockOne, mockTwo ]; + OCMStub([mockDevice formats]).andReturn(array); + + // to verify + OCMExpect([mockDevice setActiveFormat:(AVCaptureDeviceFormat*)mockTwo]); + OCMExpect( + [mockDevice setActiveVideoMinFrameDuration:CMTimeMake(1, kFramerate)]); + + // when + bool resultFormat = + webrtc::SetFormatForCaptureDevice(mockDevice, nil, expectedFormat); + + // then + EXPECT_TRUE(resultFormat); + [mockDevice verify]; +} + +TEST(AVFormatMapperTest, SetFormatWhenDeviceCannotLock) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + [[[mockDevice stub] andReturnValue:@(NO)] + lockForConfiguration:[OCMArg setTo:nil]]; + [[[mockDevice stub] andReturn:@[]] formats]; + + // when + bool resultFormat = webrtc::SetFormatForCaptureDevice(mockDevice, nil, + cricket::VideoFormat()); + + // then + EXPECT_FALSE(resultFormat); +} + +TEST(AVFormatMapperTest, SetFormatWhenFormatIsIncompatible) { + // given + id mockDevice = OCMClassMock([AVCaptureDevice class]); + OCMStub([mockDevice formats]).andReturn(@[]); + OCMStub([mockDevice lockForConfiguration:[OCMArg setTo:nil]]).andReturn(YES); + NSException* testException = + [NSException exceptionWithName:@"Test exception" + reason:@"Raised from unit tests" + userInfo:nil]; + OCMStub([mockDevice setActiveFormat:[OCMArg any]]).andThrow(testException); + OCMExpect([mockDevice unlockForConfiguration]); + + // when + bool resultFormat = webrtc::SetFormatForCaptureDevice(mockDevice, nil, + cricket::VideoFormat()); + + // then + EXPECT_FALSE(resultFormat); + + // TODO(denicija): Remove try-catch when Chromium rolls this change: + // https://github.com/erikdoe/ocmock/commit/de1419415581dc307045e54bfe9c98c86efea96b + // Without it, stubbed exceptions are being re-raised on [mock verify]. + // More information here: + //https://github.com/erikdoe/ocmock/issues/241 + @try { + [mockDevice verify]; + } @catch (NSException* exception) { + if ([exception.reason isEqual:testException.reason]) { + // Nothing dangerous here + EXPECT_TRUE([exception.reason isEqualToString:exception.reason]); + } + } +} diff --git a/third_party/libwebrtc/sdk/objc/unittests/foreman.mp4 b/third_party/libwebrtc/sdk/objc/unittests/foreman.mp4 Binary files differnew file mode 100644 index 0000000000..ccffbf4722 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/foreman.mp4 diff --git a/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.h b/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.h new file mode 100644 index 0000000000..76c0d15c7e --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.h @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <AVFoundation/AVFoundation.h> + +#include "api/video/i420_buffer.h" + +void DrawGradientInRGBPixelBuffer(CVPixelBufferRef pixelBuffer); + +rtc::scoped_refptr<webrtc::I420Buffer> CreateI420Gradient(int width, + int height); + +void CopyI420BufferToCVPixelBuffer( + rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer, + CVPixelBufferRef pixelBuffer); diff --git a/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.mm b/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.mm new file mode 100644 index 0000000000..98b86c54c0 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/frame_buffer_helpers.mm @@ -0,0 +1,126 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/objc/unittests/frame_buffer_helpers.h" + +#include "third_party/libyuv/include/libyuv.h" + +void DrawGradientInRGBPixelBuffer(CVPixelBufferRef pixelBuffer) { + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + void* baseAddr = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + int byteOrder = CVPixelBufferGetPixelFormatType(pixelBuffer) == kCVPixelFormatType_32ARGB ? + kCGBitmapByteOrder32Little : + 0; + CGContextRef cgContext = CGBitmapContextCreate(baseAddr, + width, + height, + 8, + CVPixelBufferGetBytesPerRow(pixelBuffer), + colorSpace, + byteOrder | kCGImageAlphaNoneSkipLast); + + // Create a gradient + CGFloat colors[] = { + 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, + }; + CGGradientRef gradient = CGGradientCreateWithColorComponents(colorSpace, colors, NULL, 4); + + CGContextDrawLinearGradient( + cgContext, gradient, CGPointMake(0, 0), CGPointMake(width, height), 0); + CGGradientRelease(gradient); + + CGImageRef cgImage = CGBitmapContextCreateImage(cgContext); + CGContextRelease(cgContext); + CGImageRelease(cgImage); + CGColorSpaceRelease(colorSpace); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); +} + +rtc::scoped_refptr<webrtc::I420Buffer> CreateI420Gradient(int width, int height) { + rtc::scoped_refptr<webrtc::I420Buffer> buffer(webrtc::I420Buffer::Create(width, height)); + // Initialize with gradient, Y = 128(x/w + y/h), U = 256 x/w, V = 256 y/h + for (int x = 0; x < width; x++) { + for (int y = 0; y < height; y++) { + buffer->MutableDataY()[x + y * width] = 128 * (x * height + y * width) / (width * height); + } + } + int chroma_width = buffer->ChromaWidth(); + int chroma_height = buffer->ChromaHeight(); + for (int x = 0; x < chroma_width; x++) { + for (int y = 0; y < chroma_height; y++) { + buffer->MutableDataU()[x + y * chroma_width] = 255 * x / (chroma_width - 1); + buffer->MutableDataV()[x + y * chroma_width] = 255 * y / (chroma_height - 1); + } + } + return buffer; +} + +void CopyI420BufferToCVPixelBuffer(rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer, + CVPixelBufferRef pixelBuffer) { + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + // NV12 + uint8_t* dstY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + uint8_t* dstUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + + libyuv::I420ToNV12(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dstY, + dstYStride, + dstUV, + dstUVStride, + i420Buffer->width(), + i420Buffer->height()); + } else { + uint8_t* dst = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixelBuffer)); + const int bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); + + if (pixelFormat == kCVPixelFormatType_32BGRA) { + // Corresponds to libyuv::FOURCC_ARGB + libyuv::I420ToARGB(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dst, + bytesPerRow, + i420Buffer->width(), + i420Buffer->height()); + } else if (pixelFormat == kCVPixelFormatType_32ARGB) { + // Corresponds to libyuv::FOURCC_BGRA + libyuv::I420ToBGRA(i420Buffer->DataY(), + i420Buffer->StrideY(), + i420Buffer->DataU(), + i420Buffer->StrideU(), + i420Buffer->DataV(), + i420Buffer->StrideV(), + dst, + bytesPerRow, + i420Buffer->width(), + i420Buffer->height()); + } + } + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); +} diff --git a/third_party/libwebrtc/sdk/objc/unittests/main.mm b/third_party/libwebrtc/sdk/objc/unittests/main.mm new file mode 100644 index 0000000000..9c513762c1 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/main.mm @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <UIKit/UIKit.h> +#include "rtc_base/thread.h" +#include "test/ios/coverage_util_ios.h" + +int main(int argc, char* argv[]) { + rtc::test::ConfigureCoverageReportPath(); + + rtc::AutoThread main_thread; + + @autoreleasepool { + return UIApplicationMain(argc, argv, nil, nil); + } +} diff --git a/third_party/libwebrtc/sdk/objc/unittests/nalu_rewriter_xctest.mm b/third_party/libwebrtc/sdk/objc/unittests/nalu_rewriter_xctest.mm new file mode 100644 index 0000000000..82da549bb6 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/nalu_rewriter_xctest.mm @@ -0,0 +1,374 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "common_video/h264/h264_common.h" +#include "components/video_codec/nalu_rewriter.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/gunit.h" + +#import <XCTest/XCTest.h> + +#if TARGET_OS_IPHONE +#import <AVFoundation/AVFoundation.h> +#import <UIKit/UIKit.h> +#endif + +@interface NaluRewriterTests : XCTestCase + +@end + +static const uint8_t NALU_TEST_DATA_0[] = {0xAA, 0xBB, 0xCC}; +static const uint8_t NALU_TEST_DATA_1[] = {0xDE, 0xAD, 0xBE, 0xEF}; + +// clang-format off +static const uint8_t SPS_PPS_BUFFER[] = { + // SPS nalu. + 0x00, 0x00, 0x00, 0x01, 0x27, 0x42, 0x00, 0x1E, 0xAB, 0x40, 0xF0, 0x28, + 0xD3, 0x70, 0x20, 0x20, 0x20, 0x20, + // PPS nalu. + 0x00, 0x00, 0x00, 0x01, 0x28, 0xCE, 0x3C, 0x30}; +// clang-format on + +@implementation NaluRewriterTests + +- (void)testCreateVideoFormatDescription { + CMVideoFormatDescriptionRef description = + webrtc::CreateVideoFormatDescription(SPS_PPS_BUFFER, arraysize(SPS_PPS_BUFFER)); + XCTAssertTrue(description); + if (description) { + CFRelease(description); + description = nullptr; + } + + // clang-format off + const uint8_t sps_pps_not_at_start_buffer[] = { + // Add some non-SPS/PPS NALUs at the beginning + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0xFF, 0x00, 0x00, 0x00, 0x01, + 0xAB, 0x33, 0x21, + // SPS nalu. + 0x00, 0x00, 0x01, 0x27, 0x42, 0x00, 0x1E, 0xAB, 0x40, 0xF0, 0x28, 0xD3, + 0x70, 0x20, 0x20, 0x20, 0x20, + // PPS nalu. + 0x00, 0x00, 0x01, 0x28, 0xCE, 0x3C, 0x30}; + // clang-format on + description = webrtc::CreateVideoFormatDescription(sps_pps_not_at_start_buffer, + arraysize(sps_pps_not_at_start_buffer)); + + XCTAssertTrue(description); + + if (description) { + CFRelease(description); + description = nullptr; + } + + const uint8_t other_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x28}; + XCTAssertFalse(webrtc::CreateVideoFormatDescription(other_buffer, arraysize(other_buffer))); +} + +- (void)testReadEmptyInput { + const uint8_t annex_b_test_data[] = {0x00}; + webrtc::AnnexBBufferReader reader(annex_b_test_data, 0); + const uint8_t* nalu = nullptr; + size_t nalu_length = 0; + XCTAssertEqual(0u, reader.BytesRemaining()); + XCTAssertFalse(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(nullptr, nalu); + XCTAssertEqual(0u, nalu_length); +} + +- (void)testReadSingleNalu { + const uint8_t annex_b_test_data[] = {0x00, 0x00, 0x00, 0x01, 0xAA}; + webrtc::AnnexBBufferReader reader(annex_b_test_data, arraysize(annex_b_test_data)); + const uint8_t* nalu = nullptr; + size_t nalu_length = 0; + XCTAssertEqual(arraysize(annex_b_test_data), reader.BytesRemaining()); + XCTAssertTrue(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(annex_b_test_data + 4, nalu); + XCTAssertEqual(1u, nalu_length); + XCTAssertEqual(0u, reader.BytesRemaining()); + XCTAssertFalse(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(nullptr, nalu); + XCTAssertEqual(0u, nalu_length); +} + +- (void)testReadSingleNalu3ByteHeader { + const uint8_t annex_b_test_data[] = {0x00, 0x00, 0x01, 0xAA}; + webrtc::AnnexBBufferReader reader(annex_b_test_data, arraysize(annex_b_test_data)); + const uint8_t* nalu = nullptr; + size_t nalu_length = 0; + XCTAssertEqual(arraysize(annex_b_test_data), reader.BytesRemaining()); + XCTAssertTrue(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(annex_b_test_data + 3, nalu); + XCTAssertEqual(1u, nalu_length); + XCTAssertEqual(0u, reader.BytesRemaining()); + XCTAssertFalse(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(nullptr, nalu); + XCTAssertEqual(0u, nalu_length); +} + +- (void)testReadMissingNalu { + // clang-format off + const uint8_t annex_b_test_data[] = {0x01, + 0x00, 0x01, + 0x00, 0x00, 0x00, 0xFF}; + // clang-format on + webrtc::AnnexBBufferReader reader(annex_b_test_data, arraysize(annex_b_test_data)); + const uint8_t* nalu = nullptr; + size_t nalu_length = 0; + XCTAssertEqual(0u, reader.BytesRemaining()); + XCTAssertFalse(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(nullptr, nalu); + XCTAssertEqual(0u, nalu_length); +} + +- (void)testReadMultipleNalus { + // clang-format off + const uint8_t annex_b_test_data[] = {0x00, 0x00, 0x00, 0x01, 0xFF, + 0x01, + 0x00, 0x01, + 0x00, 0x00, 0x00, 0xFF, + 0x00, 0x00, 0x01, 0xAA, 0xBB}; + // clang-format on + webrtc::AnnexBBufferReader reader(annex_b_test_data, arraysize(annex_b_test_data)); + const uint8_t* nalu = nullptr; + size_t nalu_length = 0; + XCTAssertEqual(arraysize(annex_b_test_data), reader.BytesRemaining()); + XCTAssertTrue(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(annex_b_test_data + 4, nalu); + XCTAssertEqual(8u, nalu_length); + XCTAssertEqual(5u, reader.BytesRemaining()); + XCTAssertTrue(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(annex_b_test_data + 15, nalu); + XCTAssertEqual(2u, nalu_length); + XCTAssertEqual(0u, reader.BytesRemaining()); + XCTAssertFalse(reader.ReadNalu(&nalu, &nalu_length)); + XCTAssertEqual(nullptr, nalu); + XCTAssertEqual(0u, nalu_length); +} + +- (void)testEmptyOutputBuffer { + const uint8_t expected_buffer[] = {0x00}; + const size_t buffer_size = 1; + std::unique_ptr<uint8_t[]> buffer(new uint8_t[buffer_size]); + memset(buffer.get(), 0, buffer_size); + webrtc::AvccBufferWriter writer(buffer.get(), 0); + XCTAssertEqual(0u, writer.BytesRemaining()); + XCTAssertFalse(writer.WriteNalu(NALU_TEST_DATA_0, arraysize(NALU_TEST_DATA_0))); + XCTAssertEqual(0, memcmp(expected_buffer, buffer.get(), arraysize(expected_buffer))); +} + +- (void)testWriteSingleNalu { + const uint8_t expected_buffer[] = { + 0x00, 0x00, 0x00, 0x03, 0xAA, 0xBB, 0xCC, + }; + const size_t buffer_size = arraysize(NALU_TEST_DATA_0) + 4; + std::unique_ptr<uint8_t[]> buffer(new uint8_t[buffer_size]); + webrtc::AvccBufferWriter writer(buffer.get(), buffer_size); + XCTAssertEqual(buffer_size, writer.BytesRemaining()); + XCTAssertTrue(writer.WriteNalu(NALU_TEST_DATA_0, arraysize(NALU_TEST_DATA_0))); + XCTAssertEqual(0u, writer.BytesRemaining()); + XCTAssertFalse(writer.WriteNalu(NALU_TEST_DATA_1, arraysize(NALU_TEST_DATA_1))); + XCTAssertEqual(0, memcmp(expected_buffer, buffer.get(), arraysize(expected_buffer))); +} + +- (void)testWriteMultipleNalus { + // clang-format off + const uint8_t expected_buffer[] = { + 0x00, 0x00, 0x00, 0x03, 0xAA, 0xBB, 0xCC, + 0x00, 0x00, 0x00, 0x04, 0xDE, 0xAD, 0xBE, 0xEF + }; + // clang-format on + const size_t buffer_size = arraysize(NALU_TEST_DATA_0) + arraysize(NALU_TEST_DATA_1) + 8; + std::unique_ptr<uint8_t[]> buffer(new uint8_t[buffer_size]); + webrtc::AvccBufferWriter writer(buffer.get(), buffer_size); + XCTAssertEqual(buffer_size, writer.BytesRemaining()); + XCTAssertTrue(writer.WriteNalu(NALU_TEST_DATA_0, arraysize(NALU_TEST_DATA_0))); + XCTAssertEqual(buffer_size - (arraysize(NALU_TEST_DATA_0) + 4), writer.BytesRemaining()); + XCTAssertTrue(writer.WriteNalu(NALU_TEST_DATA_1, arraysize(NALU_TEST_DATA_1))); + XCTAssertEqual(0u, writer.BytesRemaining()); + XCTAssertEqual(0, memcmp(expected_buffer, buffer.get(), arraysize(expected_buffer))); +} + +- (void)testOverflow { + const uint8_t expected_buffer[] = {0x00, 0x00, 0x00}; + const size_t buffer_size = arraysize(NALU_TEST_DATA_0); + std::unique_ptr<uint8_t[]> buffer(new uint8_t[buffer_size]); + memset(buffer.get(), 0, buffer_size); + webrtc::AvccBufferWriter writer(buffer.get(), buffer_size); + XCTAssertEqual(buffer_size, writer.BytesRemaining()); + XCTAssertFalse(writer.WriteNalu(NALU_TEST_DATA_0, arraysize(NALU_TEST_DATA_0))); + XCTAssertEqual(buffer_size, writer.BytesRemaining()); + XCTAssertEqual(0, memcmp(expected_buffer, buffer.get(), arraysize(expected_buffer))); +} + +- (void)testH264AnnexBBufferToCMSampleBuffer { + // clang-format off + const uint8_t annex_b_test_data[] = { + 0x00, + 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x01, + 0xAA, 0xFF, // second chunk, 2 bytes + 0x00, 0x00, 0x01, + 0xBB}; // third chunk, 1 byte, will not fit into output array + + const uint8_t expected_cmsample_data[] = { + 0x00, 0x00, 0x00, 0x04, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x00, 0x02, + 0xAA, 0xFF}; // second chunk, 2 bytes + // clang-format on + + CMMemoryPoolRef memory_pool = CMMemoryPoolCreate(nil); + CMSampleBufferRef out_sample_buffer = nil; + CMVideoFormatDescriptionRef description = [self createDescription]; + + Boolean result = webrtc::H264AnnexBBufferToCMSampleBuffer(annex_b_test_data, + arraysize(annex_b_test_data), + description, + &out_sample_buffer, + memory_pool); + + XCTAssertTrue(result); + + XCTAssertEqual(description, CMSampleBufferGetFormatDescription(out_sample_buffer)); + + char* data_ptr = nullptr; + CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(out_sample_buffer); + size_t block_buffer_size = CMBlockBufferGetDataLength(block_buffer); + CMBlockBufferGetDataPointer(block_buffer, 0, nullptr, nullptr, &data_ptr); + XCTAssertEqual(block_buffer_size, arraysize(annex_b_test_data)); + + int data_comparison_result = + memcmp(expected_cmsample_data, data_ptr, arraysize(expected_cmsample_data)); + + XCTAssertEqual(0, data_comparison_result); + + if (description) { + CFRelease(description); + description = nullptr; + } + + CMMemoryPoolInvalidate(memory_pool); + CFRelease(memory_pool); +} + +- (void)testH264CMSampleBufferToAnnexBBuffer { + // clang-format off + const uint8_t cmsample_data[] = { + 0x00, 0x00, 0x00, 0x04, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x00, 0x02, + 0xAA, 0xFF}; // second chunk, 2 bytes + + const uint8_t expected_annex_b_data[] = { + 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x00, 0x01, + 0xAA, 0xFF}; // second chunk, 2 bytes + // clang-format on + + rtc::Buffer annexb_buffer(arraysize(cmsample_data)); + CMSampleBufferRef sample_buffer = + [self createCMSampleBufferRef:(void*)cmsample_data cmsampleSize:arraysize(cmsample_data)]; + + Boolean result = webrtc::H264CMSampleBufferToAnnexBBuffer(sample_buffer, + /* is_keyframe */ false, + &annexb_buffer); + + XCTAssertTrue(result); + + XCTAssertEqual(arraysize(expected_annex_b_data), annexb_buffer.size()); + + int data_comparison_result = + memcmp(expected_annex_b_data, annexb_buffer.data(), arraysize(expected_annex_b_data)); + + XCTAssertEqual(0, data_comparison_result); +} + +- (void)testH264CMSampleBufferToAnnexBBufferWithKeyframe { + // clang-format off + const uint8_t cmsample_data[] = { + 0x00, 0x00, 0x00, 0x04, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x00, 0x02, + 0xAA, 0xFF}; // second chunk, 2 bytes + + const uint8_t expected_annex_b_data[] = { + 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0xFF, // first chunk, 4 bytes + 0x00, 0x00, 0x00, 0x01, + 0xAA, 0xFF}; // second chunk, 2 bytes + // clang-format on + + rtc::Buffer annexb_buffer(arraysize(cmsample_data)); + CMSampleBufferRef sample_buffer = + [self createCMSampleBufferRef:(void*)cmsample_data cmsampleSize:arraysize(cmsample_data)]; + + Boolean result = webrtc::H264CMSampleBufferToAnnexBBuffer(sample_buffer, + /* is_keyframe */ true, + &annexb_buffer); + + XCTAssertTrue(result); + + XCTAssertEqual(arraysize(SPS_PPS_BUFFER) + arraysize(expected_annex_b_data), + annexb_buffer.size()); + + XCTAssertEqual(0, memcmp(SPS_PPS_BUFFER, annexb_buffer.data(), arraysize(SPS_PPS_BUFFER))); + + XCTAssertEqual(0, + memcmp(expected_annex_b_data, + annexb_buffer.data() + arraysize(SPS_PPS_BUFFER), + arraysize(expected_annex_b_data))); +} + +- (CMVideoFormatDescriptionRef)createDescription { + CMVideoFormatDescriptionRef description = + webrtc::CreateVideoFormatDescription(SPS_PPS_BUFFER, arraysize(SPS_PPS_BUFFER)); + XCTAssertTrue(description); + return description; +} + +- (CMSampleBufferRef)createCMSampleBufferRef:(void*)cmsampleData cmsampleSize:(size_t)cmsampleSize { + CMSampleBufferRef sample_buffer = nil; + OSStatus status; + + CMVideoFormatDescriptionRef description = [self createDescription]; + CMBlockBufferRef block_buffer = nullptr; + + status = CMBlockBufferCreateWithMemoryBlock(nullptr, + cmsampleData, + cmsampleSize, + nullptr, + nullptr, + 0, + cmsampleSize, + kCMBlockBufferAssureMemoryNowFlag, + &block_buffer); + XCTAssertEqual(kCMBlockBufferNoErr, status); + + status = CMSampleBufferCreate(nullptr, + block_buffer, + true, + nullptr, + nullptr, + description, + 1, + 0, + nullptr, + 0, + nullptr, + &sample_buffer); + XCTAssertEqual(noErr, status); + + return sample_buffer; +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/objc_video_decoder_factory_tests.mm b/third_party/libwebrtc/sdk/objc/unittests/objc_video_decoder_factory_tests.mm new file mode 100644 index 0000000000..f44d831d29 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/objc_video_decoder_factory_tests.mm @@ -0,0 +1,107 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <OCMock/OCMock.h> +#import <XCTest/XCTest.h> + +#include "sdk/objc/native/src/objc_video_decoder_factory.h" + +#import "base/RTCMacros.h" +#import "base/RTCVideoDecoder.h" +#import "base/RTCVideoDecoderFactory.h" +#include "media/base/codec.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/gunit.h" + +id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> CreateDecoderFactoryReturning(int return_code) { + id decoderMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoDecoder))); + OCMStub([decoderMock startDecodeWithNumberOfCores:1]).andReturn(return_code); + OCMStub([decoderMock decode:[OCMArg any] + missingFrames:NO + codecSpecificInfo:[OCMArg any] + renderTimeMs:0]) + .andReturn(return_code); + OCMStub([decoderMock releaseDecoder]).andReturn(return_code); + + id decoderFactoryMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoDecoderFactory))); + RTC_OBJC_TYPE(RTCVideoCodecInfo)* supported = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:@"H264" parameters:nil]; + OCMStub([decoderFactoryMock supportedCodecs]).andReturn(@[ supported ]); + OCMStub([decoderFactoryMock createDecoder:[OCMArg any]]).andReturn(decoderMock); + return decoderFactoryMock; +} + +id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> CreateOKDecoderFactory() { + return CreateDecoderFactoryReturning(WEBRTC_VIDEO_CODEC_OK); +} + +id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> CreateErrorDecoderFactory() { + return CreateDecoderFactoryReturning(WEBRTC_VIDEO_CODEC_ERROR); +} + +std::unique_ptr<webrtc::VideoDecoder> GetObjCDecoder( + id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)> factory) { + webrtc::ObjCVideoDecoderFactory decoder_factory(factory); + return decoder_factory.CreateVideoDecoder(webrtc::SdpVideoFormat(cricket::kH264CodecName)); +} + +#pragma mark - + +@interface ObjCVideoDecoderFactoryTests : XCTestCase +@end + +@implementation ObjCVideoDecoderFactoryTests + +- (void)testConfigureReturnsTrueOnSuccess { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + webrtc::VideoDecoder::Settings settings; + EXPECT_TRUE(decoder->Configure(settings)); +} + +- (void)testConfigureReturnsFalseOnFail { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + webrtc::VideoDecoder::Settings settings; + EXPECT_FALSE(decoder->Configure(settings)); +} + +- (void)testDecodeReturnsOKOnSuccess { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + webrtc::EncodedImage encoded_image; + encoded_image.SetEncodedData(webrtc::EncodedImageBuffer::Create()); + + EXPECT_EQ(decoder->Decode(encoded_image, false, 0), WEBRTC_VIDEO_CODEC_OK); +} + +- (void)testDecodeReturnsErrorOnFail { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + webrtc::EncodedImage encoded_image; + encoded_image.SetEncodedData(webrtc::EncodedImageBuffer::Create()); + + EXPECT_EQ(decoder->Decode(encoded_image, false, 0), WEBRTC_VIDEO_CODEC_ERROR); +} + +- (void)testReleaseDecodeReturnsOKOnSuccess { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateOKDecoderFactory()); + + EXPECT_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_OK); +} + +- (void)testReleaseDecodeReturnsErrorOnFail { + std::unique_ptr<webrtc::VideoDecoder> decoder = GetObjCDecoder(CreateErrorDecoderFactory()); + + EXPECT_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_ERROR); +} +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/objc_video_encoder_factory_tests.mm b/third_party/libwebrtc/sdk/objc/unittests/objc_video_encoder_factory_tests.mm new file mode 100644 index 0000000000..9a4fee2e95 --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/objc_video_encoder_factory_tests.mm @@ -0,0 +1,148 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <Foundation/Foundation.h> +#import <OCMock/OCMock.h> +#import <XCTest/XCTest.h> + +#include "sdk/objc/native/src/objc_video_encoder_factory.h" + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#import "base/RTCVideoEncoder.h" +#import "base/RTCVideoEncoderFactory.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/gunit.h" +#include "sdk/objc/native/src/objc_frame_buffer.h" + +id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> CreateEncoderFactoryReturning(int return_code) { + id encoderMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoEncoder))); + OCMStub([encoderMock startEncodeWithSettings:[OCMArg any] numberOfCores:1]) + .andReturn(return_code); + OCMStub([encoderMock encode:[OCMArg any] codecSpecificInfo:[OCMArg any] frameTypes:[OCMArg any]]) + .andReturn(return_code); + OCMStub([encoderMock releaseEncoder]).andReturn(return_code); + OCMStub([encoderMock setBitrate:0 framerate:0]).andReturn(return_code); + + id encoderFactoryMock = OCMProtocolMock(@protocol(RTC_OBJC_TYPE(RTCVideoEncoderFactory))); + RTC_OBJC_TYPE(RTCVideoCodecInfo)* supported = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:@"H264" parameters:nil]; + OCMStub([encoderFactoryMock supportedCodecs]).andReturn(@[ supported ]); + OCMStub([encoderFactoryMock implementations]).andReturn(@[ supported ]); + OCMStub([encoderFactoryMock createEncoder:[OCMArg any]]).andReturn(encoderMock); + return encoderFactoryMock; +} + +id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> CreateOKEncoderFactory() { + return CreateEncoderFactoryReturning(WEBRTC_VIDEO_CODEC_OK); +} + +id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> CreateErrorEncoderFactory() { + return CreateEncoderFactoryReturning(WEBRTC_VIDEO_CODEC_ERROR); +} + +std::unique_ptr<webrtc::VideoEncoder> GetObjCEncoder( + id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> factory) { + webrtc::ObjCVideoEncoderFactory encoder_factory(factory); + webrtc::SdpVideoFormat format("H264"); + return encoder_factory.CreateVideoEncoder(format); +} + +#pragma mark - + +@interface ObjCVideoEncoderFactoryTests : XCTestCase +@end + +@implementation ObjCVideoEncoderFactoryTests + +- (void)testInitEncodeReturnsOKOnSuccess { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + auto* settings = new webrtc::VideoCodec(); + const webrtc::VideoEncoder::Capabilities kCapabilities(false); + EXPECT_EQ(encoder->InitEncode(settings, webrtc::VideoEncoder::Settings(kCapabilities, 1, 0)), + WEBRTC_VIDEO_CODEC_OK); +} + +- (void)testInitEncodeReturnsErrorOnFail { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + auto* settings = new webrtc::VideoCodec(); + const webrtc::VideoEncoder::Capabilities kCapabilities(false); + EXPECT_EQ(encoder->InitEncode(settings, webrtc::VideoEncoder::Settings(kCapabilities, 1, 0)), + WEBRTC_VIDEO_CODEC_ERROR); +} + +- (void)testEncodeReturnsOKOnSuccess { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + CVPixelBufferRef pixel_buffer; + CVPixelBufferCreate(kCFAllocatorDefault, 640, 480, kCVPixelFormatType_32ARGB, nil, &pixel_buffer); + rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = + rtc::make_ref_counted<webrtc::ObjCFrameBuffer>( + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixel_buffer]); + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + std::vector<webrtc::VideoFrameType> frame_types; + + EXPECT_EQ(encoder->Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK); +} + +- (void)testEncodeReturnsErrorOnFail { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + CVPixelBufferRef pixel_buffer; + CVPixelBufferCreate(kCFAllocatorDefault, 640, 480, kCVPixelFormatType_32ARGB, nil, &pixel_buffer); + rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = + rtc::make_ref_counted<webrtc::ObjCFrameBuffer>( + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixel_buffer]); + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + std::vector<webrtc::VideoFrameType> frame_types; + + EXPECT_EQ(encoder->Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_ERROR); +} + +- (void)testReleaseEncodeReturnsOKOnSuccess { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateOKEncoderFactory()); + + EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK); +} + +- (void)testReleaseEncodeReturnsErrorOnFail { + std::unique_ptr<webrtc::VideoEncoder> encoder = GetObjCEncoder(CreateErrorEncoderFactory()); + + EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_ERROR); +} + +- (void)testGetSupportedFormats { + webrtc::ObjCVideoEncoderFactory encoder_factory(CreateOKEncoderFactory()); + std::vector<webrtc::SdpVideoFormat> supportedFormats = encoder_factory.GetSupportedFormats(); + EXPECT_EQ(supportedFormats.size(), 1u); + EXPECT_EQ(supportedFormats[0].name, "H264"); +} + +- (void)testGetImplementations { + webrtc::ObjCVideoEncoderFactory encoder_factory(CreateOKEncoderFactory()); + std::vector<webrtc::SdpVideoFormat> supportedFormats = encoder_factory.GetImplementations(); + EXPECT_EQ(supportedFormats.size(), 1u); + EXPECT_EQ(supportedFormats[0].name, "H264"); +} + +@end diff --git a/third_party/libwebrtc/sdk/objc/unittests/scoped_cftyperef_tests.mm b/third_party/libwebrtc/sdk/objc/unittests/scoped_cftyperef_tests.mm new file mode 100644 index 0000000000..a354410ede --- /dev/null +++ b/third_party/libwebrtc/sdk/objc/unittests/scoped_cftyperef_tests.mm @@ -0,0 +1,111 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import <XCTest/XCTest.h> + +#include "sdk/objc/helpers/scoped_cftyperef.h" + +namespace { +struct TestType { + TestType() : has_value(true) {} + TestType(bool b) : has_value(b) {} + explicit operator bool() { return has_value; } + bool has_value; + int retain_count = 0; +}; + +typedef TestType* TestTypeRef; + +struct TestTypeTraits { + static TestTypeRef InvalidValue() { return TestTypeRef(false); } + static void Release(TestTypeRef t) { t->retain_count--; } + static TestTypeRef Retain(TestTypeRef t) { + t->retain_count++; + return t; + } +}; +} // namespace + +using ScopedTestType = rtc::internal::ScopedTypeRef<TestTypeRef, TestTypeTraits>; + +// In these tests we sometime introduce variables just to +// observe side-effects. Ignore the compilers complaints. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-variable" + +@interface ScopedTypeRefTests : XCTestCase +@end + +@implementation ScopedTypeRefTests + +- (void)testShouldNotRetainByDefault { + TestType a; + ScopedTestType ref(&a); + XCTAssertEqual(0, a.retain_count); +} + +- (void)testShouldRetainWithPolicy { + TestType a; + ScopedTestType ref(&a, rtc::RetainPolicy::RETAIN); + XCTAssertEqual(1, a.retain_count); +} + +- (void)testShouldReleaseWhenLeavingScope { + TestType a; + XCTAssertEqual(0, a.retain_count); + { + ScopedTestType ref(&a, rtc::RetainPolicy::RETAIN); + XCTAssertEqual(1, a.retain_count); + } + XCTAssertEqual(0, a.retain_count); +} + +- (void)testShouldBeCopyable { + TestType a; + XCTAssertEqual(0, a.retain_count); + { + ScopedTestType ref1(&a, rtc::RetainPolicy::RETAIN); + XCTAssertEqual(1, a.retain_count); + ScopedTestType ref2 = ref1; + XCTAssertEqual(2, a.retain_count); + } + XCTAssertEqual(0, a.retain_count); +} + +- (void)testCanReleaseOwnership { + TestType a; + XCTAssertEqual(0, a.retain_count); + { + ScopedTestType ref(&a, rtc::RetainPolicy::RETAIN); + XCTAssertEqual(1, a.retain_count); + TestTypeRef b = ref.release(); + } + XCTAssertEqual(1, a.retain_count); +} + +- (void)testShouldBeTestableForTruthiness { + ScopedTestType ref; + XCTAssertFalse(ref); + TestType a; + ref = &a; + XCTAssertTrue(ref); + ref.release(); + XCTAssertFalse(ref); +} + +- (void)testShouldProvideAccessToWrappedType { + TestType a; + ScopedTestType ref(&a); + XCTAssertEqual(&(a.retain_count), &(ref->retain_count)); +} + +@end + +#pragma clang diagnostic pop diff --git a/third_party/libwebrtc/sdk/videocapture_objc_gn/moz.build b/third_party/libwebrtc/sdk/videocapture_objc_gn/moz.build new file mode 100644 index 0000000000..7bfe020e4d --- /dev/null +++ b/third_party/libwebrtc/sdk/videocapture_objc_gn/moz.build @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +CMFLAGS += [ + "-fobjc-arc" +] + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MAC"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" +DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True +DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" +DEFINES["__STDC_CONSTANT_MACROS"] = True +DEFINES["__STDC_FORMAT_MACROS"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/sdk/objc/", + "/third_party/libwebrtc/sdk/objc/base/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/sdk/objc/components/capturer/RTCCameraVideoCapturer.m", + "/third_party/libwebrtc/sdk/objc/components/capturer/RTCFileVideoCapturer.m" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["_DEBUG"] = True + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +Library("videocapture_objc_gn") diff --git a/third_party/libwebrtc/sdk/videoframebuffer_objc_gn/moz.build b/third_party/libwebrtc/sdk/videoframebuffer_objc_gn/moz.build new file mode 100644 index 0000000000..9e7f52704a --- /dev/null +++ b/third_party/libwebrtc/sdk/videoframebuffer_objc_gn/moz.build @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +CMMFLAGS += [ + "-fobjc-arc" +] + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MAC"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_POSIX"] = True +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" +DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True +DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" +DEFINES["__STDC_CONSTANT_MACROS"] = True +DEFINES["__STDC_FORMAT_MACROS"] = True + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/media/libyuv/", + "/media/libyuv/libyuv/include/", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/sdk/objc/", + "/third_party/libwebrtc/sdk/objc/base/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm", + "/third_party/libwebrtc/sdk/objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.mm", + "/third_party/libwebrtc/sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + DEFINES["_DEBUG"] = True + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +Library("videoframebuffer_objc_gn") |