summaryrefslogtreecommitdiffstats
path: root/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/webrtc/libwebrtcglue/VideoConduit.cpp')
-rw-r--r--dom/media/webrtc/libwebrtcglue/VideoConduit.cpp1902
1 files changed, 1902 insertions, 0 deletions
diff --git a/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp b/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp
new file mode 100644
index 0000000000..f99f818d48
--- /dev/null
+++ b/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp
@@ -0,0 +1,1902 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "VideoConduit.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+
+#include "common/browser_logging/CSFLog.h"
+#include "common/YuvStamper.h"
+#include "GmpVideoCodec.h"
+#include "MediaConduitControl.h"
+#include "MediaDataCodec.h"
+#include "mozilla/dom/RTCRtpSourcesBinding.h"
+#include "mozilla/media/MediaUtils.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "mozilla/TemplateLib.h"
+#include "nsIGfxInfo.h"
+#include "nsIPrefBranch.h"
+#include "nsIPrefService.h"
+#include "nsServiceManagerUtils.h"
+#include "RtpRtcpConfig.h"
+#include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION
+#include "Tracing.h"
+#include "VideoStreamFactory.h"
+#include "WebrtcCallWrapper.h"
+#include "WebrtcGmpVideoCodec.h"
+
+// libwebrtc includes
+#include "api/transport/bitrate_settings.h"
+#include "api/video_codecs/h264_profile_level_id.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+#include "media/base/media_constants.h"
+#include "media/engine/encoder_simulcast_proxy.h"
+#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+
+#ifdef MOZ_WIDGET_ANDROID
+# include "VideoEngine.h"
+#endif
+
+// for ntohs
+#ifdef _MSC_VER
+# include "Winsock2.h"
+#else
+# include <netinet/in.h>
+#endif
+
+#define INVALID_RTP_PAYLOAD 255 // valid payload types are 0 to 127
+
+namespace mozilla {
+
+namespace {
+
+const char* vcLogTag = "WebrtcVideoSessionConduit";
+#ifdef LOGTAG
+# undef LOGTAG
+#endif
+#define LOGTAG vcLogTag
+
+using namespace cricket;
+using LocalDirection = MediaSessionConduitLocalDirection;
+
+const int kNullPayloadType = -1;
+const char kRtcpFbCcmParamTmmbr[] = "tmmbr";
+
+// The number of frame buffers WebrtcVideoConduit may create before returning
+// errors.
+// Sometimes these are released synchronously but they can be forwarded all the
+// way to the encoder for asynchronous encoding. With a pool size of 5,
+// we allow 1 buffer for the current conversion, and 4 buffers to be queued at
+// the encoder.
+#define SCALER_BUFFER_POOL_SIZE 5
+
+// The pixel alignment to use for the highest resolution layer when simulcast
+// is active and one or more layers are being scaled.
+#define SIMULCAST_RESOLUTION_ALIGNMENT 16
+
+template <class t>
+void ConstrainPreservingAspectRatioExact(uint32_t max_fs, t* width, t* height) {
+ // We could try to pick a better starting divisor, but it won't make any real
+ // performance difference.
+ for (size_t d = 1; d < std::min(*width, *height); ++d) {
+ if ((*width % d) || (*height % d)) {
+ continue; // Not divisible
+ }
+
+ if (((*width) * (*height)) / (d * d) <= max_fs) {
+ *width /= d;
+ *height /= d;
+ return;
+ }
+ }
+
+ *width = 0;
+ *height = 0;
+}
+
+/**
+ * Perform validation on the codecConfig to be applied
+ */
+MediaConduitErrorCode ValidateCodecConfig(const VideoCodecConfig& codecInfo) {
+ if (codecInfo.mName.empty()) {
+ CSFLogError(LOGTAG, "%s Empty Payload Name ", __FUNCTION__);
+ return kMediaConduitMalformedArgument;
+ }
+
+ return kMediaConduitNoError;
+}
+
+webrtc::VideoCodecType SupportedCodecType(webrtc::VideoCodecType aType) {
+ switch (aType) {
+ case webrtc::VideoCodecType::kVideoCodecVP8:
+ case webrtc::VideoCodecType::kVideoCodecVP9:
+ case webrtc::VideoCodecType::kVideoCodecH264:
+ return aType;
+ default:
+ return webrtc::VideoCodecType::kVideoCodecGeneric;
+ }
+ // NOTREACHED
+}
+
+// Call thread only.
+rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
+ConfigureVideoEncoderSettings(const VideoCodecConfig& aConfig,
+ const WebrtcVideoConduit* aConduit,
+ webrtc::SdpVideoFormat::Parameters& aParameters) {
+ bool is_screencast =
+ aConduit->CodecMode() == webrtc::VideoCodecMode::kScreensharing;
+ // No automatic resizing when using simulcast or screencast.
+ bool automatic_resize = !is_screencast && aConfig.mEncodings.size() <= 1;
+ bool denoising;
+ bool codec_default_denoising = false;
+ if (is_screencast) {
+ denoising = false;
+ } else {
+ // Use codec default if video_noise_reduction is unset.
+ denoising = aConduit->Denoising();
+ codec_default_denoising = !denoising;
+ }
+
+ if (aConfig.mName == kH264CodecName) {
+ aParameters[kH264FmtpPacketizationMode] =
+ std::to_string(aConfig.mPacketizationMode);
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << std::setw(2) << static_cast<uint32_t>(aConfig.mProfile);
+ ss << std::setw(2) << static_cast<uint32_t>(aConfig.mConstraints);
+ ss << std::setw(2) << static_cast<uint32_t>(aConfig.mLevel);
+ std::string profileLevelId = ss.str();
+ auto parsedProfileLevelId =
+ webrtc::ParseH264ProfileLevelId(profileLevelId.c_str());
+ MOZ_DIAGNOSTIC_ASSERT(parsedProfileLevelId);
+ if (parsedProfileLevelId) {
+ aParameters[kH264FmtpProfileLevelId] = profileLevelId;
+ }
+ }
+ aParameters[kH264FmtpSpropParameterSets] = aConfig.mSpropParameterSets;
+ }
+ if (aConfig.mName == kVp8CodecName) {
+ webrtc::VideoCodecVP8 vp8_settings =
+ webrtc::VideoEncoder::GetDefaultVp8Settings();
+ vp8_settings.automaticResizeOn = automatic_resize;
+ // VP8 denoising is enabled by default.
+ vp8_settings.denoisingOn = codec_default_denoising ? true : denoising;
+ return rtc::scoped_refptr<
+ webrtc::VideoEncoderConfig::EncoderSpecificSettings>(
+ new rtc::RefCountedObject<
+ webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(
+ vp8_settings));
+ }
+ if (aConfig.mName == kVp9CodecName) {
+ webrtc::VideoCodecVP9 vp9_settings =
+ webrtc::VideoEncoder::GetDefaultVp9Settings();
+ if (!is_screencast) {
+ // Always configure only 1 spatial layer for screencapture as libwebrtc
+ // has some special requirements when SVC is active. For non-screencapture
+ // the spatial layers are experimentally configurable via a pref.
+ vp9_settings.numberOfSpatialLayers = aConduit->SpatialLayers();
+ }
+ // VP9 denoising is disabled by default.
+ vp9_settings.denoisingOn = codec_default_denoising ? false : denoising;
+ return rtc::scoped_refptr<
+ webrtc::VideoEncoderConfig::EncoderSpecificSettings>(
+ new rtc::RefCountedObject<
+ webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings));
+ }
+ return nullptr;
+}
+
+uint32_t GenerateRandomSSRC() {
+ uint32_t ssrc;
+ do {
+ SECStatus rv = PK11_GenerateRandom(reinterpret_cast<unsigned char*>(&ssrc),
+ sizeof(ssrc));
+ MOZ_RELEASE_ASSERT(rv == SECSuccess);
+ } while (ssrc == 0); // webrtc.org code has fits if you select an SSRC of 0
+
+ return ssrc;
+}
+
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator==(const rtc::VideoSinkWants& aThis,
+ const rtc::VideoSinkWants& aOther) {
+ // This would have to be expanded should we make use of more members of
+ // rtc::VideoSinkWants.
+ return aThis.max_pixel_count == aOther.max_pixel_count &&
+ aThis.max_framerate_fps == aOther.max_framerate_fps &&
+ aThis.resolution_alignment == aOther.resolution_alignment;
+}
+
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator!=(const rtc::VideoSinkWants& aThis,
+ const rtc::VideoSinkWants& aOther) {
+ return !(aThis == aOther);
+}
+
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator!=(
+ const webrtc::VideoReceiveStreamInterface::Config::Rtp& aThis,
+ const webrtc::VideoReceiveStreamInterface::Config::Rtp& aOther) {
+ return aThis.remote_ssrc != aOther.remote_ssrc ||
+ aThis.local_ssrc != aOther.local_ssrc ||
+ aThis.rtcp_mode != aOther.rtcp_mode ||
+ aThis.rtcp_xr.receiver_reference_time_report !=
+ aOther.rtcp_xr.receiver_reference_time_report ||
+ aThis.remb != aOther.remb || aThis.tmmbr != aOther.tmmbr ||
+ aThis.keyframe_method != aOther.keyframe_method ||
+ aThis.lntf.enabled != aOther.lntf.enabled ||
+ aThis.nack.rtp_history_ms != aOther.nack.rtp_history_ms ||
+ aThis.ulpfec_payload_type != aOther.ulpfec_payload_type ||
+ aThis.red_payload_type != aOther.red_payload_type ||
+ aThis.rtx_ssrc != aOther.rtx_ssrc ||
+ aThis.protected_by_flexfec != aOther.protected_by_flexfec ||
+ aThis.rtx_associated_payload_types !=
+ aOther.rtx_associated_payload_types ||
+ aThis.raw_payload_types != aOther.raw_payload_types ||
+ aThis.extensions != aOther.extensions;
+}
+
+#ifdef DEBUG
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator==(
+ const webrtc::VideoReceiveStreamInterface::Config::Rtp& aThis,
+ const webrtc::VideoReceiveStreamInterface::Config::Rtp& aOther) {
+ return !(aThis != aOther);
+}
+#endif
+
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator!=(const webrtc::RtpConfig& aThis,
+ const webrtc::RtpConfig& aOther) {
+ return aThis.ssrcs != aOther.ssrcs || aThis.rids != aOther.rids ||
+ aThis.mid != aOther.mid || aThis.rtcp_mode != aOther.rtcp_mode ||
+ aThis.max_packet_size != aOther.max_packet_size ||
+ aThis.extmap_allow_mixed != aOther.extmap_allow_mixed ||
+ aThis.extensions != aOther.extensions ||
+ aThis.payload_name != aOther.payload_name ||
+ aThis.payload_type != aOther.payload_type ||
+ aThis.raw_payload != aOther.raw_payload ||
+ aThis.lntf.enabled != aOther.lntf.enabled ||
+ aThis.nack.rtp_history_ms != aOther.nack.rtp_history_ms ||
+ !(aThis.ulpfec == aOther.ulpfec) ||
+ aThis.flexfec.payload_type != aOther.flexfec.payload_type ||
+ aThis.flexfec.ssrc != aOther.flexfec.ssrc ||
+ aThis.flexfec.protected_media_ssrcs !=
+ aOther.flexfec.protected_media_ssrcs ||
+ aThis.rtx.ssrcs != aOther.rtx.ssrcs ||
+ aThis.rtx.payload_type != aOther.rtx.payload_type ||
+ aThis.c_name != aOther.c_name;
+}
+
+#ifdef DEBUG
+// TODO: Make this a defaulted operator when we have c++20 (bug 1731036).
+bool operator==(const webrtc::RtpConfig& aThis,
+ const webrtc::RtpConfig& aOther) {
+ return !(aThis != aOther);
+}
+#endif
+
+} // namespace
+
+/**
+ * Factory Method for VideoConduit
+ */
+RefPtr<VideoSessionConduit> VideoSessionConduit::Create(
+ RefPtr<WebrtcCallWrapper> aCall, nsCOMPtr<nsISerialEventTarget> aStsThread,
+ Options aOptions, std::string aPCHandle,
+ const TrackingId& aRecvTrackingId) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(aCall, "missing required parameter: aCall");
+ CSFLogVerbose(LOGTAG, "%s", __FUNCTION__);
+
+ if (!aCall) {
+ return nullptr;
+ }
+
+ auto obj = MakeRefPtr<WebrtcVideoConduit>(
+ std::move(aCall), std::move(aStsThread), std::move(aOptions),
+ std::move(aPCHandle), aRecvTrackingId);
+ if (obj->Init() != kMediaConduitNoError) {
+ CSFLogError(LOGTAG, "%s VideoConduit Init Failed ", __FUNCTION__);
+ return nullptr;
+ }
+ CSFLogVerbose(LOGTAG, "%s Successfully created VideoConduit ", __FUNCTION__);
+ return obj.forget();
+}
+
+#define INIT_MIRROR(name, val) \
+ name(aCallThread, val, "WebrtcVideoConduit::Control::" #name " (Mirror)")
+WebrtcVideoConduit::Control::Control(const RefPtr<AbstractThread>& aCallThread)
+ : INIT_MIRROR(mReceiving, false),
+ INIT_MIRROR(mTransmitting, false),
+ INIT_MIRROR(mLocalSsrcs, Ssrcs()),
+ INIT_MIRROR(mLocalRtxSsrcs, Ssrcs()),
+ INIT_MIRROR(mLocalCname, std::string()),
+ INIT_MIRROR(mMid, std::string()),
+ INIT_MIRROR(mRemoteSsrc, 0),
+ INIT_MIRROR(mRemoteRtxSsrc, 0),
+ INIT_MIRROR(mSyncGroup, std::string()),
+ INIT_MIRROR(mLocalRecvRtpExtensions, RtpExtList()),
+ INIT_MIRROR(mLocalSendRtpExtensions, RtpExtList()),
+ INIT_MIRROR(mSendCodec, Nothing()),
+ INIT_MIRROR(mSendRtpRtcpConfig, Nothing()),
+ INIT_MIRROR(mRecvCodecs, std::vector<VideoCodecConfig>()),
+ INIT_MIRROR(mRecvRtpRtcpConfig, Nothing()),
+ INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo) {}
+#undef INIT_MIRROR
+
+WebrtcVideoConduit::WebrtcVideoConduit(
+ RefPtr<WebrtcCallWrapper> aCall, nsCOMPtr<nsISerialEventTarget> aStsThread,
+ Options aOptions, std::string aPCHandle, const TrackingId& aRecvTrackingId)
+ : mRendererMonitor("WebrtcVideoConduit::mRendererMonitor"),
+ mCallThread(aCall->mCallThread),
+ mStsThread(std::move(aStsThread)),
+ mControl(aCall->mCallThread),
+ mWatchManager(this, aCall->mCallThread),
+ mMutex("WebrtcVideoConduit::mMutex"),
+ mDecoderFactory(MakeUnique<WebrtcVideoDecoderFactory>(
+ mCallThread.get(), aPCHandle, aRecvTrackingId)),
+ mEncoderFactory(MakeUnique<WebrtcVideoEncoderFactory>(
+ mCallThread.get(), std::move(aPCHandle))),
+ mBufferPool(false, SCALER_BUFFER_POOL_SIZE),
+ mEngineTransmitting(false),
+ mEngineReceiving(false),
+ mVideoLatencyTestEnable(aOptions.mVideoLatencyTestEnable),
+ mMinBitrate(aOptions.mMinBitrate),
+ mStartBitrate(aOptions.mStartBitrate),
+ mPrefMaxBitrate(aOptions.mPrefMaxBitrate),
+ mMinBitrateEstimate(aOptions.mMinBitrateEstimate),
+ mDenoising(aOptions.mDenoising),
+ mLockScaling(aOptions.mLockScaling),
+ mSpatialLayers(aOptions.mSpatialLayers),
+ mTemporalLayers(aOptions.mTemporalLayers),
+ mCall(std::move(aCall)),
+ mSendTransport(this),
+ mRecvTransport(this),
+ mSendStreamConfig(&mSendTransport),
+ mVideoStreamFactory("WebrtcVideoConduit::mVideoStreamFactory"),
+ mRecvStreamConfig(&mRecvTransport) {
+ mRecvStreamConfig.rtp.rtcp_event_observer = this;
+}
+
+WebrtcVideoConduit::~WebrtcVideoConduit() {
+ CSFLogDebug(LOGTAG, "%s ", __FUNCTION__);
+
+ MOZ_ASSERT(!mSendStream && !mRecvStream,
+ "Call DeleteStreams prior to ~WebrtcVideoConduit.");
+}
+
+#define CONNECT(aCanonical, aMirror) \
+ do { \
+ (aMirror).Connect(aCanonical); \
+ mWatchManager.Watch(aMirror, &WebrtcVideoConduit::OnControlConfigChange); \
+ } while (0)
+
+void WebrtcVideoConduit::InitControl(VideoConduitControlInterface* aControl) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ CONNECT(aControl->CanonicalReceiving(), mControl.mReceiving);
+ CONNECT(aControl->CanonicalTransmitting(), mControl.mTransmitting);
+ CONNECT(aControl->CanonicalLocalSsrcs(), mControl.mLocalSsrcs);
+ CONNECT(aControl->CanonicalLocalVideoRtxSsrcs(), mControl.mLocalRtxSsrcs);
+ CONNECT(aControl->CanonicalLocalCname(), mControl.mLocalCname);
+ CONNECT(aControl->CanonicalMid(), mControl.mMid);
+ CONNECT(aControl->CanonicalRemoteSsrc(), mControl.mRemoteSsrc);
+ CONNECT(aControl->CanonicalRemoteVideoRtxSsrc(), mControl.mRemoteRtxSsrc);
+ CONNECT(aControl->CanonicalSyncGroup(), mControl.mSyncGroup);
+ CONNECT(aControl->CanonicalLocalRecvRtpExtensions(),
+ mControl.mLocalRecvRtpExtensions);
+ CONNECT(aControl->CanonicalLocalSendRtpExtensions(),
+ mControl.mLocalSendRtpExtensions);
+ CONNECT(aControl->CanonicalVideoSendCodec(), mControl.mSendCodec);
+ CONNECT(aControl->CanonicalVideoSendRtpRtcpConfig(),
+ mControl.mSendRtpRtcpConfig);
+ CONNECT(aControl->CanonicalVideoRecvCodecs(), mControl.mRecvCodecs);
+ CONNECT(aControl->CanonicalVideoRecvRtpRtcpConfig(),
+ mControl.mRecvRtpRtcpConfig);
+ CONNECT(aControl->CanonicalVideoCodecMode(), mControl.mCodecMode);
+}
+
+#undef CONNECT
+
+void WebrtcVideoConduit::OnControlConfigChange() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ bool encoderReconfigureNeeded = false;
+ bool remoteSsrcUpdateNeeded = false;
+ bool sendStreamRecreationNeeded = false;
+
+ if (mControl.mRemoteSsrc.Ref() != mControl.mConfiguredRemoteSsrc) {
+ mControl.mConfiguredRemoteSsrc = mControl.mRemoteSsrc;
+ remoteSsrcUpdateNeeded = true;
+ }
+
+ if (mControl.mRemoteRtxSsrc.Ref() != mControl.mConfiguredRemoteRtxSsrc) {
+ mControl.mConfiguredRemoteRtxSsrc = mControl.mRemoteRtxSsrc;
+ remoteSsrcUpdateNeeded = true;
+ }
+
+ if (mControl.mSyncGroup.Ref() != mRecvStreamConfig.sync_group) {
+ mRecvStreamConfig.sync_group = mControl.mSyncGroup;
+ }
+
+ if (mControl.mLocalRecvRtpExtensions.Ref() !=
+ mRecvStreamConfig.rtp.extensions) {
+ mRecvStreamConfig.rtp.extensions = mControl.mLocalRecvRtpExtensions;
+ }
+
+ if (const auto [codecConfigList, rtpRtcpConfig] = std::make_pair(
+ mControl.mRecvCodecs.Ref(), mControl.mRecvRtpRtcpConfig.Ref());
+ !codecConfigList.empty() && rtpRtcpConfig.isSome() &&
+ (codecConfigList != mControl.mConfiguredRecvCodecs ||
+ rtpRtcpConfig != mControl.mConfiguredRecvRtpRtcpConfig)) {
+ mControl.mConfiguredRecvCodecs = codecConfigList;
+ mControl.mConfiguredRecvRtpRtcpConfig = rtpRtcpConfig;
+
+ webrtc::VideoReceiveStreamInterface::Config::Rtp newRtp(
+ mRecvStreamConfig.rtp);
+ MOZ_ASSERT(newRtp == mRecvStreamConfig.rtp);
+ newRtp.rtx_associated_payload_types.clear();
+ newRtp.rtcp_mode = rtpRtcpConfig->GetRtcpMode();
+ newRtp.nack.rtp_history_ms = 0;
+ newRtp.remb = false;
+ newRtp.tmmbr = false;
+ newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kNone;
+ newRtp.ulpfec_payload_type = kNullPayloadType;
+ newRtp.red_payload_type = kNullPayloadType;
+ bool use_fec = false;
+ bool configuredH264 = false;
+ std::vector<webrtc::VideoReceiveStreamInterface::Decoder> recv_codecs;
+
+ // Try Applying the codecs in the list
+ // we treat as success if at least one codec was applied and reception was
+ // started successfully.
+ for (const auto& codec_config : codecConfigList) {
+ if (auto condError = ValidateCodecConfig(codec_config);
+ condError != kMediaConduitNoError) {
+ CSFLogError(LOGTAG, "Invalid recv codec config for %s decoder: %i",
+ codec_config.mName.c_str(), condError);
+ continue;
+ }
+
+ if (codec_config.mName == kH264CodecName) {
+ // TODO(bug 1200768): We can only handle configuring one recv H264 codec
+ if (configuredH264) {
+ continue;
+ }
+ configuredH264 = true;
+ }
+
+ if (codec_config.mName == kUlpfecCodecName) {
+ newRtp.ulpfec_payload_type = codec_config.mType;
+ continue;
+ }
+
+ if (codec_config.mName == kRedCodecName) {
+ newRtp.red_payload_type = codec_config.mType;
+ continue;
+ }
+
+ if (SupportedCodecType(
+ webrtc::PayloadStringToCodecType(codec_config.mName)) ==
+ webrtc::VideoCodecType::kVideoCodecGeneric) {
+ CSFLogError(LOGTAG, "%s Unknown decoder type: %s", __FUNCTION__,
+ codec_config.mName.c_str());
+ continue;
+ }
+
+ // Check for the keyframe request type: PLI is preferred over FIR, and FIR
+ // is preferred over none.
+ if (codec_config.RtcpFbNackIsSet(kRtcpFbNackParamPli)) {
+ newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kPliRtcp;
+ } else if (newRtp.keyframe_method !=
+ webrtc::KeyFrameReqMethod::kPliRtcp &&
+ codec_config.RtcpFbCcmIsSet(kRtcpFbCcmParamFir)) {
+ newRtp.keyframe_method = webrtc::KeyFrameReqMethod::kFirRtcp;
+ }
+
+ // What if codec A has Nack and REMB, and codec B has TMMBR, and codec C
+ // has none? In practice, that's not a useful configuration, and
+ // VideoReceiveStream::Config can't represent that, so simply union the
+ // (boolean) settings
+ if (codec_config.RtcpFbNackIsSet(kParamValueEmpty)) {
+ newRtp.nack.rtp_history_ms = 1000;
+ }
+ newRtp.tmmbr |= codec_config.RtcpFbCcmIsSet(kRtcpFbCcmParamTmmbr);
+ newRtp.remb |= codec_config.RtcpFbRembIsSet();
+ use_fec |= codec_config.RtcpFbFECIsSet();
+
+ if (codec_config.RtxPayloadTypeIsSet()) {
+ newRtp.rtx_associated_payload_types[codec_config.mRTXPayloadType] =
+ codec_config.mType;
+ }
+
+ auto& decoder = recv_codecs.emplace_back();
+ decoder.video_format = webrtc::SdpVideoFormat(codec_config.mName);
+ decoder.payload_type = codec_config.mType;
+ }
+
+ if (!use_fec) {
+ // Reset to defaults
+ newRtp.ulpfec_payload_type = kNullPayloadType;
+ newRtp.red_payload_type = kNullPayloadType;
+ }
+
+ // TODO: This would be simpler, but for some reason gives
+ // "error: invalid operands to binary expression
+ // ('webrtc::VideoReceiveStreamInterface::Decoder' and
+ // 'webrtc::VideoReceiveStreamInterface::Decoder')"
+ // if (recv_codecs != mRecvStreamConfig.decoders) {
+ if (!std::equal(recv_codecs.begin(), recv_codecs.end(),
+ mRecvStreamConfig.decoders.begin(),
+ mRecvStreamConfig.decoders.end(),
+ [](const auto& aLeft, const auto& aRight) {
+ return aLeft == aRight;
+ })) {
+ if (recv_codecs.empty()) {
+ CSFLogError(LOGTAG, "%s Found no valid receive codecs", __FUNCTION__);
+ }
+ mRecvStreamConfig.decoders = std::move(recv_codecs);
+ }
+
+ if (mRecvStreamConfig.rtp != newRtp) {
+ mRecvStreamConfig.rtp = newRtp;
+ }
+ }
+
+ {
+ // mSendStreamConfig and other members need the lock
+ MutexAutoLock lock(mMutex);
+ if (mControl.mLocalSsrcs.Ref() != mSendStreamConfig.rtp.ssrcs) {
+ mSendStreamConfig.rtp.ssrcs = mControl.mLocalSsrcs;
+ sendStreamRecreationNeeded = true;
+
+ const uint32_t localSsrc = mSendStreamConfig.rtp.ssrcs.empty()
+ ? 0
+ : mSendStreamConfig.rtp.ssrcs.front();
+ if (localSsrc != mRecvStreamConfig.rtp.local_ssrc) {
+ mRecvStreamConfig.rtp.local_ssrc = localSsrc;
+ }
+ }
+
+ {
+ Ssrcs localRtxSsrcs = mControl.mLocalRtxSsrcs.Ref();
+ if (!mControl.mSendCodec.Ref()
+ .map([](const auto& aCodec) {
+ return aCodec.RtxPayloadTypeIsSet();
+ })
+ .valueOr(false)) {
+ localRtxSsrcs.clear();
+ }
+ if (localRtxSsrcs != mSendStreamConfig.rtp.rtx.ssrcs) {
+ mSendStreamConfig.rtp.rtx.ssrcs = localRtxSsrcs;
+ sendStreamRecreationNeeded = true;
+ }
+ }
+
+ if (mControl.mLocalCname.Ref() != mSendStreamConfig.rtp.c_name) {
+ mSendStreamConfig.rtp.c_name = mControl.mLocalCname;
+ sendStreamRecreationNeeded = true;
+ }
+
+ if (mControl.mMid.Ref() != mSendStreamConfig.rtp.mid) {
+ mSendStreamConfig.rtp.mid = mControl.mMid;
+ sendStreamRecreationNeeded = true;
+ }
+
+ if (mControl.mLocalSendRtpExtensions.Ref() !=
+ mSendStreamConfig.rtp.extensions) {
+ mSendStreamConfig.rtp.extensions = mControl.mLocalSendRtpExtensions;
+ sendStreamRecreationNeeded = true;
+ }
+
+ if (const auto [codecConfig, rtpRtcpConfig] = std::make_pair(
+ mControl.mSendCodec.Ref(), mControl.mSendRtpRtcpConfig.Ref());
+ codecConfig.isSome() && rtpRtcpConfig.isSome() &&
+ (codecConfig != mControl.mConfiguredSendCodec ||
+ rtpRtcpConfig != mControl.mConfiguredSendRtpRtcpConfig)) {
+ CSFLogDebug(LOGTAG, "Configuring codec %s", codecConfig->mName.c_str());
+ mControl.mConfiguredSendCodec = codecConfig;
+ mControl.mConfiguredSendRtpRtcpConfig = rtpRtcpConfig;
+
+ if (ValidateCodecConfig(*codecConfig) == kMediaConduitNoError) {
+ encoderReconfigureNeeded = true;
+
+ mCurSendCodecConfig = codecConfig;
+
+ size_t streamCount = std::min(codecConfig->mEncodings.size(),
+ (size_t)webrtc::kMaxSimulcastStreams);
+ size_t highestResolutionIndex = 0;
+ for (size_t i = 1; i < streamCount; ++i) {
+ if (codecConfig->mEncodings[i].constraints.scaleDownBy <
+ codecConfig->mEncodings[highestResolutionIndex]
+ .constraints.scaleDownBy) {
+ highestResolutionIndex = i;
+ }
+ }
+ MOZ_RELEASE_ASSERT(streamCount >= 1,
+ "streamCount should be at least one");
+
+ CSFLogDebug(LOGTAG,
+ "Updating send codec for VideoConduit:%p stream count:%zu",
+ this, streamCount);
+
+ // So we can comply with b=TIAS/b=AS/maxbr=X when input resolution
+ // changes
+ MOZ_ASSERT(codecConfig->mTias < INT_MAX);
+ mNegotiatedMaxBitrate = static_cast<int>(codecConfig->mTias);
+
+ if (mLastWidth == 0 && mMinBitrateEstimate != 0) {
+ // Only do this at the start; use "have we sent a frame" as a
+ // reasonable stand-in. min <= start <= max (but all three parameters
+ // are optional)
+ webrtc::BitrateSettings settings;
+ settings.min_bitrate_bps = mMinBitrateEstimate;
+ settings.start_bitrate_bps = mMinBitrateEstimate;
+ mCall->Call()->SetClientBitratePreferences(settings);
+ }
+
+ // XXX parse the encoded SPS/PPS data and set
+ // spsData/spsLen/ppsData/ppsLen
+ mEncoderConfig.video_format =
+ webrtc::SdpVideoFormat(codecConfig->mName);
+ mEncoderConfig.encoder_specific_settings =
+ ConfigureVideoEncoderSettings(
+ *codecConfig, this, mEncoderConfig.video_format.parameters);
+
+ mEncoderConfig.codec_type = SupportedCodecType(
+ webrtc::PayloadStringToCodecType(codecConfig->mName));
+ MOZ_RELEASE_ASSERT(mEncoderConfig.codec_type !=
+ webrtc::VideoCodecType::kVideoCodecGeneric);
+
+ mEncoderConfig.content_type =
+ mControl.mCodecMode.Ref() == webrtc::VideoCodecMode::kRealtimeVideo
+ ? webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo
+ : webrtc::VideoEncoderConfig::ContentType::kScreen;
+
+ mEncoderConfig.frame_drop_enabled =
+ mControl.mCodecMode.Ref() != webrtc::VideoCodecMode::kScreensharing;
+
+ mEncoderConfig.min_transmit_bitrate_bps = mMinBitrate;
+
+ // Set the max bitrate, defaulting to 10Mbps, checking:
+ // - pref
+ // - b=TIAS
+ // - codec constraints
+ // - encoding parameter if there's a single stream
+ int maxBps = KBPS(10000);
+ maxBps = MinIgnoreZero(maxBps, mPrefMaxBitrate);
+ maxBps = MinIgnoreZero(maxBps, mNegotiatedMaxBitrate);
+ maxBps = MinIgnoreZero(
+ maxBps, static_cast<int>(codecConfig->mEncodingConstraints.maxBr));
+ if (codecConfig->mEncodings.size() == 1) {
+ maxBps = MinIgnoreZero(
+ maxBps,
+ static_cast<int>(codecConfig->mEncodings[0].constraints.maxBr));
+ }
+ mEncoderConfig.max_bitrate_bps = maxBps;
+
+ // TODO this is for webrtc-priority, but needs plumbing bits
+ mEncoderConfig.bitrate_priority = 1.0;
+
+ // Expected max number of encodings
+ mEncoderConfig.number_of_streams = streamCount;
+
+ // libwebrtc disables this by default.
+ mSendStreamConfig.suspend_below_min_bitrate = false;
+
+ webrtc::RtpConfig newRtp = mSendStreamConfig.rtp;
+ MOZ_ASSERT(newRtp == mSendStreamConfig.rtp);
+ newRtp.payload_name = codecConfig->mName;
+ newRtp.payload_type = codecConfig->mType;
+ newRtp.rtcp_mode = rtpRtcpConfig->GetRtcpMode();
+ newRtp.max_packet_size = kVideoMtu;
+ newRtp.rtx.payload_type = codecConfig->RtxPayloadTypeIsSet()
+ ? codecConfig->mRTXPayloadType
+ : kNullPayloadType;
+
+ {
+ // See Bug 1297058, enabling FEC when basic NACK is to be enabled in
+ // H.264 is problematic
+ const bool useFECDefaults =
+ !codecConfig->RtcpFbFECIsSet() ||
+ (codecConfig->mName == kH264CodecName &&
+ codecConfig->RtcpFbNackIsSet(kParamValueEmpty));
+ newRtp.ulpfec.ulpfec_payload_type =
+ useFECDefaults ? kNullPayloadType
+ : codecConfig->mULPFECPayloadType;
+ newRtp.ulpfec.red_payload_type =
+ useFECDefaults ? kNullPayloadType : codecConfig->mREDPayloadType;
+ newRtp.ulpfec.red_rtx_payload_type =
+ useFECDefaults ? kNullPayloadType
+ : codecConfig->mREDRTXPayloadType;
+ }
+
+ newRtp.nack.rtp_history_ms =
+ codecConfig->RtcpFbNackIsSet(kParamValueEmpty) ? 1000 : 0;
+
+ {
+ newRtp.rids.clear();
+ bool has_rid = false;
+ for (size_t idx = 0; idx < streamCount; idx++) {
+ const auto& encoding = codecConfig->mEncodings[idx];
+ if (encoding.rid[0]) {
+ has_rid = true;
+ break;
+ }
+ }
+ if (has_rid) {
+ for (size_t idx = streamCount; idx > 0; idx--) {
+ const auto& encoding = codecConfig->mEncodings[idx - 1];
+ newRtp.rids.push_back(encoding.rid);
+ }
+ }
+ }
+ if (mSendStreamConfig.rtp != newRtp) {
+ mSendStreamConfig.rtp = newRtp;
+ sendStreamRecreationNeeded = true;
+ }
+
+ mEncoderConfig.video_stream_factory = CreateVideoStreamFactory();
+ }
+ }
+
+ {
+ const auto& mode = mControl.mCodecMode.Ref();
+ MOZ_ASSERT(mode == webrtc::VideoCodecMode::kRealtimeVideo ||
+ mode == webrtc::VideoCodecMode::kScreensharing);
+
+ auto contentType =
+ mode == webrtc::VideoCodecMode::kRealtimeVideo
+ ? webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo
+ : webrtc::VideoEncoderConfig::ContentType::kScreen;
+
+ if (contentType != mEncoderConfig.content_type) {
+ mEncoderConfig.video_stream_factory = CreateVideoStreamFactory();
+ encoderReconfigureNeeded = true;
+ }
+ }
+
+ if (remoteSsrcUpdateNeeded) {
+ SetRemoteSSRCConfig(mControl.mConfiguredRemoteSsrc,
+ mControl.mConfiguredRemoteRtxSsrc);
+ }
+
+ // Handle un-signalled SSRCs by creating random ones and then when they
+ // actually get set, we'll destroy and recreate.
+ if (mControl.mReceiving || mControl.mTransmitting) {
+ const auto remoteSsrc = mRecvStreamConfig.rtp.remote_ssrc;
+ const auto localSsrc = mRecvStreamConfig.rtp.local_ssrc;
+ const auto localSsrcs = mSendStreamConfig.rtp.ssrcs;
+ EnsureLocalSSRC();
+ if (mControl.mReceiving) {
+ EnsureRemoteSSRC();
+ }
+ if (localSsrc != mRecvStreamConfig.rtp.local_ssrc ||
+ remoteSsrc != mRecvStreamConfig.rtp.remote_ssrc) {
+ }
+ if (localSsrcs != mSendStreamConfig.rtp.ssrcs) {
+ sendStreamRecreationNeeded = true;
+ }
+ }
+
+ // Recreate receiving streams
+ if (mControl.mReceiving) {
+ DeleteRecvStream();
+ CreateRecvStream();
+ }
+ if (sendStreamRecreationNeeded) {
+ DeleteSendStream();
+ }
+ if (mControl.mTransmitting) {
+ CreateSendStream();
+ }
+ }
+
+ // We make sure to not hold the lock while stopping/starting/reconfiguring
+ // streams, so as to not cause deadlocks. These methods can cause our platform
+ // codecs to dispatch sync runnables to main, and main may grab the lock.
+
+ if (mSendStream && encoderReconfigureNeeded) {
+ MOZ_DIAGNOSTIC_ASSERT(
+ mSendStreamConfig.rtp.ssrcs.size() == mEncoderConfig.number_of_streams,
+ "Each video substream must have a corresponding ssrc.");
+ mSendStream->ReconfigureVideoEncoder(mEncoderConfig.Copy());
+ }
+
+ if (!mControl.mReceiving) {
+ StopReceiving();
+ }
+ if (!mControl.mTransmitting) {
+ StopTransmitting();
+ }
+
+ if (mControl.mReceiving) {
+ StartReceiving();
+ }
+ if (mControl.mTransmitting) {
+ StartTransmitting();
+ }
+}
+
+std::vector<unsigned int> WebrtcVideoConduit::GetLocalSSRCs() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ return mSendStreamConfig.rtp.ssrcs;
+}
+
+Maybe<Ssrc> WebrtcVideoConduit::GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ for (size_t i = 0; i < mSendStreamConfig.rtp.ssrcs.size() &&
+ i < mSendStreamConfig.rtp.rtx.ssrcs.size();
+ ++i) {
+ if (mSendStreamConfig.rtp.ssrcs[i] == aSsrc) {
+ return Some(mSendStreamConfig.rtp.rtx.ssrcs[i]);
+ }
+ }
+ return Nothing();
+}
+
+void WebrtcVideoConduit::DeleteSendStream() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ if (!mSendStream) {
+ return;
+ }
+
+ mCall->Call()->DestroyVideoSendStream(mSendStream);
+ mEngineTransmitting = false;
+ mSendStream = nullptr;
+
+ // Reset base_seqs in case ssrcs get re-used.
+ mRtpSendBaseSeqs.clear();
+}
+
+void WebrtcVideoConduit::CreateSendStream() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ if (mSendStream) {
+ return;
+ }
+
+ nsAutoString codecName;
+ codecName.AssignASCII(mSendStreamConfig.rtp.payload_name.c_str());
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_VIDEO_SEND_CODEC_USED,
+ codecName, 1);
+
+ mSendStreamConfig.encoder_settings.encoder_factory = mEncoderFactory.get();
+ mSendStreamConfig.encoder_settings.bitrate_allocator_factory =
+ mCall->mVideoBitrateAllocatorFactory.get();
+
+ MOZ_DIAGNOSTIC_ASSERT(
+ mSendStreamConfig.rtp.ssrcs.size() == mEncoderConfig.number_of_streams,
+ "Each video substream must have a corresponding ssrc.");
+
+ mSendStream = mCall->Call()->CreateVideoSendStream(mSendStreamConfig.Copy(),
+ mEncoderConfig.Copy());
+
+ mSendStream->SetSource(this, webrtc::DegradationPreference::BALANCED);
+}
+
+void WebrtcVideoConduit::DeleteRecvStream() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ if (!mRecvStream) {
+ return;
+ }
+
+ mCall->Call()->DestroyVideoReceiveStream(mRecvStream);
+ mEngineReceiving = false;
+ mRecvStream = nullptr;
+}
+
+void WebrtcVideoConduit::CreateRecvStream() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ if (mRecvStream) {
+ return;
+ }
+
+ mRecvStreamConfig.renderer = this;
+
+ for (auto& decoder : mRecvStreamConfig.decoders) {
+ nsAutoString codecName;
+ codecName.AssignASCII(decoder.video_format.name.c_str());
+ Telemetry::ScalarAdd(Telemetry::ScalarID::WEBRTC_VIDEO_RECV_CODEC_USED,
+ codecName, 1);
+ }
+
+ mRecvStreamConfig.decoder_factory = mDecoderFactory.get();
+
+ mRecvStream =
+ mCall->Call()->CreateVideoReceiveStream(mRecvStreamConfig.Copy());
+ // Ensure that we set the jitter buffer target on this stream.
+ mRecvStream->SetBaseMinimumPlayoutDelayMs(mJitterBufferTargetMs);
+
+ CSFLogDebug(LOGTAG, "Created VideoReceiveStream %p for SSRC %u (0x%x)",
+ mRecvStream, mRecvStreamConfig.rtp.remote_ssrc,
+ mRecvStreamConfig.rtp.remote_ssrc);
+}
+
+void WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ CSFLogDebug(LOGTAG, "%s (%p): Unsetting SSRC %u in other conduits",
+ __FUNCTION__, this, mRecvStreamConfig.rtp.remote_ssrc);
+ mCall->UnregisterConduit(this);
+ mCall->UnsetRemoteSSRC(mRecvStreamConfig.rtp.remote_ssrc);
+ mCall->RegisterConduit(this);
+}
+
+void WebrtcVideoConduit::SetRemoteSSRCConfig(uint32_t aSsrc,
+ uint32_t aRtxSsrc) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ CSFLogDebug(LOGTAG, "%s: SSRC %u (0x%x)", __FUNCTION__, aSsrc, aSsrc);
+
+ if (mRecvStreamConfig.rtp.remote_ssrc != aSsrc) {
+ nsCOMPtr<nsIDirectTaskDispatcher> dtd = do_QueryInterface(mCallThread);
+ MOZ_ALWAYS_SUCCEEDS(dtd->DispatchDirectTask(NewRunnableMethod(
+ "WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC", this,
+ &WebrtcVideoConduit::NotifyUnsetCurrentRemoteSSRC)));
+ }
+
+ mRecvSSRC = mRecvStreamConfig.rtp.remote_ssrc = aSsrc;
+ mRecvStreamConfig.rtp.rtx_ssrc = aRtxSsrc;
+}
+
+void WebrtcVideoConduit::SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc,
+ uint32_t aRtxSsrc) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ if (mRecvStreamConfig.rtp.remote_ssrc == aSsrc &&
+ mRecvStreamConfig.rtp.rtx_ssrc == aRtxSsrc) {
+ return;
+ }
+
+ SetRemoteSSRCConfig(aSsrc, aRtxSsrc);
+
+ const bool wasReceiving = mEngineReceiving;
+ const bool hadRecvStream = mRecvStream;
+
+ StopReceiving();
+
+ if (hadRecvStream) {
+ MutexAutoLock lock(mMutex);
+ DeleteRecvStream();
+ CreateRecvStream();
+ }
+
+ if (wasReceiving) {
+ StartReceiving();
+ }
+}
+
+void WebrtcVideoConduit::EnsureRemoteSSRC() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ const auto& ssrcs = mSendStreamConfig.rtp.ssrcs;
+ if (mRecvStreamConfig.rtp.remote_ssrc != 0 &&
+ std::find(ssrcs.begin(), ssrcs.end(),
+ mRecvStreamConfig.rtp.remote_ssrc) == ssrcs.end()) {
+ return;
+ }
+
+ uint32_t ssrc;
+ do {
+ ssrc = GenerateRandomSSRC();
+ } while (
+ NS_WARN_IF(std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()));
+ CSFLogDebug(LOGTAG, "VideoConduit %p: Generated remote SSRC %u", this, ssrc);
+ SetRemoteSSRCConfig(ssrc, 0);
+}
+
+void WebrtcVideoConduit::EnsureLocalSSRC() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertCurrentThreadOwns();
+
+ auto& ssrcs = mSendStreamConfig.rtp.ssrcs;
+ if (ssrcs.empty()) {
+ ssrcs.push_back(GenerateRandomSSRC());
+ }
+
+ // Reverse-iterating here so that the first dupe in `ssrcs` always wins.
+ for (auto& ssrc : Reversed(ssrcs)) {
+ if (ssrc != 0 && ssrc != mRecvStreamConfig.rtp.remote_ssrc &&
+ std::count(ssrcs.begin(), ssrcs.end(), ssrc) == 1) {
+ continue;
+ }
+ do {
+ ssrc = GenerateRandomSSRC();
+ } while (NS_WARN_IF(ssrc == mRecvStreamConfig.rtp.remote_ssrc) ||
+ NS_WARN_IF(std::count(ssrcs.begin(), ssrcs.end(), ssrc) > 1));
+ CSFLogDebug(LOGTAG, "%s (%p): Generated local SSRC %u", __FUNCTION__, this,
+ ssrc);
+ }
+ mRecvStreamConfig.rtp.local_ssrc = ssrcs[0];
+}
+
+void WebrtcVideoConduit::UnsetRemoteSSRC(uint32_t aSsrc) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertNotCurrentThreadOwns();
+
+ if (mRecvStreamConfig.rtp.remote_ssrc != aSsrc &&
+ mRecvStreamConfig.rtp.rtx_ssrc != aSsrc) {
+ return;
+ }
+
+ const auto& ssrcs = mSendStreamConfig.rtp.ssrcs;
+ uint32_t our_ssrc = 0;
+ do {
+ our_ssrc = GenerateRandomSSRC();
+ } while (NS_WARN_IF(our_ssrc == aSsrc) ||
+ NS_WARN_IF(std::find(ssrcs.begin(), ssrcs.end(), our_ssrc) !=
+ ssrcs.end()));
+
+ CSFLogDebug(LOGTAG, "%s (%p): Generated remote SSRC %u", __FUNCTION__, this,
+ our_ssrc);
+
+ // There is a (tiny) chance that this new random ssrc will collide with some
+ // other conduit's remote ssrc, in which case that conduit will choose a new
+ // one.
+ SetRemoteSSRCAndRestartAsNeeded(our_ssrc, 0);
+}
+
+/*static*/
+unsigned WebrtcVideoConduit::ToLibwebrtcMaxFramerate(
+ const Maybe<double>& aMaxFramerate) {
+ Maybe<unsigned> negotiatedMaxFps;
+ if (aMaxFramerate.isSome()) {
+ // libwebrtc does not handle non-integer max framerate.
+ unsigned integerMaxFps = static_cast<unsigned>(std::round(*aMaxFramerate));
+ // libwebrtc crashes with a max framerate of 0, even though the
+ // spec says this is valid. For now, we treat this as no limit.
+ if (integerMaxFps) {
+ negotiatedMaxFps = Some(integerMaxFps);
+ }
+ }
+ // We do not use DEFAULT_VIDEO_MAX_FRAMERATE here; that is used at the very
+ // end in VideoStreamFactory, once codec-wide and per-encoding limits are
+ // known.
+ return negotiatedMaxFps.refOr(std::numeric_limits<unsigned int>::max());
+}
+
+Maybe<Ssrc> WebrtcVideoConduit::GetRemoteSSRC() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ // libwebrtc uses 0 to mean a lack of SSRC. That is not to spec.
+ return mRecvStreamConfig.rtp.remote_ssrc == 0
+ ? Nothing()
+ : Some(mRecvStreamConfig.rtp.remote_ssrc);
+}
+
+Maybe<webrtc::VideoReceiveStreamInterface::Stats>
+WebrtcVideoConduit::GetReceiverStats() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ if (!mRecvStream) {
+ return Nothing();
+ }
+ return Some(mRecvStream->GetStats());
+}
+
+Maybe<webrtc::VideoSendStream::Stats> WebrtcVideoConduit::GetSenderStats()
+ const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ if (!mSendStream) {
+ return Nothing();
+ }
+ return Some(mSendStream->GetStats());
+}
+
+Maybe<webrtc::Call::Stats> WebrtcVideoConduit::GetCallStats() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ if (!mCall->Call()) {
+ return Nothing();
+ }
+ return Some(mCall->Call()->GetStats());
+}
+
+MediaConduitErrorCode WebrtcVideoConduit::Init() {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ CSFLogDebug(LOGTAG, "%s this=%p", __FUNCTION__, this);
+
+#ifdef MOZ_WIDGET_ANDROID
+ if (mozilla::camera::VideoEngine::SetAndroidObjects() != 0) {
+ CSFLogError(LOGTAG, "%s: could not set Android objects", __FUNCTION__);
+ return kMediaConduitSessionNotInited;
+ }
+#endif // MOZ_WIDGET_ANDROID
+
+ mSendPluginCreated = mEncoderFactory->CreatedGmpPluginEvent().Connect(
+ GetMainThreadSerialEventTarget(),
+ [self = detail::RawPtr(this)](uint64_t aPluginID) {
+ self.get()->mSendCodecPluginIDs.AppendElement(aPluginID);
+ });
+ mSendPluginReleased = mEncoderFactory->ReleasedGmpPluginEvent().Connect(
+ GetMainThreadSerialEventTarget(),
+ [self = detail::RawPtr(this)](uint64_t aPluginID) {
+ self.get()->mSendCodecPluginIDs.RemoveElement(aPluginID);
+ });
+ mRecvPluginCreated = mDecoderFactory->CreatedGmpPluginEvent().Connect(
+ GetMainThreadSerialEventTarget(),
+ [self = detail::RawPtr(this)](uint64_t aPluginID) {
+ self.get()->mRecvCodecPluginIDs.AppendElement(aPluginID);
+ });
+ mRecvPluginReleased = mDecoderFactory->ReleasedGmpPluginEvent().Connect(
+ GetMainThreadSerialEventTarget(),
+ [self = detail::RawPtr(this)](uint64_t aPluginID) {
+ self.get()->mRecvCodecPluginIDs.RemoveElement(aPluginID);
+ });
+
+ MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction(
+ __func__, [this, self = RefPtr<WebrtcVideoConduit>(this)] {
+ mCall->RegisterConduit(this);
+ })));
+
+ CSFLogDebug(LOGTAG, "%s Initialization Done", __FUNCTION__);
+ return kMediaConduitNoError;
+}
+
+RefPtr<GenericPromise> WebrtcVideoConduit::Shutdown() {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ mSendPluginCreated.DisconnectIfExists();
+ mSendPluginReleased.DisconnectIfExists();
+ mRecvPluginCreated.DisconnectIfExists();
+ mRecvPluginReleased.DisconnectIfExists();
+
+ return InvokeAsync(
+ mCallThread, __func__, [this, self = RefPtr<WebrtcVideoConduit>(this)] {
+ using namespace Telemetry;
+ if (mSendBitrate.NumDataValues() > 0) {
+ Accumulate(WEBRTC_VIDEO_ENCODER_BITRATE_AVG_PER_CALL_KBPS,
+ static_cast<unsigned>(mSendBitrate.Mean() / 1000));
+ Accumulate(
+ WEBRTC_VIDEO_ENCODER_BITRATE_STD_DEV_PER_CALL_KBPS,
+ static_cast<unsigned>(mSendBitrate.StandardDeviation() / 1000));
+ mSendBitrate.Clear();
+ }
+ if (mSendFramerate.NumDataValues() > 0) {
+ Accumulate(WEBRTC_VIDEO_ENCODER_FRAMERATE_AVG_PER_CALL,
+ static_cast<unsigned>(mSendFramerate.Mean()));
+ Accumulate(
+ WEBRTC_VIDEO_ENCODER_FRAMERATE_10X_STD_DEV_PER_CALL,
+ static_cast<unsigned>(mSendFramerate.StandardDeviation() * 10));
+ mSendFramerate.Clear();
+ }
+
+ if (mRecvBitrate.NumDataValues() > 0) {
+ Accumulate(WEBRTC_VIDEO_DECODER_BITRATE_AVG_PER_CALL_KBPS,
+ static_cast<unsigned>(mRecvBitrate.Mean() / 1000));
+ Accumulate(
+ WEBRTC_VIDEO_DECODER_BITRATE_STD_DEV_PER_CALL_KBPS,
+ static_cast<unsigned>(mRecvBitrate.StandardDeviation() / 1000));
+ mRecvBitrate.Clear();
+ }
+ if (mRecvFramerate.NumDataValues() > 0) {
+ Accumulate(WEBRTC_VIDEO_DECODER_FRAMERATE_AVG_PER_CALL,
+ static_cast<unsigned>(mRecvFramerate.Mean()));
+ Accumulate(
+ WEBRTC_VIDEO_DECODER_FRAMERATE_10X_STD_DEV_PER_CALL,
+ static_cast<unsigned>(mRecvFramerate.StandardDeviation() * 10));
+ mRecvFramerate.Clear();
+ }
+
+ mControl.mReceiving.DisconnectIfConnected();
+ mControl.mTransmitting.DisconnectIfConnected();
+ mControl.mLocalSsrcs.DisconnectIfConnected();
+ mControl.mLocalRtxSsrcs.DisconnectIfConnected();
+ mControl.mLocalCname.DisconnectIfConnected();
+ mControl.mMid.DisconnectIfConnected();
+ mControl.mRemoteSsrc.DisconnectIfConnected();
+ mControl.mRemoteRtxSsrc.DisconnectIfConnected();
+ mControl.mSyncGroup.DisconnectIfConnected();
+ mControl.mLocalRecvRtpExtensions.DisconnectIfConnected();
+ mControl.mLocalSendRtpExtensions.DisconnectIfConnected();
+ mControl.mSendCodec.DisconnectIfConnected();
+ mControl.mSendRtpRtcpConfig.DisconnectIfConnected();
+ mControl.mRecvCodecs.DisconnectIfConnected();
+ mControl.mRecvRtpRtcpConfig.DisconnectIfConnected();
+ mControl.mCodecMode.DisconnectIfConnected();
+ mWatchManager.Shutdown();
+
+ mCall->UnregisterConduit(this);
+ mDecoderFactory->DisconnectAll();
+ mEncoderFactory->DisconnectAll();
+ {
+ MutexAutoLock lock(mMutex);
+ DeleteSendStream();
+ DeleteRecvStream();
+ }
+
+ return GenericPromise::CreateAndResolve(true, __func__);
+ });
+}
+
+webrtc::VideoCodecMode WebrtcVideoConduit::CodecMode() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ return mControl.mCodecMode;
+}
+
+MediaConduitErrorCode WebrtcVideoConduit::AttachRenderer(
+ RefPtr<mozilla::VideoRenderer> aVideoRenderer) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ CSFLogDebug(LOGTAG, "%s", __FUNCTION__);
+
+ // null renderer
+ if (!aVideoRenderer) {
+ CSFLogError(LOGTAG, "%s NULL Renderer", __FUNCTION__);
+ MOZ_ASSERT(false);
+ return kMediaConduitInvalidRenderer;
+ }
+
+ // This function is called only from main, so we only need to protect against
+ // modifying mRenderer while any webrtc.org code is trying to use it.
+ {
+ ReentrantMonitorAutoEnter enter(mRendererMonitor);
+ mRenderer = aVideoRenderer;
+ // Make sure the renderer knows the resolution
+ mRenderer->FrameSizeChange(mReceivingWidth, mReceivingHeight);
+ }
+
+ return kMediaConduitNoError;
+}
+
+void WebrtcVideoConduit::DetachRenderer() {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ ReentrantMonitorAutoEnter enter(mRendererMonitor);
+ if (mRenderer) {
+ mRenderer = nullptr;
+ }
+}
+
+rtc::RefCountedObject<mozilla::VideoStreamFactory>*
+WebrtcVideoConduit::CreateVideoStreamFactory() {
+ auto videoStreamFactory = mVideoStreamFactory.Lock();
+ *videoStreamFactory = new rtc::RefCountedObject<VideoStreamFactory>(
+ *mCurSendCodecConfig, mControl.mCodecMode, mMinBitrate, mStartBitrate,
+ mPrefMaxBitrate, mNegotiatedMaxBitrate, mVideoBroadcaster.wants(),
+ mLockScaling);
+ return videoStreamFactory->get();
+}
+
+void WebrtcVideoConduit::AddOrUpdateSink(
+ rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
+ const rtc::VideoSinkWants& wants) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ if (!mRegisteredSinks.Contains(sink)) {
+ mRegisteredSinks.AppendElement(sink);
+ }
+ auto oldWants = mVideoBroadcaster.wants();
+ mVideoBroadcaster.AddOrUpdateSink(sink, wants);
+ if (oldWants != mVideoBroadcaster.wants()) {
+ mEncoderConfig.video_stream_factory = CreateVideoStreamFactory();
+ mSendStream->ReconfigureVideoEncoder(mEncoderConfig.Copy());
+ }
+}
+
+void WebrtcVideoConduit::RemoveSink(
+ rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ mRegisteredSinks.RemoveElement(sink);
+ auto oldWants = mVideoBroadcaster.wants();
+ mVideoBroadcaster.RemoveSink(sink);
+ if (oldWants != mVideoBroadcaster.wants()) {
+ mEncoderConfig.video_stream_factory = CreateVideoStreamFactory();
+ mSendStream->ReconfigureVideoEncoder(mEncoderConfig.Copy());
+ }
+}
+
+MediaConduitErrorCode WebrtcVideoConduit::SendVideoFrame(
+ webrtc::VideoFrame aFrame) {
+ // XXX Google uses a "timestamp_aligner" to translate timestamps from the
+ // camera via TranslateTimestamp(); we should look at doing the same. This
+ // avoids sampling error when capturing frames, but google had to deal with
+ // some broken cameras, include Logitech c920's IIRC.
+
+ {
+ MutexAutoLock lock(mMutex);
+ if (mSendStreamConfig.rtp.ssrcs.empty()) {
+ CSFLogVerbose(LOGTAG, "WebrtcVideoConduit %p %s No SSRC set", this,
+ __FUNCTION__);
+ return kMediaConduitNoError;
+ }
+ if (!mCurSendCodecConfig) {
+ CSFLogVerbose(LOGTAG, "WebrtcVideoConduit %p %s No send codec set", this,
+ __FUNCTION__);
+ return kMediaConduitNoError;
+ }
+
+ // Workaround for bug in libwebrtc where all encodings are transmitted
+ // if they are all inactive.
+ bool anyActive = false;
+ for (const auto& encoding : mCurSendCodecConfig->mEncodings) {
+ if (encoding.active) {
+ anyActive = true;
+ break;
+ }
+ }
+ if (!anyActive) {
+ CSFLogVerbose(LOGTAG, "WebrtcVideoConduit %p %s No active encodings",
+ this, __FUNCTION__);
+ return kMediaConduitNoError;
+ }
+
+ CSFLogVerbose(LOGTAG, "WebrtcVideoConduit %p %s (send SSRC %u (0x%x))",
+ this, __FUNCTION__, mSendStreamConfig.rtp.ssrcs.front(),
+ mSendStreamConfig.rtp.ssrcs.front());
+
+ if (aFrame.width() != mLastWidth || aFrame.height() != mLastHeight) {
+ // See if we need to recalculate what we're sending.
+ CSFLogVerbose(LOGTAG, "%s: call SelectSendResolution with %ux%u",
+ __FUNCTION__, aFrame.width(), aFrame.height());
+ MOZ_ASSERT(aFrame.width() != 0 && aFrame.height() != 0);
+ // Note coverity will flag this since it thinks they can be 0
+ MOZ_ASSERT(mCurSendCodecConfig);
+
+ mLastWidth = aFrame.width();
+ mLastHeight = aFrame.height();
+ }
+
+ // adapt input video to wants of sink
+ if (!mVideoBroadcaster.frame_wanted()) {
+ return kMediaConduitNoError;
+ }
+
+ // Check if we need to drop this frame to meet a requested FPS
+ auto videoStreamFactory = mVideoStreamFactory.Lock();
+ auto& videoStreamFactoryRef = videoStreamFactory.ref();
+ if (videoStreamFactoryRef->ShouldDropFrame(aFrame)) {
+ return kMediaConduitNoError;
+ }
+ }
+
+ // If we have zero width or height, drop the frame here. Attempting to send
+ // it will cause all sorts of problems in the webrtc.org code.
+ if (aFrame.width() == 0 || aFrame.height() == 0) {
+ return kMediaConduitNoError;
+ }
+
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
+ aFrame.video_frame_buffer();
+
+ MOZ_ASSERT(!aFrame.color_space(), "Unexpected use of color space");
+ MOZ_ASSERT(!aFrame.has_update_rect(), "Unexpected use of update rect");
+
+#ifdef MOZ_REAL_TIME_TRACING
+ if (profiler_is_active()) {
+ MutexAutoLock lock(mMutex);
+ nsAutoCStringN<256> ssrcsCommaSeparated;
+ bool first = true;
+ for (auto ssrc : mSendStreamConfig.rtp.ssrcs) {
+ if (!first) {
+ ssrcsCommaSeparated.AppendASCII(", ");
+ } else {
+ first = false;
+ }
+ ssrcsCommaSeparated.AppendInt(ssrc);
+ }
+ // The first frame has a delta of zero.
+ uint64_t timestampDelta =
+ mLastTimestampSendUs.isSome()
+ ? aFrame.timestamp_us() - mLastTimestampSendUs.value()
+ : 0;
+ mLastTimestampSendUs = Some(aFrame.timestamp_us());
+ TRACE_COMMENT("VideoConduit::SendVideoFrame", "t-delta=%.1fms, ssrcs=%s",
+ timestampDelta / 1000.f, ssrcsCommaSeparated.get());
+ }
+#endif
+
+ mVideoBroadcaster.OnFrame(aFrame);
+
+ return kMediaConduitNoError;
+}
+
+// Transport Layer Callbacks
+
+void WebrtcVideoConduit::DeliverPacket(rtc::CopyOnWriteBuffer packet,
+ PacketType type) {
+ // Currently unused.
+ MOZ_ASSERT(false);
+}
+
+void WebrtcVideoConduit::OnRtpReceived(webrtc::RtpPacketReceived&& aPacket,
+ webrtc::RTPHeader&& aHeader) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ mRemoteSendSSRC = aHeader.ssrc;
+
+ if (mAllowSsrcChange || mRecvStreamConfig.rtp.remote_ssrc == 0) {
+ bool switchRequired = mRecvStreamConfig.rtp.remote_ssrc != aHeader.ssrc;
+ if (switchRequired) {
+ // Handle the unknown ssrc (and ssrc-not-signaled case).
+
+ // We need to check that the newly received ssrc is not already
+ // associated with ulpfec or rtx. This is how webrtc.org handles
+ // things, see https://codereview.webrtc.org/1226093002.
+ const webrtc::VideoReceiveStreamInterface::Config::Rtp& rtp =
+ mRecvStreamConfig.rtp;
+ switchRequired =
+ rtp.rtx_associated_payload_types.find(aHeader.payloadType) ==
+ rtp.rtx_associated_payload_types.end() &&
+ rtp.ulpfec_payload_type != aHeader.payloadType;
+ }
+
+ if (switchRequired) {
+ CSFLogInfo(LOGTAG, "VideoConduit %p: Switching remote SSRC from %u to %u",
+ this, mRecvStreamConfig.rtp.remote_ssrc, aHeader.ssrc);
+ SetRemoteSSRCAndRestartAsNeeded(aHeader.ssrc, 0);
+ }
+ }
+
+ CSFLogVerbose(LOGTAG, "%s: seq# %u, Len %zu, SSRC %u (0x%x) ", __FUNCTION__,
+ aPacket.SequenceNumber(), aPacket.size(), aPacket.Ssrc(),
+ aPacket.Ssrc());
+
+ mRtpPacketEvent.Notify();
+ if (mCall->Call()) {
+ mCall->Call()->Receiver()->DeliverRtpPacket(
+ webrtc::MediaType::VIDEO, std::move(aPacket),
+ [self = RefPtr<WebrtcVideoConduit>(this)](
+ const webrtc::RtpPacketReceived& packet) {
+ CSFLogVerbose(
+ LOGTAG,
+ "VideoConduit %p: failed demuxing packet, ssrc: %u seq: %u",
+ self.get(), packet.Ssrc(), packet.SequenceNumber());
+ return false;
+ });
+ }
+}
+
+void WebrtcVideoConduit::OnRtcpReceived(MediaPacket&& aPacket) {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ CSFLogVerbose(LOGTAG, "VideoConduit %p: Received RTCP Packet, len %zu ", this,
+ aPacket.len());
+
+ if (mCall->Call()) {
+ mCall->Call()->Receiver()->DeliverRtcpPacket(
+ rtc::CopyOnWriteBuffer(aPacket.data(), aPacket.len()));
+ }
+}
+
+Maybe<uint16_t> WebrtcVideoConduit::RtpSendBaseSeqFor(uint32_t aSsrc) const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ auto it = mRtpSendBaseSeqs.find(aSsrc);
+ if (it == mRtpSendBaseSeqs.end()) {
+ return Nothing();
+ }
+ return Some(it->second);
+}
+
+const dom::RTCStatsTimestampMaker& WebrtcVideoConduit::GetTimestampMaker()
+ const {
+ return mCall->GetTimestampMaker();
+}
+
+void WebrtcVideoConduit::StopTransmitting() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertNotCurrentThreadOwns();
+
+ if (!mEngineTransmitting) {
+ return;
+ }
+
+ if (mSendStream) {
+ CSFLogDebug(LOGTAG, "%s Stopping send stream", __FUNCTION__);
+ mSendStream->Stop();
+ }
+
+ mEngineTransmitting = false;
+}
+
+void WebrtcVideoConduit::StartTransmitting() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ MOZ_ASSERT(mSendStream);
+ mMutex.AssertNotCurrentThreadOwns();
+
+ if (mEngineTransmitting) {
+ return;
+ }
+
+ CSFLogDebug(LOGTAG, "%s Starting send stream", __FUNCTION__);
+
+ mSendStream->Start();
+ // XXX File a bug to consider hooking this up to the state of mtransport
+ mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::VIDEO,
+ webrtc::kNetworkUp);
+ mEngineTransmitting = true;
+}
+
+void WebrtcVideoConduit::StopReceiving() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ mMutex.AssertNotCurrentThreadOwns();
+
+ // Are we receiving already? If so, stop receiving and playout
+ // since we can't apply new recv codec when the engine is playing.
+ if (!mEngineReceiving) {
+ return;
+ }
+
+ if (mRecvStream) {
+ CSFLogDebug(LOGTAG, "%s Stopping receive stream", __FUNCTION__);
+ mRecvStream->Stop();
+ }
+
+ mEngineReceiving = false;
+}
+
+void WebrtcVideoConduit::StartReceiving() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+ MOZ_ASSERT(mRecvStream);
+ mMutex.AssertNotCurrentThreadOwns();
+
+ if (mEngineReceiving) {
+ return;
+ }
+
+ CSFLogDebug(LOGTAG, "%s Starting receive stream (SSRC %u (0x%x))",
+ __FUNCTION__, mRecvStreamConfig.rtp.remote_ssrc,
+ mRecvStreamConfig.rtp.remote_ssrc);
+ // Start Receiving on the video engine
+ mRecvStream->Start();
+
+ // XXX File a bug to consider hooking this up to the state of mtransport
+ mCall->Call()->SignalChannelNetworkState(webrtc::MediaType::VIDEO,
+ webrtc::kNetworkUp);
+ mEngineReceiving = true;
+}
+
+bool WebrtcVideoConduit::SendRtp(const uint8_t* aData, size_t aLength,
+ const webrtc::PacketOptions& aOptions) {
+ MOZ_ASSERT(aLength >= 12);
+ const uint16_t seqno = ntohs(*((uint16_t*)&aData[2]));
+ const uint32_t ssrc = ntohl(*((uint32_t*)&aData[8]));
+
+ CSFLogVerbose(
+ LOGTAG,
+ "VideoConduit %p: Sending RTP Packet seq# %u, len %zu, SSRC %u (0x%x)",
+ this, seqno, aLength, ssrc, ssrc);
+
+ if (!mTransportActive) {
+ CSFLogError(LOGTAG, "VideoConduit %p: RTP Packet Send Failed", this);
+ return false;
+ }
+
+ MediaPacket packet;
+ packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION);
+ packet.SetType(MediaPacket::RTP);
+ mSenderRtpSendEvent.Notify(std::move(packet));
+
+ // Parse the sequence number of the first rtp packet as base_seq.
+ const auto inserted = mRtpSendBaseSeqs_n.insert({ssrc, seqno}).second;
+
+ if (inserted || aOptions.packet_id >= 0) {
+ int64_t now_ms = PR_Now() / 1000;
+ MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction(
+ __func__, [this, self = RefPtr<WebrtcVideoConduit>(this),
+ packet_id = aOptions.packet_id, now_ms, ssrc, seqno] {
+ mRtpSendBaseSeqs.insert({ssrc, seqno});
+ if (packet_id >= 0) {
+ if (mCall->Call()) {
+ // TODO: This notification should ideally happen after the
+ // transport layer has sent the packet on the wire.
+ mCall->Call()->OnSentPacket({packet_id, now_ms});
+ }
+ }
+ })));
+ }
+ return true;
+}
+
+bool WebrtcVideoConduit::SendSenderRtcp(const uint8_t* aData, size_t aLength) {
+ CSFLogVerbose(
+ LOGTAG,
+ "VideoConduit %p: Sending RTCP SR Packet, len %zu, SSRC %u (0x%x)", this,
+ aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])),
+ (uint32_t)ntohl(*((uint32_t*)&aData[4])));
+
+ if (!mTransportActive) {
+ CSFLogError(LOGTAG, "VideoConduit %p: RTCP SR Packet Send Failed", this);
+ return false;
+ }
+
+ MediaPacket packet;
+ packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION);
+ packet.SetType(MediaPacket::RTCP);
+ mSenderRtcpSendEvent.Notify(std::move(packet));
+ return true;
+}
+
+bool WebrtcVideoConduit::SendReceiverRtcp(const uint8_t* aData,
+ size_t aLength) {
+ CSFLogVerbose(
+ LOGTAG,
+ "VideoConduit %p: Sending RTCP RR Packet, len %zu, SSRC %u (0x%x)", this,
+ aLength, (uint32_t)ntohl(*((uint32_t*)&aData[4])),
+ (uint32_t)ntohl(*((uint32_t*)&aData[4])));
+
+ if (!mTransportActive) {
+ CSFLogError(LOGTAG, "VideoConduit %p: RTCP RR Packet Send Failed", this);
+ return false;
+ }
+
+ MediaPacket packet;
+ packet.Copy(aData, aLength, aLength + SRTP_MAX_EXPANSION);
+ packet.SetType(MediaPacket::RTCP);
+ mReceiverRtcpSendEvent.Notify(std::move(packet));
+ return true;
+}
+
+void WebrtcVideoConduit::OnFrame(const webrtc::VideoFrame& video_frame) {
+ const uint32_t localRecvSsrc = mRecvSSRC;
+ const uint32_t remoteSendSsrc = mRemoteSendSSRC;
+
+ CSFLogVerbose(
+ LOGTAG,
+ "VideoConduit %p: Rendering frame, Remote SSRC %u (0x%x), size %ux%u",
+ this, static_cast<uint32_t>(remoteSendSsrc),
+ static_cast<uint32_t>(remoteSendSsrc), video_frame.width(),
+ video_frame.height());
+ ReentrantMonitorAutoEnter enter(mRendererMonitor);
+
+ if (!mRenderer) {
+ CSFLogError(LOGTAG, "VideoConduit %p: Cannot render frame, no renderer",
+ this);
+ return;
+ }
+
+ bool needsNewHistoryElement = mReceivedFrameHistory.mEntries.IsEmpty();
+
+ if (mReceivingWidth != video_frame.width() ||
+ mReceivingHeight != video_frame.height()) {
+ mReceivingWidth = video_frame.width();
+ mReceivingHeight = video_frame.height();
+ mRenderer->FrameSizeChange(mReceivingWidth, mReceivingHeight);
+ needsNewHistoryElement = true;
+ }
+
+ if (!needsNewHistoryElement) {
+ auto& currentEntry = mReceivedFrameHistory.mEntries.LastElement();
+ needsNewHistoryElement =
+ currentEntry.mRotationAngle !=
+ static_cast<unsigned long>(video_frame.rotation()) ||
+ currentEntry.mLocalSsrc != localRecvSsrc ||
+ currentEntry.mRemoteSsrc != remoteSendSsrc;
+ }
+
+ // Record frame history
+ const auto historyNow = mCall->GetTimestampMaker().GetNow().ToDom();
+ if (needsNewHistoryElement) {
+ dom::RTCVideoFrameHistoryEntryInternal frameHistoryElement;
+ frameHistoryElement.mConsecutiveFrames = 0;
+ frameHistoryElement.mWidth = video_frame.width();
+ frameHistoryElement.mHeight = video_frame.height();
+ frameHistoryElement.mRotationAngle =
+ static_cast<unsigned long>(video_frame.rotation());
+ frameHistoryElement.mFirstFrameTimestamp = historyNow;
+ frameHistoryElement.mLocalSsrc = localRecvSsrc;
+ frameHistoryElement.mRemoteSsrc = remoteSendSsrc;
+ if (!mReceivedFrameHistory.mEntries.AppendElement(frameHistoryElement,
+ fallible)) {
+ mozalloc_handle_oom(0);
+ }
+ }
+ auto& currentEntry = mReceivedFrameHistory.mEntries.LastElement();
+
+ currentEntry.mConsecutiveFrames++;
+ currentEntry.mLastFrameTimestamp = historyNow;
+ // Attempt to retrieve an timestamp encoded in the image pixels if enabled.
+ if (mVideoLatencyTestEnable && mReceivingWidth && mReceivingHeight) {
+ uint64_t now = PR_Now();
+ uint64_t timestamp = 0;
+ uint8_t* data = const_cast<uint8_t*>(
+ video_frame.video_frame_buffer()->GetI420()->DataY());
+ bool ok = YuvStamper::Decode(
+ mReceivingWidth, mReceivingHeight, mReceivingWidth, data,
+ reinterpret_cast<unsigned char*>(&timestamp), sizeof(timestamp), 0, 0);
+ if (ok) {
+ VideoLatencyUpdate(now - timestamp);
+ }
+ }
+#ifdef MOZ_REAL_TIME_TRACING
+ if (profiler_is_active()) {
+ MutexAutoLock lock(mMutex);
+ // The first frame has a delta of zero.
+ uint32_t rtpTimestamp = video_frame.timestamp();
+ uint32_t timestampDelta =
+ mLastRTPTimestampReceive.isSome()
+ ? rtpTimestamp - mLastRTPTimestampReceive.value()
+ : 0;
+ mLastRTPTimestampReceive = Some(rtpTimestamp);
+ TRACE_COMMENT("VideoConduit::OnFrame", "t-delta=%.1fms, ssrc=%u",
+ timestampDelta * 1000.f / webrtc::kVideoPayloadTypeFrequency,
+ localRecvSsrc);
+ }
+#endif
+
+ mRenderer->RenderVideoFrame(*video_frame.video_frame_buffer(),
+ video_frame.timestamp(),
+ video_frame.render_time_ms());
+}
+
+bool WebrtcVideoConduit::AddFrameHistory(
+ dom::Sequence<dom::RTCVideoFrameHistoryInternal>* outHistories) const {
+ ReentrantMonitorAutoEnter enter(mRendererMonitor);
+ if (!outHistories->AppendElement(mReceivedFrameHistory, fallible)) {
+ mozalloc_handle_oom(0);
+ return false;
+ }
+ return true;
+}
+
+void WebrtcVideoConduit::SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) {
+ MOZ_RELEASE_ASSERT(aTargetMs <= std::numeric_limits<uint16_t>::max());
+ MOZ_RELEASE_ASSERT(aTargetMs >= 0);
+
+ MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction(
+ __func__,
+ [this, self = RefPtr<WebrtcVideoConduit>(this), targetMs = aTargetMs] {
+ mJitterBufferTargetMs = static_cast<uint16_t>(targetMs);
+ if (mRecvStream) {
+ mRecvStream->SetBaseMinimumPlayoutDelayMs(targetMs);
+ }
+ })));
+}
+
+void WebrtcVideoConduit::DumpCodecDB() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ for (const auto& entry : mControl.mConfiguredRecvCodecs) {
+ CSFLogDebug(LOGTAG, "Payload Name: %s", entry.mName.c_str());
+ CSFLogDebug(LOGTAG, "Payload Type: %d", entry.mType);
+ CSFLogDebug(LOGTAG, "Payload Max Frame Size: %d",
+ entry.mEncodingConstraints.maxFs);
+ if (entry.mEncodingConstraints.maxFps.isSome()) {
+ CSFLogDebug(LOGTAG, "Payload Max Frame Rate: %f",
+ *entry.mEncodingConstraints.maxFps);
+ }
+ }
+}
+
+void WebrtcVideoConduit::VideoLatencyUpdate(uint64_t aNewSample) {
+ mRendererMonitor.AssertCurrentThreadIn();
+
+ mVideoLatencyAvg =
+ (sRoundingPadding * aNewSample + sAlphaNum * mVideoLatencyAvg) /
+ sAlphaDen;
+}
+
+uint64_t WebrtcVideoConduit::MozVideoLatencyAvg() {
+ mRendererMonitor.AssertCurrentThreadIn();
+
+ return mVideoLatencyAvg / sRoundingPadding;
+}
+
+void WebrtcVideoConduit::CollectTelemetryData() {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ if (mEngineTransmitting) {
+ webrtc::VideoSendStream::Stats stats = mSendStream->GetStats();
+ mSendBitrate.Push(stats.media_bitrate_bps);
+ mSendFramerate.Push(stats.encode_frame_rate);
+ }
+ if (mEngineReceiving) {
+ webrtc::VideoReceiveStreamInterface::Stats stats = mRecvStream->GetStats();
+ mRecvBitrate.Push(stats.total_bitrate_bps);
+ mRecvFramerate.Push(stats.decode_frame_rate);
+ }
+}
+
+void WebrtcVideoConduit::OnRtcpBye() { mRtcpByeEvent.Notify(); }
+
+void WebrtcVideoConduit::OnRtcpTimeout() { mRtcpTimeoutEvent.Notify(); }
+
+void WebrtcVideoConduit::SetTransportActive(bool aActive) {
+ MOZ_ASSERT(mStsThread->IsOnCurrentThread());
+ if (mTransportActive == aActive) {
+ return;
+ }
+
+ // If false, This stops us from sending
+ mTransportActive = aActive;
+
+ // We queue this because there might be notifications to these listeners
+ // pending, and we don't want to drop them by letting this jump ahead of
+ // those notifications. We move the listeners into the lambda in case the
+ // transport comes back up before we disconnect them. (The Connect calls
+ // happen in MediaPipeline)
+ // We retain a strong reference to ourself, because the listeners are holding
+ // a non-refcounted reference to us, and moving them into the lambda could
+ // conceivably allow them to outlive us.
+ if (!aActive) {
+ MOZ_ALWAYS_SUCCEEDS(mCallThread->Dispatch(NS_NewRunnableFunction(
+ __func__,
+ [self = RefPtr<WebrtcVideoConduit>(this),
+ recvRtpListener = std::move(mReceiverRtpEventListener),
+ recvRtcpListener = std::move(mReceiverRtcpEventListener),
+ sendRtcpListener = std::move(mSenderRtcpEventListener)]() mutable {
+ recvRtpListener.DisconnectIfExists();
+ recvRtcpListener.DisconnectIfExists();
+ sendRtcpListener.DisconnectIfExists();
+ })));
+ }
+}
+
+std::vector<webrtc::RtpSource> WebrtcVideoConduit::GetUpstreamRtpSources()
+ const {
+ MOZ_ASSERT(NS_IsMainThread());
+ std::vector<webrtc::RtpSource> sources;
+ {
+ MutexAutoLock lock(mMutex);
+ if (mRecvStream) {
+ sources = mRecvStream->GetSources();
+ }
+ }
+ return sources;
+}
+
+bool WebrtcVideoConduit::HasCodecPluginID(uint64_t aPluginID) const {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ return mSendCodecPluginIDs.Contains(aPluginID) ||
+ mRecvCodecPluginIDs.Contains(aPluginID);
+}
+
+bool WebrtcVideoConduit::HasH264Hardware() {
+ nsCOMPtr<nsIGfxInfo> gfxInfo = do_GetService("@mozilla.org/gfx/info;1");
+ if (!gfxInfo) {
+ return false;
+ }
+ int32_t status;
+ nsCString discardFailureId;
+ return NS_SUCCEEDED(gfxInfo->GetFeatureStatus(
+ nsIGfxInfo::FEATURE_WEBRTC_HW_ACCELERATION_H264, discardFailureId,
+ &status)) &&
+ status == nsIGfxInfo::FEATURE_STATUS_OK;
+}
+
+Maybe<int> WebrtcVideoConduit::ActiveSendPayloadType() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ if (!mSendStream) {
+ return Nothing();
+ }
+
+ if (mSendStreamConfig.rtp.payload_type == -1) {
+ return Nothing();
+ }
+
+ return Some(mSendStreamConfig.rtp.payload_type);
+}
+
+Maybe<int> WebrtcVideoConduit::ActiveRecvPayloadType() const {
+ MOZ_ASSERT(mCallThread->IsOnCurrentThread());
+
+ auto stats = GetReceiverStats();
+ if (!stats) {
+ return Nothing();
+ }
+
+ if (stats->current_payload_type == -1) {
+ return Nothing();
+ }
+
+ return Some(stats->current_payload_type);
+}
+
+} // namespace mozilla
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-25 13:56:56 +0000