diff options
Diffstat (limited to 'dom/media')
346 files changed, 6850 insertions, 2628 deletions
diff --git a/dom/media/ADTSDemuxer.cpp b/dom/media/ADTSDemuxer.cpp index 29ea270461..49135efc52 100644 --- a/dom/media/ADTSDemuxer.cpp +++ b/dom/media/ADTSDemuxer.cpp @@ -10,6 +10,7 @@ #include "VideoUtils.h" #include "mozilla/Logging.h" #include "mozilla/UniquePtr.h" +#include "Adts.h" #include <inttypes.h> extern mozilla::LazyLogModule gMediaDemuxerLog; @@ -21,227 +22,6 @@ extern mozilla::LazyLogModule gMediaDemuxerLog; DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Verbose, msg, ##__VA_ARGS__) namespace mozilla { -namespace adts { - -// adts::FrameHeader - Holds the ADTS frame header and its parsing -// state. -// -// ADTS Frame Structure -// -// 11111111 1111BCCD EEFFFFGH HHIJKLMM MMMMMMMM MMMOOOOO OOOOOOPP(QQQQQQQQ -// QQQQQQQQ) -// -// Header consists of 7 or 9 bytes(without or with CRC). -// Letter Length(bits) Description -// { sync } 12 syncword 0xFFF, all bits must be 1 -// B 1 MPEG Version: 0 for MPEG-4, 1 for MPEG-2 -// C 2 Layer: always 0 -// D 1 protection absent, Warning, set to 1 if there is no -// CRC and 0 if there is CRC -// E 2 profile, the MPEG-4 Audio Object Type minus 1 -// F 4 MPEG-4 Sampling Frequency Index (15 is forbidden) -// H 3 MPEG-4 Channel Configuration (in the case of 0, the -// channel configuration is sent via an in-band PCE) -// M 13 frame length, this value must include 7 or 9 bytes of -// header length: FrameLength = -// (ProtectionAbsent == 1 ? 7 : 9) + size(AACFrame) -// O 11 Buffer fullness -// P 2 Number of AAC frames(RDBs) in ADTS frame minus 1, for -// maximum compatibility always use 1 AAC frame per ADTS -// frame -// Q 16 CRC if protection absent is 0 -class FrameHeader { - public: - uint32_t mFrameLength{}; - uint32_t mSampleRate{}; - uint32_t mSamples{}; - uint32_t mChannels{}; - uint8_t mObjectType{}; - uint8_t mSamplingIndex{}; - uint8_t mChannelConfig{}; - uint8_t mNumAACFrames{}; - bool mHaveCrc{}; - - // Returns whether aPtr matches a valid ADTS header sync marker - static bool MatchesSync(const uint8_t* aPtr) { - return aPtr[0] == 0xFF && (aPtr[1] & 0xF6) == 0xF0; - } - - FrameHeader() { Reset(); } - - // Header size - uint64_t HeaderSize() const { return (mHaveCrc) ? 9 : 7; } - - bool IsValid() const { return mFrameLength > 0; } - - // Resets the state to allow for a new parsing session. - void Reset() { PodZero(this); } - - // Returns whether the byte creates a valid sequence up to this point. - bool Parse(const uint8_t* aPtr) { - const uint8_t* p = aPtr; - - if (!MatchesSync(p)) { - return false; - } - - // AAC has 1024 samples per frame per channel. - mSamples = 1024; - - mHaveCrc = !(p[1] & 0x01); - mObjectType = ((p[2] & 0xC0) >> 6) + 1; - mSamplingIndex = (p[2] & 0x3C) >> 2; - mChannelConfig = (p[2] & 0x01) << 2 | (p[3] & 0xC0) >> 6; - mFrameLength = static_cast<uint32_t>( - (p[3] & 0x03) << 11 | (p[4] & 0xFF) << 3 | (p[5] & 0xE0) >> 5); - mNumAACFrames = (p[6] & 0x03) + 1; - - static const uint32_t SAMPLE_RATES[] = {96000, 88200, 64000, 48000, 44100, - 32000, 24000, 22050, 16000, 12000, - 11025, 8000, 7350}; - if (mSamplingIndex >= ArrayLength(SAMPLE_RATES)) { - LOG(("ADTS: Init() failure: invalid sample-rate index value: %" PRIu32 - ".", - mSamplingIndex)); - return false; - } - mSampleRate = SAMPLE_RATES[mSamplingIndex]; - - MOZ_ASSERT(mChannelConfig < 8); - mChannels = (mChannelConfig == 7) ? 8 : mChannelConfig; - - return true; - } -}; - -// adts::Frame - Frame meta container used to parse and hold a frame -// header and side info. -class Frame { - public: - Frame() : mOffset(0) {} - - uint64_t Offset() const { return mOffset; } - size_t Length() const { - // TODO: If fields are zero'd when invalid, this check wouldn't be - // necessary. - if (!mHeader.IsValid()) { - return 0; - } - - return mHeader.mFrameLength; - } - - // Returns the offset to the start of frame's raw data. - uint64_t PayloadOffset() const { return mOffset + mHeader.HeaderSize(); } - - // Returns the length of the frame's raw data (excluding the header) in bytes. - size_t PayloadLength() const { - // TODO: If fields are zero'd when invalid, this check wouldn't be - // necessary. - if (!mHeader.IsValid()) { - return 0; - } - - return mHeader.mFrameLength - mHeader.HeaderSize(); - } - - // Returns the parsed frame header. - const FrameHeader& Header() const { return mHeader; } - - bool IsValid() const { return mHeader.IsValid(); } - - // Resets the frame header and data. - void Reset() { - mHeader.Reset(); - mOffset = 0; - } - - // Returns whether the valid - bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd) { - MOZ_ASSERT(aStart && aEnd); - - bool found = false; - const uint8_t* ptr = aStart; - // Require at least 7 bytes of data at the end of the buffer for the minimum - // ADTS frame header. - while (ptr < aEnd - 7 && !found) { - found = mHeader.Parse(ptr); - ptr++; - } - - mOffset = aOffset + (static_cast<size_t>(ptr - aStart)) - 1u; - - return found; - } - - private: - // The offset to the start of the header. - uint64_t mOffset; - - // The currently parsed frame header. - FrameHeader mHeader; -}; - -class FrameParser { - public: - // Returns the currently parsed frame. Reset via Reset or EndFrameSession. - const Frame& CurrentFrame() const { return mFrame; } - - // Returns the first parsed frame. Reset via Reset. - const Frame& FirstFrame() const { return mFirstFrame; } - - // Resets the parser. Don't use between frames as first frame data is reset. - void Reset() { - EndFrameSession(); - mFirstFrame.Reset(); - } - - // Clear the last parsed frame to allow for next frame parsing, i.e.: - // - sets PrevFrame to CurrentFrame - // - resets the CurrentFrame - // - resets ID3Header if no valid header was parsed yet - void EndFrameSession() { mFrame.Reset(); } - - // Parses contents of given ByteReader for a valid frame header and returns - // true if one was found. After returning, the variable passed to - // 'aBytesToSkip' holds the amount of bytes to be skipped (if any) in order to - // jump across a large ID3v2 tag spanning multiple buffers. - bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd) { - const bool found = mFrame.Parse(aOffset, aStart, aEnd); - - if (mFrame.Length() && !mFirstFrame.Length()) { - mFirstFrame = mFrame; - } - - return found; - } - - private: - // We keep the first parsed frame around for static info access, the - // previously parsed frame for debugging and the currently parsed frame. - Frame mFirstFrame; - Frame mFrame; -}; - -// Initialize the AAC AudioSpecificConfig. -// Only handles two-byte version for AAC-LC. -static void InitAudioSpecificConfig(const Frame& frame, - MediaByteBuffer* aBuffer) { - const FrameHeader& header = frame.Header(); - MOZ_ASSERT(header.IsValid()); - - int audioObjectType = header.mObjectType; - int samplingFrequencyIndex = header.mSamplingIndex; - int channelConfig = header.mChannelConfig; - - uint8_t asc[2]; - asc[0] = (audioObjectType & 0x1F) << 3 | (samplingFrequencyIndex & 0x0E) >> 1; - asc[1] = (samplingFrequencyIndex & 0x01) << 7 | (channelConfig & 0x0F) << 3; - - aBuffer->AppendElements(asc, 2); -} - -} // namespace adts using media::TimeUnit; @@ -292,7 +72,7 @@ bool ADTSDemuxer::IsSeekable() const { // ADTSTrackDemuxer ADTSTrackDemuxer::ADTSTrackDemuxer(MediaResource* aSource) : mSource(aSource), - mParser(new adts::FrameParser()), + mParser(new ADTS::FrameParser()), mOffset(0), mNumParsedFrames(0), mFrameIndex(0), @@ -535,7 +315,7 @@ TimeUnit ADTSTrackDemuxer::Duration(int64_t aNumFrames) const { return TimeUnit(aNumFrames * mSamplesPerFrame, mSamplesPerSecond); } -const adts::Frame& ADTSTrackDemuxer::FindNextFrame( +const ADTS::Frame& ADTSTrackDemuxer::FindNextFrame( bool findFirstFrame /*= false*/) { static const int BUFFER_SIZE = 4096; static const int MAX_SKIPPED_BYTES = 10 * BUFFER_SIZE; @@ -568,7 +348,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( break; } - const adts::Frame& currentFrame = mParser->CurrentFrame(); + const ADTS::Frame& currentFrame = mParser->CurrentFrame(); foundFrame = mParser->Parse(frameHeaderOffset, buffer, buffer + read); if (findFirstFrame && foundFrame) { // Check for sync marker after the found frame, since it's @@ -579,7 +359,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( currentFrame.Offset() + currentFrame.Length(); uint32_t read = Read(buffer, AssertedCast<int64_t>(nextFrameHeaderOffset), 2); - if (read != 2 || !adts::FrameHeader::MatchesSync(buffer)) { + if (read != 2 || !ADTS::FrameHeader::MatchesSync(buffer)) { frameHeaderOffset = currentFrame.Offset() + 1; mParser->Reset(); foundFrame = false; @@ -621,7 +401,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( return mParser->CurrentFrame(); } -bool ADTSTrackDemuxer::SkipNextFrame(const adts::Frame& aFrame) { +bool ADTSTrackDemuxer::SkipNextFrame(const ADTS::Frame& aFrame) { if (!mNumParsedFrames || !aFrame.Length()) { RefPtr<MediaRawData> frame(GetNextFrame(aFrame)); return frame; @@ -639,7 +419,7 @@ bool ADTSTrackDemuxer::SkipNextFrame(const adts::Frame& aFrame) { } already_AddRefed<MediaRawData> ADTSTrackDemuxer::GetNextFrame( - const adts::Frame& aFrame) { + const ADTS::Frame& aFrame) { ADTSLOG("GetNext() Begin({mOffset=%" PRIu64 " HeaderSize()=%" PRIu64 " Length()=%zu})", aFrame.Offset(), aFrame.Header().HeaderSize(), @@ -735,7 +515,7 @@ int64_t ADTSTrackDemuxer::FrameIndexFromTime(const TimeUnit& aTime) const { return std::max<int64_t>(0, frameIndex); } -void ADTSTrackDemuxer::UpdateState(const adts::Frame& aFrame) { +void ADTSTrackDemuxer::UpdateState(const ADTS::Frame& aFrame) { uint32_t frameLength = aFrame.Length(); // Prevent overflow. if (mTotalFrameLen + frameLength < mTotalFrameLen) { @@ -750,7 +530,7 @@ void ADTSTrackDemuxer::UpdateState(const adts::Frame& aFrame) { mTotalFrameLen += frameLength; if (!mSamplesPerFrame) { - const adts::FrameHeader& header = aFrame.Header(); + const ADTS::FrameHeader& header = aFrame.Header(); mSamplesPerFrame = header.mSamples; mSamplesPerSecond = header.mSampleRate; mChannels = header.mChannels; @@ -795,15 +575,15 @@ bool ADTSDemuxer::ADTSSniffer(const uint8_t* aData, const uint32_t aLength) { if (aLength < 7) { return false; } - if (!adts::FrameHeader::MatchesSync(aData)) { + if (!ADTS::FrameHeader::MatchesSync(Span(aData, aLength))) { return false; } - auto parser = MakeUnique<adts::FrameParser>(); + auto parser = MakeUnique<ADTS::FrameParser>(); if (!parser->Parse(0, aData, aData + aLength)) { return false; } - const adts::Frame& currentFrame = parser->CurrentFrame(); + const ADTS::Frame& currentFrame = parser->CurrentFrame(); // Check for sync marker after the found frame, since it's // possible to find sync marker in AAC data. If sync marker // exists after the current frame then we've found a frame @@ -812,7 +592,8 @@ bool ADTSDemuxer::ADTSSniffer(const uint8_t* aData, const uint32_t aLength) { currentFrame.Offset() + currentFrame.Length(); return aLength > nextFrameHeaderOffset && aLength - nextFrameHeaderOffset >= 2 && - adts::FrameHeader::MatchesSync(aData + nextFrameHeaderOffset); + ADTS::FrameHeader::MatchesSync(Span(aData + nextFrameHeaderOffset, + aLength - nextFrameHeaderOffset)); } } // namespace mozilla diff --git a/dom/media/ADTSDemuxer.h b/dom/media/ADTSDemuxer.h index 40ff44898e..67a661883f 100644 --- a/dom/media/ADTSDemuxer.h +++ b/dom/media/ADTSDemuxer.h @@ -11,14 +11,10 @@ #include "mozilla/Maybe.h" #include "MediaDataDemuxer.h" #include "MediaResource.h" +#include "Adts.h" namespace mozilla { -namespace adts { -class Frame; -class FrameParser; -} // namespace adts - class ADTSTrackDemuxer; DDLoggedTypeDeclNameAndBase(ADTSDemuxer, MediaDataDemuxer); @@ -87,16 +83,16 @@ class ADTSTrackDemuxer : public MediaTrackDemuxer, media::TimeUnit ScanUntil(const media::TimeUnit& aTime); // Finds the next valid frame and returns its byte range. - const adts::Frame& FindNextFrame(bool findFirstFrame = false); + const ADTS::Frame& FindNextFrame(bool findFirstFrame = false); // Skips the next frame given the provided byte range. - bool SkipNextFrame(const adts::Frame& aFrame); + bool SkipNextFrame(const ADTS::Frame& aFrame); // Returns the next ADTS frame, if available. - already_AddRefed<MediaRawData> GetNextFrame(const adts::Frame& aFrame); + already_AddRefed<MediaRawData> GetNextFrame(const ADTS::Frame& aFrame); // Updates post-read meta data. - void UpdateState(const adts::Frame& aFrame); + void UpdateState(const ADTS::Frame& aFrame); // Returns the frame index for the given offset. int64_t FrameIndexFromOffset(uint64_t aOffset) const; @@ -115,7 +111,7 @@ class ADTSTrackDemuxer : public MediaTrackDemuxer, MediaResourceIndex mSource; // ADTS frame parser used to detect frames and extract side info. - adts::FrameParser* mParser; + ADTS::FrameParser* mParser; // Current byte offset in the source stream. uint64_t mOffset; diff --git a/dom/media/AudibilityMonitor.h b/dom/media/AudibilityMonitor.h index fdcf474403..25a715cebf 100644 --- a/dom/media/AudibilityMonitor.h +++ b/dom/media/AudibilityMonitor.h @@ -67,7 +67,7 @@ class AudibilityMonitor { for (uint32_t i = 0; i < frameCount; i++) { bool atLeastOneAudible = false; for (uint32_t j = 0; j < aChannels; j++) { - if (std::fabs(AudioSampleToFloat(samples[readIndex++])) > + if (std::fabs(ConvertAudioSample<float>(samples[readIndex++])) > AUDIBILITY_THRESHOLD) { atLeastOneAudible = true; } diff --git a/dom/media/AudioSampleFormat.h b/dom/media/AudioSampleFormat.h index 1cec31a385..7a329d06df 100644 --- a/dom/media/AudioSampleFormat.h +++ b/dom/media/AudioSampleFormat.h @@ -1,13 +1,16 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* vim: set ts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ #ifndef MOZILLA_AUDIOSAMPLEFORMAT_H_ #define MOZILLA_AUDIOSAMPLEFORMAT_H_ #include "mozilla/Assertions.h" +#include "mozilla/PodOperations.h" #include <algorithm> +#include <type_traits> +#include <limits> namespace mozilla { @@ -62,113 +65,191 @@ class AudioSampleTypeToFormat<short> { static const AudioSampleFormat Format = AUDIO_FORMAT_S16; }; -// Single-sample conversion -/* - * Use "2^N" conversion since it's simple, fast, "bit transparent", used by - * many other libraries and apparently behaves reasonably. - * http://blog.bjornroche.com/2009/12/int-float-int-its-jungle-out-there.html - * http://blog.bjornroche.com/2009/12/linearity-and-dynamic-range-in-int.html - */ -inline float AudioSampleToFloat(float aValue) { return aValue; } -inline float AudioSampleToFloat(int16_t aValue) { - return static_cast<float>(aValue) / 32768.0f; -} -inline float AudioSampleToFloat(int32_t aValue) { - return static_cast<float>(aValue) / (float)(1U << 31); +template <typename T> +constexpr float MaxAsFloat() { + return static_cast<float>(std::numeric_limits<T>::max()); } template <typename T> -T FloatToAudioSample(float aValue); - -template <> -inline float FloatToAudioSample<float>(float aValue) { - return aValue; -} -template <> -inline int16_t FloatToAudioSample<int16_t>(float aValue) { - float v = aValue * 32768.0f; - float clamped = std::max(-32768.0f, std::min(32767.0f, v)); - return int16_t(clamped); +constexpr float LowestAsFloat() { + return static_cast<float>(std::numeric_limits<T>::lowest()); } +// The maximum value for an audio sample. If T is signed, the absolute value of +// this number is smaller (by exactly 1) than ::Min(). template <typename T> -T UInt8bitToAudioSample(uint8_t aValue); - -template <> -inline float UInt8bitToAudioSample<float>(uint8_t aValue) { - return static_cast<float>(aValue) * (static_cast<float>(2) / UINT8_MAX) - - static_cast<float>(1); -} -template <> -inline int16_t UInt8bitToAudioSample<int16_t>(uint8_t aValue) { - return static_cast<int16_t>((aValue << 8) + aValue + INT16_MIN); +constexpr T Max() { + return std::numeric_limits<T>::max(); } +// The minimum value for an audio sample. If T is signed, the absolute value of +// this number is greater (by exactly 1) than ::Max() template <typename T> -T IntegerToAudioSample(int16_t aValue); +constexpr T Min() { + return std::numeric_limits<T>::lowest(); +} template <> -inline float IntegerToAudioSample<float>(int16_t aValue) { - return static_cast<float>(aValue) / 32768.0f; +constexpr float Max<float>() { + return 1.0f; } + template <> -inline int16_t IntegerToAudioSample<int16_t>(int16_t aValue) { - return aValue; +constexpr float Min<float>() { + return -1.0f; } +// The bias value is the middle of the range. In linear PCM audio, if the +// values are all equal to the bias value, the audio is silent. template <typename T> -T Int24bitToAudioSample(int32_t aValue); +constexpr T Bias() { + return 0; +} template <> -inline float Int24bitToAudioSample<float>(int32_t aValue) { - return static_cast<float>(aValue) / static_cast<float>(1 << 23); +constexpr uint8_t Bias<uint8_t>() { + return 128; } -template <> -inline int16_t Int24bitToAudioSample<int16_t>(int32_t aValue) { - return static_cast<int16_t>(aValue / 256); + +// Clip a floating point audio sample to its nominal range. This is +// destructive, and is only used here for avoiding overflow in some edge cases, +// so it's not going to be generally audible. +inline float Clip(float aValue) { return std::clamp(aValue, -1.0f, 1.0f); } + +template <typename T> +T FloatToAudioSample(float aValue) { + if constexpr (std::is_same_v<float, T>) { + return aValue; + } + if constexpr (std::is_same_v<uint8_t, T>) { + return static_cast<T>(std::clamp((aValue + 1.0f) * 128.f, + LowestAsFloat<T>(), MaxAsFloat<T>())); + } else if constexpr (std::is_same_v<int16_t, T>) { + // This produces correct results accross the range. + return static_cast<T>(std::clamp(aValue * -LowestAsFloat<T>(), + LowestAsFloat<T>(), MaxAsFloat<T>())); + } else if constexpr (std::is_same_v<int32_t, T>) { + // We need to handle this differently because of rounding between INT32_MAX + // and float 32-bits, to maximise precision. + if (aValue >= 0.) { + // if the input sample is greater OR EQUAL to 1.0, then clip and return + // the max value. + if (aValue >= 1.0) { + return std::numeric_limits<T>::max(); + } + // otherwise cast to a double and map to the positive range. + // float 32-bits cannot represent int32_max (but can represent int32_min) + constexpr double magnitudePos = std::numeric_limits<int32_t>::max(); + return static_cast<int32_t>(aValue * magnitudePos); + } + // Similarly for the negative range. + if (aValue <= -1.0) { + return std::numeric_limits<T>::lowest(); + } + constexpr double magnitudeNegative = + -1.0 * std::numeric_limits<int32_t>::lowest(); + return static_cast<int32_t>(aValue * magnitudeNegative); + } } -template <typename SrcT, typename DstT> -inline void ConvertAudioSample(SrcT aIn, DstT& aOut); +template <typename T> +T UInt8bitToAudioSample(uint8_t aValue) { + if constexpr (std::is_same_v<uint8_t, T>) { + return aValue; + } else if constexpr (std::is_same_v<int16_t, T>) { + return (static_cast<int16_t>(aValue) << 8) - (1 << 15); + } else if constexpr (std::is_same_v<int32_t, T>) { + return (static_cast<int32_t>(aValue) << 24) - (1 << 31); + } else if constexpr (std::is_same_v<float, T>) { + float biased = static_cast<float>(aValue) - Bias<uint8_t>(); + if (aValue >= Bias<uint8_t>()) { + return Clip(biased / MaxAsFloat<int8_t>()); + } + return Clip(biased / -LowestAsFloat<int8_t>()); + } +} -template <> -inline void ConvertAudioSample(int16_t aIn, int16_t& aOut) { - aOut = aIn; +template <typename T> +T Int16ToAudioSample(int16_t aValue) { + if constexpr (std::is_same_v<uint8_t, T>) { + return static_cast<uint8_t>(aValue >> 8) + 128; + } else if constexpr (std::is_same_v<int16_t, T>) { + return aValue; + } else if constexpr (std::is_same_v<int32_t, T>) { + return aValue << 16; + } else if constexpr (std::is_same_v<float, T>) { + if (aValue >= 0) { + return Clip(static_cast<float>(aValue) / MaxAsFloat<int16_t>()); + } + return Clip(static_cast<float>(aValue) / -LowestAsFloat<int16_t>()); + } } -template <> -inline void ConvertAudioSample(int16_t aIn, float& aOut) { - aOut = AudioSampleToFloat(aIn); +// 24-bits audio samples are stored in 32-bits variables. +template <typename T> +T Int24ToAudioSample(int32_t aValue) { + if constexpr (std::is_same_v<uint8_t, T>) { + return static_cast<uint8_t>(aValue >> 16) + 128; + } else if constexpr (std::is_same_v<int16_t, T>) { + return static_cast<int16_t>(aValue >> 8); + } else if constexpr (std::is_same_v<int32_t, T>) { + return aValue << 8; + } else if constexpr (std::is_same_v<float, T>) { + const int32_t min = -(2 << 22); + const int32_t max = (2 << 22) - 1; + if (aValue >= 0) { + return Clip(static_cast<float>(aValue) / static_cast<float>(max)); + } + return Clip(static_cast<float>(aValue) / -static_cast<float>(min)); + } } -template <> -inline void ConvertAudioSample(float aIn, float& aOut) { - aOut = aIn; +template <typename T> +T Int32ToAudioSample(int32_t aValue) { + if constexpr (std::is_same_v<uint8_t, T>) { + return static_cast<uint8_t>(aValue >> 24) + 128; + } else if constexpr (std::is_same_v<int16_t, T>) { + return aValue >> 16; + } else if constexpr (std::is_same_v<int32_t, T>) { + return aValue; + } else if constexpr (std::is_same_v<float, T>) { + if (aValue >= 0) { + return Clip(static_cast<float>(aValue) / MaxAsFloat<int32_t>()); + } + return Clip(static_cast<float>(aValue) / -LowestAsFloat<int32_t>()); + } } -template <> -inline void ConvertAudioSample(float aIn, int16_t& aOut) { - aOut = FloatToAudioSample<int16_t>(aIn); +// This does not handle 24-bits audio, call the function explicitly when +// needed. +template <typename D, typename S> +inline D ConvertAudioSample(const S& aSource) { + if constexpr (std::is_same_v<S, D>) { + return aSource; + } else if constexpr (std::is_same_v<S, uint8_t>) { + return UInt8bitToAudioSample<D>(aSource); + } else if constexpr (std::is_same_v<S, int16_t>) { + return Int16ToAudioSample<D>(aSource); + } else if constexpr (std::is_same_v<S, int32_t>) { + return Int32ToAudioSample<D>(aSource); + } else if constexpr (std::is_same_v<S, float>) { + return FloatToAudioSample<D>(aSource); + } } // Sample buffer conversion - template <typename From, typename To> inline void ConvertAudioSamples(const From* aFrom, To* aTo, int aCount) { + if constexpr (std::is_same_v<From, To>) { + PodCopy(aTo, aFrom, aCount); + return; + } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample<To>(AudioSampleToFloat(aFrom[i])); + aTo[i] = ConvertAudioSample<To>(aFrom[i]); } } -inline void ConvertAudioSamples(const int16_t* aFrom, int16_t* aTo, - int aCount) { - memcpy(aTo, aFrom, sizeof(*aTo) * aCount); -} -inline void ConvertAudioSamples(const float* aFrom, float* aTo, int aCount) { - memcpy(aTo, aFrom, sizeof(*aTo) * aCount); -} // Sample buffer conversion with scale - template <typename From, typename To> inline void ConvertAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, float aScale) { @@ -177,7 +258,8 @@ inline void ConvertAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, return; } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample<To>(AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = + ConvertAudioSample<To>(ConvertAudioSample<float>(aFrom[i]) * aScale); } } inline void ConvertAudioSamplesWithScale(const int16_t* aFrom, int16_t* aTo, @@ -194,7 +276,8 @@ inline void ConvertAudioSamplesWithScale(const int16_t* aFrom, int16_t* aTo, return; } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample<int16_t>(AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = FloatToAudioSample<int16_t>(ConvertAudioSample<float>(aFrom[i]) * + aScale); } } @@ -202,8 +285,9 @@ template <typename From, typename To> inline void AddAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, float aScale) { for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample<To>(AudioSampleToFloat(aTo[i]) + - AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = + ConvertAudioSample<To>(ConvertAudioSample<float>(aTo[i]) + + ConvertAudioSample<float>(aFrom[i]) * aScale); } } diff --git a/dom/media/AudioSegment.h b/dom/media/AudioSegment.h index 006f996c39..7d60bf7ca5 100644 --- a/dom/media/AudioSegment.h +++ b/dom/media/AudioSegment.h @@ -79,7 +79,8 @@ static void InterleaveAndConvertBuffer(const SrcT* const* aSourceChannels, DestT* output = aOutput; for (size_t i = 0; i < aLength; ++i) { for (size_t channel = 0; channel < aChannels; ++channel) { - float v = AudioSampleToFloat(aSourceChannels[channel][i]) * aVolume; + float v = + ConvertAudioSample<float>(aSourceChannels[channel][i]) * aVolume; *output = FloatToAudioSample<DestT>(v); ++output; } @@ -93,7 +94,8 @@ static void DeinterleaveAndConvertBuffer(const SrcT* aSourceBuffer, for (size_t i = 0; i < aChannels; i++) { size_t interleavedIndex = i; for (size_t j = 0; j < aFrames; j++) { - ConvertAudioSample(aSourceBuffer[interleavedIndex], aOutput[i][j]); + aOutput[i][j] = + ConvertAudioSample<DestT>(aSourceBuffer[interleavedIndex]); interleavedIndex += aChannels; } } @@ -148,7 +150,7 @@ void DownmixAndInterleave(Span<const SrcT* const> aChannelData, * separate pointers to each channel's buffer. */ struct AudioChunk { - typedef mozilla::AudioSampleFormat SampleFormat; + using SampleFormat = mozilla::AudioSampleFormat; AudioChunk() = default; @@ -318,7 +320,7 @@ struct AudioChunk { * A list of audio samples consisting of a sequence of slices of SharedBuffers. * The audio rate is determined by the track, not stored in this class. */ -class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> { +class AudioSegment final : public MediaSegmentBase<AudioSegment, AudioChunk> { // The channel count that MaxChannelCount() returned last time it was called. uint32_t mMemoizedMaxChannelCount = 0; diff --git a/dom/media/CallbackThreadRegistry.cpp b/dom/media/CallbackThreadRegistry.cpp index f4d2af5bd1..84ef7b7cb4 100644 --- a/dom/media/CallbackThreadRegistry.cpp +++ b/dom/media/CallbackThreadRegistry.cpp @@ -6,6 +6,7 @@ #include "CallbackThreadRegistry.h" #include "mozilla/ClearOnShutdown.h" +#include "nsThreadUtils.h" namespace mozilla { struct CallbackThreadRegistrySingleton { diff --git a/dom/media/ChannelMediaDecoder.cpp b/dom/media/ChannelMediaDecoder.cpp index c6da221f94..12f6c11e47 100644 --- a/dom/media/ChannelMediaDecoder.cpp +++ b/dom/media/ChannelMediaDecoder.cpp @@ -221,9 +221,11 @@ MediaDecoderStateMachineBase* ChannelMediaDecoder::CreateStateMachine( mReader = DecoderTraits::CreateReader(ContainerType(), init); #ifdef MOZ_WMF_MEDIA_ENGINE - // TODO : Only for testing development for now. In the future this should be - // used for encrypted content only. - if (StaticPrefs::media_wmf_media_engine_enabled() && + // This state machine is mainly used for the encrypted playback. However, for + // testing purpose we would also use it the non-encrypted playback. + // 1=enabled encrypted and clear, 3=enabled clear + if ((StaticPrefs::media_wmf_media_engine_enabled() == 1 || + StaticPrefs::media_wmf_media_engine_enabled() == 3) && StaticPrefs::media_wmf_media_engine_channel_decoder_enabled() && !aDisableExternalEngine) { return new ExternalEngineStateMachine(this, mReader); diff --git a/dom/media/ExternalEngineStateMachine.cpp b/dom/media/ExternalEngineStateMachine.cpp index 68fb053b83..acfc1f5fa2 100644 --- a/dom/media/ExternalEngineStateMachine.cpp +++ b/dom/media/ExternalEngineStateMachine.cpp @@ -10,6 +10,7 @@ # include "mozilla/MFMediaEngineChild.h" # include "mozilla/StaticPrefs_media.h" #endif +#include "mozilla/AppShutdown.h" #include "mozilla/Atomics.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/ProfilerLabels.h" @@ -203,6 +204,10 @@ ExternalEngineStateMachine::ExternalEngineStateMachine( InitEngine(); } +ExternalEngineStateMachine::~ExternalEngineStateMachine() { + LOG("ExternalEngineStateMachine is destroyed"); +} + void ExternalEngineStateMachine::InitEngine() { MOZ_ASSERT(mState.IsInitEngine() || mState.IsRecoverEngine()); #ifdef MOZ_WMF_MEDIA_ENGINE @@ -565,6 +570,9 @@ RefPtr<ShutdownPromise> ExternalEngineStateMachine::Shutdown() { mSetCDMProxyPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_ABORT_ERR, __func__); mSetCDMProxyRequest.DisconnectIfExists(); + mInitEngineForCDMRequest.DisconnectIfExists(); + + mPendingTasks.Clear(); mEngine->Shutdown(); @@ -607,49 +615,59 @@ void ExternalEngineStateMachine::BufferedRangeUpdated() { } } -// Note: the variadic only supports passing member variables. -#define PERFORM_WHEN_ALLOW(Func, ...) \ - do { \ - /* Initialzation is not done yet, postpone the operation */ \ - if ((mState.IsInitEngine() || mState.IsRecoverEngine()) && \ - mState.AsInitEngine()->mInitPromise) { \ - LOG("%s is called before init", __func__); \ - mState.AsInitEngine()->mInitPromise->Then( \ - OwnerThread(), __func__, \ - [self = RefPtr{this}, this]( \ - const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { \ - if (aVal.IsResolve()) { \ - Func(__VA_ARGS__); \ - } \ - }); \ - return; \ - } else if (mState.IsShutdownEngine()) { \ - return; \ - } \ +#define PERFORM_WHEN_ALLOW(Func) \ + do { \ + if (mState.IsShutdownEngine() || mHasFatalError || \ + AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { \ + return; \ + } \ + /* Initialzation is not done yet, postpone the operation */ \ + if ((mState.IsInitEngine() || mState.IsRecoverEngine()) && \ + mState.AsInitEngine()->mInitPromise) { \ + LOG("%s is called before init", __func__); \ + mPendingTasks.AppendElement(NewRunnableMethod( \ + __func__, this, &ExternalEngineStateMachine::Func)); \ + return; \ + } \ } while (false) void ExternalEngineStateMachine::SetPlaybackRate(double aPlaybackRate) { AssertOnTaskQueue(); + // TODO : consider to make `mPlaybackRate` a mirror to fit other usages like + // `mVolume` and `mPreservesPitch`. mPlaybackRate = aPlaybackRate; - PERFORM_WHEN_ALLOW(SetPlaybackRate, mPlaybackRate); - mEngine->SetPlaybackRate(aPlaybackRate); + PlaybackRateChanged(); +} + +void ExternalEngineStateMachine::PlaybackRateChanged() { + AssertOnTaskQueue(); + PERFORM_WHEN_ALLOW(PlaybackRateChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); + mEngine->SetPlaybackRate(mPlaybackRate); } void ExternalEngineStateMachine::VolumeChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(VolumeChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetVolume(mVolume); } void ExternalEngineStateMachine::PreservesPitchChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(PreservesPitchChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetPreservesPitch(mPreservesPitch); } void ExternalEngineStateMachine::PlayStateChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(PlayStateChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); if (mPlayState == MediaDecoder::PLAY_STATE_PLAYING) { mEngine->Play(); } else if (mPlayState == MediaDecoder::PLAY_STATE_PAUSED) { @@ -660,6 +678,8 @@ void ExternalEngineStateMachine::PlayStateChanged() { void ExternalEngineStateMachine::LoopingChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(LoopingChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetLooping(mLooping); } @@ -775,6 +795,13 @@ void ExternalEngineStateMachine::StartRunningEngine() { if (HasVideo()) { RunningEngineUpdate(MediaData::Type::VIDEO_DATA); } + // Run tasks which was called before the engine is ready. + if (!mPendingTasks.IsEmpty()) { + for (auto& task : mPendingTasks) { + Unused << OwnerThread()->Dispatch(task.forget()); + } + mPendingTasks.Clear(); + } } void ExternalEngineStateMachine::RunningEngineUpdate(MediaData::Type aType) { @@ -1141,6 +1168,14 @@ void ExternalEngineStateMachine::RecoverFromCDMProcessCrashIfNeeded() { return; } + if (mState.IsInitEngine()) { + LOG("Failed on the engine initialization, the media engine playback might " + "not be supported"); + DecodeError( + MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); + return; + } + LOG("CDM process crashed, recover the engine again (last time=%" PRId64 ")", mCurrentPosition.Ref().ToMicroseconds()); ChangeStateTo(State::RecoverEngine); @@ -1180,24 +1215,28 @@ RefPtr<SetCDMPromise> ExternalEngineStateMachine::SetCDMProxy( if (mState.IsInitEngine() && mState.AsInitEngine()->mInitPromise) { LOG("SetCDMProxy is called before init"); - mState.AsInitEngine()->mInitPromise->Then( - OwnerThread(), __func__, - [self = RefPtr{this}, proxy = RefPtr{aProxy}, - this](const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { - SetCDMProxy(proxy) - ->Then(OwnerThread(), __func__, - [self = RefPtr{this}, - this](const SetCDMPromise::ResolveOrRejectValue& aVal) { - mSetCDMProxyRequest.Complete(); - if (aVal.IsResolve()) { - mSetCDMProxyPromise.Resolve(true, __func__); - } else { - mSetCDMProxyPromise.Reject(NS_ERROR_DOM_MEDIA_CDM_ERR, - __func__); - } - }) - ->Track(mSetCDMProxyRequest); - }); + mState.AsInitEngine() + ->mInitPromise + ->Then( + OwnerThread(), __func__, + [self = RefPtr{this}, proxy = RefPtr{aProxy}, this]( + const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { + mInitEngineForCDMRequest.Complete(); + SetCDMProxy(proxy) + ->Then(OwnerThread(), __func__, + [self = RefPtr{this}, this]( + const SetCDMPromise::ResolveOrRejectValue& aVal) { + mSetCDMProxyRequest.Complete(); + if (aVal.IsResolve()) { + mSetCDMProxyPromise.Resolve(true, __func__); + } else { + mSetCDMProxyPromise.Reject( + NS_ERROR_DOM_MEDIA_CDM_ERR, __func__); + } + }) + ->Track(mSetCDMProxyRequest); + }) + ->Track(mInitEngineForCDMRequest); return mSetCDMProxyPromise.Ensure(__func__); } @@ -1232,6 +1271,7 @@ bool ExternalEngineStateMachine::IsCDMProxySupported(CDMProxy* aProxy) { void ExternalEngineStateMachine::ReportTelemetry(const MediaResult& aError) { glean::mfcdm::ErrorExtra extraData; extraData.errorName = Some(aError.ErrorName()); + extraData.currentState = Some(nsAutoCString{StateToStr(mState.mName)}); nsAutoCString resolution; if (mInfo) { if (mInfo->HasAudio()) { @@ -1268,6 +1308,14 @@ void ExternalEngineStateMachine::ReportTelemetry(const MediaResult& aError) { } } +void ExternalEngineStateMachine::DecodeError(const MediaResult& aError) { + if (aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA || + aError != NS_ERROR_DOM_MEDIA_CANCELED) { + mHasFatalError = true; + } + MediaDecoderStateMachineBase ::DecodeError(aError); +} + #undef FMT #undef LOG #undef LOGV diff --git a/dom/media/ExternalEngineStateMachine.h b/dom/media/ExternalEngineStateMachine.h index 84dedbe717..79183f894d 100644 --- a/dom/media/ExternalEngineStateMachine.h +++ b/dom/media/ExternalEngineStateMachine.h @@ -102,8 +102,10 @@ class ExternalEngineStateMachine final bool IsCDMProxySupported(CDMProxy* aProxy) override; + bool IsExternalEngineStateMachine() const override { return true; } + private: - ~ExternalEngineStateMachine() = default; + ~ExternalEngineStateMachine(); void AssertOnTaskQueue() const { MOZ_ASSERT(OnTaskQueue()); } @@ -233,6 +235,7 @@ class ExternalEngineStateMachine final void PreservesPitchChanged() override; void PlayStateChanged() override; void LoopingChanged() override; + void PlaybackRateChanged(); // Not supported. void SetIsLiveStream(bool aIsLiveStream) override {} @@ -293,6 +296,8 @@ class ExternalEngineStateMachine final void ReportTelemetry(const MediaResult& aError); + void DecodeError(const MediaResult& aError) override; + UniquePtr<ExternalPlaybackEngine> mEngine; bool mHasEnoughAudio = false; @@ -303,12 +308,20 @@ class ExternalEngineStateMachine final // Only used if setting CDM happens before the engine finishes initialization. MozPromiseHolder<SetCDMPromise> mSetCDMProxyPromise; MozPromiseRequestHolder<SetCDMPromise> mSetCDMProxyRequest; + MozPromiseRequestHolder<GenericNonExclusivePromise> mInitEngineForCDMRequest; // It would be zero for audio-only playback. gfx::IntSize mVideoDisplay; // It would be set if playback is encrypted. nsCString mKeySystem; + + // This array stores the tasks which needs to be executed only after the + // engine is ready but is called before that. It will be executed when + // starting running the engine. + nsTArray<RefPtr<nsIRunnable>> mPendingTasks; + + bool mHasFatalError = false; }; class ExternalPlaybackEngine { diff --git a/dom/media/IdpSandbox.sys.mjs b/dom/media/IdpSandbox.sys.mjs index 8b94abf7dd..c8735c223d 100644 --- a/dom/media/IdpSandbox.sys.mjs +++ b/dom/media/IdpSandbox.sys.mjs @@ -56,7 +56,7 @@ ResourceLoader.prototype = { this.data += stream.read(count); }, - onStartRequest(request) {}, + onStartRequest() {}, onStopRequest(request, status) { if (Components.isSuccessCode(status)) { diff --git a/dom/media/ImageToI420.cpp b/dom/media/ImageToI420.cpp index 8fc5198b4a..0f7976cb63 100644 --- a/dom/media/ImageToI420.cpp +++ b/dom/media/ImageToI420.cpp @@ -11,6 +11,7 @@ #include "mozilla/dom/ImageUtils.h" #include "mozilla/gfx/Point.h" #include "mozilla/RefPtr.h" +#include "mozilla/Result.h" #include "nsThreadUtils.h" using mozilla::ImageFormat; @@ -75,7 +76,12 @@ nsresult ConvertToI420(Image* aImage, uint8_t* aDestY, int aDestStrideY, if (const PlanarYCbCrData* data = GetPlanarYCbCrData(aImage)) { const ImageUtils imageUtils(aImage); - switch (imageUtils.GetFormat()) { + Maybe<dom::ImageBitmapFormat> format = imageUtils.GetFormat(); + if (format.isNothing()) { + MOZ_ASSERT_UNREACHABLE("YUV format conversion not implemented"); + return NS_ERROR_NOT_IMPLEMENTED; + } + switch (format.value()) { case ImageBitmapFormat::YUV420P: return MapRv(libyuv::I420ToI420( data->mYChannel, data->mYStride, data->mCbChannel, diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h index ee1e204815..3ae8c1dbc2 100644 --- a/dom/media/MediaData.h +++ b/dom/media/MediaData.h @@ -97,8 +97,16 @@ class AlignedBuffer { } AlignedBuffer& operator=(AlignedBuffer&& aOther) noexcept { - this->~AlignedBuffer(); - new (this) AlignedBuffer(std::move(aOther)); + if (&aOther == this) { + return *this; + } + mData = aOther.mData; + mLength = aOther.mLength; + mBuffer = std::move(aOther.mBuffer); + mCapacity = aOther.mCapacity; + aOther.mData = nullptr; + aOther.mLength = 0; + aOther.mCapacity = 0; return *this; } @@ -268,7 +276,7 @@ class InflatableShortBuffer { // capacity, and the loop goes backward. float* output = reinterpret_cast<float*>(mBuffer.mData); for (size_t i = Length(); i--;) { - output[i] = AudioSampleToFloat(mBuffer.mData[i]); + output[i] = ConvertAudioSample<float>(mBuffer.mData[i]); } AlignedFloatBuffer rv; rv.mBuffer = std::move(mBuffer.mBuffer); diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index c7fdcb6844..23c30eed2a 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -140,9 +140,7 @@ void MediaDecoder::InitStatics() { # if defined(MOZ_FFMPEG) Preferences::Lock("media.utility-ffmpeg.enabled"); # endif // defined(MOZ_FFMPEG) -# if defined(MOZ_FFVPX) Preferences::Lock("media.utility-ffvpx.enabled"); -# endif // defined(MOZ_FFVPX) # if defined(MOZ_WMF) Preferences::Lock("media.utility-wmf.enabled"); # endif // defined(MOZ_WMF) @@ -194,6 +192,12 @@ void MediaDecoder::SetOutputCaptureState(OutputCaptureState aState, MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); MOZ_ASSERT_IF(aState == OutputCaptureState::Capture, aDummyTrack); + + if (mOutputCaptureState.Ref() != aState) { + LOG("Capture state change from %s to %s", + OutputCaptureStateToStr(mOutputCaptureState.Ref()), + OutputCaptureStateToStr(aState)); + } mOutputCaptureState = aState; if (mOutputDummyTrack.Ref().get() != aDummyTrack) { mOutputDummyTrack = nsMainThreadPtrHandle<SharedDummyTrack>( @@ -450,6 +454,7 @@ void MediaDecoder::OnPlaybackErrorEvent(const MediaResult& aError) { } LOG("Need to create a new %s state machine", needExternalEngine ? "external engine" : "normal"); + mStateMachineRecreated = true; nsresult rv = CreateAndInitStateMachine( false /* live stream */, @@ -611,6 +616,7 @@ nsresult MediaDecoder::CreateAndInitStateMachine(bool aIsLiveStream, NS_ENSURE_TRUE(GetStateMachine(), NS_ERROR_FAILURE); GetStateMachine()->DispatchIsLiveStream(aIsLiveStream); + mMDSMCreationTime = Some(TimeStamp::Now()); nsresult rv = mDecoderStateMachine->Init(this); NS_ENSURE_SUCCESS(rv, rv); @@ -885,6 +891,14 @@ void MediaDecoder::FirstFrameLoaded( ChangeState(mNextState); } + // We only care about video first frame. + if (mInfo->HasVideo() && mMDSMCreationTime) { + mTelemetryProbesReporter->OntFirstFrameLoaded( + TimeStamp::Now() - *mMDSMCreationTime, IsMSE(), + mDecoderStateMachine->IsExternalEngineStateMachine()); + mMDSMCreationTime.reset(); + } + // GetOwner()->FirstFrameLoaded() might call us back. Put it at the bottom of // this function to avoid unexpected shutdown from reentrant calls. if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) { diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index f2f10a67c6..a5494e9a84 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -207,6 +207,20 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> { // not connected to streams created by captureStreamUntilEnded. enum class OutputCaptureState { Capture, Halt, None }; + const char* OutputCaptureStateToStr(OutputCaptureState aState) const { + switch (aState) { + case OutputCaptureState::Capture: + return "Capture"; + case OutputCaptureState::Halt: + return "Halt"; + case OutputCaptureState::None: + return "None"; + default: + MOZ_ASSERT_UNREACHABLE("Not defined state!"); + return "Not-defined"; + } + } + // Set the output capture state of this decoder. // @param aState Capture: Output is captured into output tracks, and // aDummyTrack must be provided. @@ -742,6 +756,12 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> { bool mShouldDelaySeek = false; Maybe<SeekTarget> mDelayedSeekTarget; +# ifdef MOZ_WMF_MEDIA_ENGINE + // True if we've ever recreated a new state machine due to the previous state + // didn't support the media format or key system. + bool mStateMachineRecreated = false; +# endif + public: Canonical<double>& CanonicalVolume() { return mVolume; } Canonical<bool>& CanonicalPreservesPitch() { return mPreservesPitch; } @@ -815,6 +835,11 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> { // consistent with the previous destroyed one. bool mPendingStatusUpdateForNewlyCreatedStateMachine = false; # endif + + // The time of creating the media decoder state machine, it's used to record + // the probe for measuring the first video frame loaded time. Reset after + // reporting the measurement to avoid a dulpicated report. + Maybe<TimeStamp> mMDSMCreationTime; }; } // namespace mozilla diff --git a/dom/media/MediaDecoderStateMachineBase.cpp b/dom/media/MediaDecoderStateMachineBase.cpp index e60937c31f..38d51dbd8e 100644 --- a/dom/media/MediaDecoderStateMachineBase.cpp +++ b/dom/media/MediaDecoderStateMachineBase.cpp @@ -163,7 +163,9 @@ bool MediaDecoderStateMachineBase::OnTaskQueue() const { void MediaDecoderStateMachineBase::DecodeError(const MediaResult& aError) { MOZ_ASSERT(OnTaskQueue()); - LOGE("Decode error: %s", aError.Description().get()); + if (aError != NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR) { + LOGE("Decode error: %s", aError.Description().get()); + } PROFILER_MARKER_TEXT("MDSMBase::DecodeError", MEDIA_PLAYBACK, {}, aError.Description()); // Notify the decode error and MediaDecoder will shut down MDSM. diff --git a/dom/media/MediaDecoderStateMachineBase.h b/dom/media/MediaDecoderStateMachineBase.h index b4950746e8..5872151015 100644 --- a/dom/media/MediaDecoderStateMachineBase.h +++ b/dom/media/MediaDecoderStateMachineBase.h @@ -169,6 +169,8 @@ class MediaDecoderStateMachineBase { virtual bool IsCDMProxySupported(CDMProxy* aProxy) = 0; + virtual bool IsExternalEngineStateMachine() const { return false; } + protected: virtual ~MediaDecoderStateMachineBase() = default; @@ -195,7 +197,7 @@ class MediaDecoderStateMachineBase { virtual RefPtr<MediaDecoder::SeekPromise> Seek(const SeekTarget& aTarget) = 0; - void DecodeError(const MediaResult& aError); + virtual void DecodeError(const MediaResult& aError); // Functions used by assertions to ensure we're calling things // on the appropriate threads. diff --git a/dom/media/MediaDevices.cpp b/dom/media/MediaDevices.cpp index cfbc148337..b312fc1ec3 100644 --- a/dom/media/MediaDevices.cpp +++ b/dom/media/MediaDevices.cpp @@ -315,8 +315,6 @@ RefPtr<MediaDeviceSetRefCnt> MediaDevices::FilterExposedDevices( } haveDefaultOutput = true; break; - case MediaDeviceKind::EndGuard_: - continue; // Avoid `default:` so that `-Wswitch` catches missing // enumerators at compile time. } @@ -334,8 +332,6 @@ bool MediaDevices::CanExposeInfo(MediaDeviceKind aKind) const { case MediaDeviceKind::Audiooutput: // Assumes caller has used FilterExposedDevices() return true; - case MediaDeviceKind::EndGuard_: - break; // Avoid `default:` so that `-Wswitch` catches missing enumerators at // compile time. } @@ -550,7 +546,7 @@ already_AddRefed<Promise> MediaDevices::GetDisplayMedia( // for us. vc.mMediaSource.Reset(); vc.mMediaSource.Construct().AssignASCII( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Screen)); + dom::GetEnumString(MediaSourceEnum::Screen)); RefPtr<MediaDevices> self(this); MediaManager::Get() diff --git a/dom/media/MediaInfo.h b/dom/media/MediaInfo.h index abe5dfd9da..73704d1593 100644 --- a/dom/media/MediaInfo.h +++ b/dom/media/MediaInfo.h @@ -219,14 +219,6 @@ inline already_AddRefed<MediaByteBuffer> ForceGetAudioCodecSpecificBlob( // information as a blob or where a blob is ambiguous. inline already_AddRefed<MediaByteBuffer> GetAudioCodecSpecificBlob( const AudioCodecSpecificVariant& v) { - MOZ_ASSERT(!v.is<NoCodecSpecificData>(), - "NoCodecSpecificData shouldn't be used as a blob"); - MOZ_ASSERT(!v.is<AacCodecSpecificData>(), - "AacCodecSpecificData has 2 blobs internally, one should " - "explicitly be selected"); - MOZ_ASSERT(!v.is<Mp3CodecSpecificData>(), - "Mp3CodecSpecificData shouldn't be used as a blob"); - return ForceGetAudioCodecSpecificBlob(v); } @@ -470,7 +462,8 @@ class VideoInfo : public TrackInfo { rv.AppendPrintf("extra data: %zu bytes", mExtraData->Length()); } rv.AppendPrintf("rotation: %d", static_cast<int>(mRotation)); - rv.AppendPrintf("colors: %s", ColorDepthStrings[static_cast<int>(mColorDepth)]); + rv.AppendPrintf("colors: %s", + ColorDepthStrings[static_cast<int>(mColorDepth)]); if (mColorSpace) { rv.AppendPrintf( "YUV colorspace: %s ", @@ -486,7 +479,8 @@ class VideoInfo : public TrackInfo { "transfer function %s ", TransferFunctionStrings[static_cast<int>(mTransferFunction.value())]); } - rv.AppendPrintf("color range: %s", ColorRangeStrings[static_cast<int>(mColorRange)]); + rv.AppendPrintf("color range: %s", + ColorRangeStrings[static_cast<int>(mColorRange)]); if (mImageRect) { rv.AppendPrintf("image rect: %dx%d", mImageRect->Width(), mImageRect->Height()); diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 25eeb876d4..422769587a 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -872,8 +872,7 @@ MediaDevice::MediaDevice(MediaEngine* aEngine, MediaSourceEnum aMediaSource, mCanRequestOsLevelPrompt(canRequestOsLevelPrompt == OsPromptable::Yes), mIsFake(mEngine->IsFake()), mIsPlaceholder(aIsPlaceholder == IsPlaceholder::Yes), - mType( - NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))), + mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))), mRawID(aRawID), mRawGroupID(aRawGroupID), mRawName(aRawName) { @@ -895,8 +894,7 @@ MediaDevice::MediaDevice(MediaEngine* aEngine, mCanRequestOsLevelPrompt(false), mIsFake(false), mIsPlaceholder(false), - mType( - NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))), + mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))), mRawID(aRawID), mRawGroupID(mAudioDeviceInfo->GroupID()), mRawName(mAudioDeviceInfo->Name()) {} @@ -1064,8 +1062,7 @@ LocalMediaDevice::GetMediaSource(nsAString& aMediaSource) { if (Kind() == MediaDeviceKind::Audiooutput) { aMediaSource.Truncate(); } else { - aMediaSource.AssignASCII( - dom::MediaSourceEnumValues::GetString(GetMediaSource())); + aMediaSource.AssignASCII(dom::GetEnumString(GetMediaSource())); } return NS_OK; } @@ -2249,14 +2246,6 @@ MediaManager::MediaManager(already_AddRefed<TaskQueue> aMediaThread) GetPrefs(branch, nullptr); } } - LOG("%s: default prefs: %dx%d @%dfps, %dHz test tones, aec: %s," - "agc: %s, hpf: %s, noise: %s, agc level: %d, agc version: %s, noise " - "level: %d, transient: %s, channels %d", - __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mFreq, - mPrefs.mAecOn ? "on" : "off", mPrefs.mAgcOn ? "on" : "off", - mPrefs.mHPFOn ? "on" : "off", mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAgc, - mPrefs.mAgc2Forced ? "2" : "1", mPrefs.mNoise, - mPrefs.mTransientOn ? "on" : "off", mPrefs.mChannels); } NS_IMPL_ISUPPORTS(MediaManager, nsIMediaManagerService, nsIMemoryReporter, @@ -2755,10 +2744,10 @@ RefPtr<MediaManager::StreamPromise> MediaManager::GetUserMedia( auto& vc = c.mVideo.GetAsMediaTrackConstraints(); if (!vc.mMediaSource.WasPassed()) { vc.mMediaSource.Construct().AssignASCII( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Camera)); + dom::GetEnumString(MediaSourceEnum::Camera)); } - videoType = StringToEnum(dom::MediaSourceEnumValues::strings, - vc.mMediaSource.Value(), MediaSourceEnum::Other); + videoType = dom::StringToEnum<MediaSourceEnum>(vc.mMediaSource.Value()) + .valueOr(MediaSourceEnum::Other); Telemetry::Accumulate(Telemetry::WEBRTC_GET_USER_MEDIA_TYPE, (uint32_t)videoType); switch (videoType) { @@ -2823,8 +2812,7 @@ RefPtr<MediaManager::StreamPromise> MediaManager::GetUserMedia( if (videoType == MediaSourceEnum::Screen || videoType == MediaSourceEnum::Browser) { videoType = MediaSourceEnum::Window; - vc.mMediaSource.Value().AssignASCII( - dom::MediaSourceEnumValues::GetString(videoType)); + vc.mMediaSource.Value().AssignASCII(dom::GetEnumString(videoType)); } // only allow privileged content to set the window id if (vc.mBrowserWindow.WasPassed()) { @@ -2848,10 +2836,10 @@ RefPtr<MediaManager::StreamPromise> MediaManager::GetUserMedia( auto& ac = c.mAudio.GetAsMediaTrackConstraints(); if (!ac.mMediaSource.WasPassed()) { ac.mMediaSource.Construct(NS_ConvertASCIItoUTF16( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Microphone))); + dom::GetEnumString(MediaSourceEnum::Microphone))); } - audioType = StringToEnum(dom::MediaSourceEnumValues::strings, - ac.mMediaSource.Value(), MediaSourceEnum::Other); + audioType = dom::StringToEnum<MediaSourceEnum>(ac.mMediaSource.Value()) + .valueOr(MediaSourceEnum::Other); Telemetry::Accumulate(Telemetry::WEBRTC_GET_USER_MEDIA_TYPE, (uint32_t)audioType); @@ -3498,6 +3486,14 @@ void MediaManager::GetPrefs(nsIPrefBranch* aBranch, const char* aData) { GetPref(aBranch, "media.getusermedia.noise", aData, &mPrefs.mNoise); GetPref(aBranch, "media.getusermedia.channels", aData, &mPrefs.mChannels); #endif + LOG("%s: default prefs: %dx%d @%dfps, %dHz test tones, aec: %s, " + "agc: %s, hpf: %s, noise: %s, agc level: %d, agc version: %s, noise " + "level: %d, transient: %s, channels %d", + __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mFreq, + mPrefs.mAecOn ? "on" : "off", mPrefs.mAgcOn ? "on" : "off", + mPrefs.mHPFOn ? "on" : "off", mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAgc, + mPrefs.mAgc2Forced ? "2" : "1", mPrefs.mNoise, + mPrefs.mTransientOn ? "on" : "off", mPrefs.mChannels); } void MediaManager::Shutdown() { @@ -3650,8 +3646,6 @@ nsresult MediaManager::Observe(nsISupports* aSubject, const char* aTopic, nsCOMPtr<nsIPrefBranch> branch(do_QueryInterface(aSubject)); if (branch) { GetPrefs(branch, NS_ConvertUTF16toUTF8(aData).get()); - LOG("%s: %dx%d @%dfps", __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, - mPrefs.mFPS); DeviceListChanged(); } } else if (!strcmp(aTopic, "last-pb-context-exited")) { @@ -4006,8 +4000,7 @@ void DeviceListener::Activate(RefPtr<LocalMediaDevice> aDevice, MOZ_ASSERT(NS_IsMainThread(), "Only call on main thread"); LOG("DeviceListener %p activating %s device %p", this, - nsCString(dom::MediaDeviceKindValues::GetString(aDevice->Kind())).get(), - aDevice.get()); + dom::GetEnumString(aDevice->Kind()).get(), aDevice.get()); MOZ_ASSERT(!mStopped, "Cannot activate stopped device listener"); MOZ_ASSERT(!Activated(), "Already activated"); @@ -4063,18 +4056,15 @@ DeviceListener::InitializeAsync() { } if (NS_FAILED(rv)) { nsCString log; - log.AppendPrintf( - "Starting %s failed", - nsCString(dom::MediaDeviceKindValues::GetString(kind)) - .get()); + log.AppendPrintf("Starting %s failed", + dom::GetEnumString(kind).get()); aHolder.Reject( MakeRefPtr<MediaMgrError>(MediaMgrError::Name::AbortError, std::move(log)), __func__); return; } - LOG("started %s device %p", - nsCString(dom::MediaDeviceKindValues::GetString(kind)).get(), + LOG("started %s device %p", dom::GetEnumString(kind).get(), device.get()); aHolder.Resolve(true, __func__); }) @@ -4182,9 +4172,7 @@ auto DeviceListener::UpdateDevice(bool aOn) -> RefPtr<DeviceOperationPromise> { } LOG("DeviceListener %p turning %s %s input device %s", this, aOn ? "on" : "off", - nsCString( - dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get(), + dom::GetEnumString(GetDevice()->Kind()).get(), NS_SUCCEEDED(aResult) ? "succeeded" : "failed"); if (NS_FAILED(aResult) && aResult != NS_ERROR_ABORT) { @@ -4217,8 +4205,7 @@ void DeviceListener::SetDeviceEnabled(bool aEnable) { LOG("DeviceListener %p %s %s device", this, aEnable ? "enabling" : "disabling", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); state.mTrackEnabled = aEnable; @@ -4274,9 +4261,7 @@ void DeviceListener::SetDeviceEnabled(bool aEnable) { LOG("DeviceListener %p %s %s device - starting device operation", this, aEnable ? "enabling" : "disabling", - nsCString( - dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); if (state.mStopped) { // Source was stopped between timer resolving and this runnable. @@ -4345,8 +4330,7 @@ void DeviceListener::SetDeviceMuted(bool aMute) { DeviceState& state = *mDeviceState; LOG("DeviceListener %p %s %s device", this, aMute ? "muting" : "unmuting", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); if (state.mStopped) { // Device terminally stopped. Updating device state is pointless. @@ -4360,8 +4344,7 @@ void DeviceListener::SetDeviceMuted(bool aMute) { LOG("DeviceListener %p %s %s device - starting device operation", this, aMute ? "muting" : "unmuting", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); state.mDeviceMuted = aMute; @@ -4467,9 +4450,7 @@ RefPtr<DeviceListener::DeviceListenerPromise> DeviceListener::ApplyConstraints( if (mStopped || mDeviceState->mStopped) { LOG("DeviceListener %p %s device applyConstraints, but device is stopped", - this, - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + this, dom::GetEnumString(GetDevice()->Kind()).get()); return DeviceListenerPromise::CreateAndResolve(false, __func__); } diff --git a/dom/media/MediaQueue.h b/dom/media/MediaQueue.h index 4609493339..9c4c829c87 100644 --- a/dom/media/MediaQueue.h +++ b/dom/media/MediaQueue.h @@ -25,6 +25,7 @@ extern LazyLogModule gMediaDecoderLog; class AudioData; class VideoData; +class EncodedFrame; template <typename T> struct TimestampAdjustmentTrait { @@ -46,13 +47,24 @@ struct NonTimestampAdjustmentTrait { static const bool mValue = !TimestampAdjustmentTrait<T>::mValue; }; +template <typename T> +struct DurationTypeTrait { + using type = media::TimeUnit; +}; + +template <> +struct DurationTypeTrait<EncodedFrame> { + using type = uint64_t; +}; + template <class T> class MediaQueue : private nsRefPtrDeque<T> { public: - MediaQueue() + explicit MediaQueue(bool aEnablePreciseDuration = false) : nsRefPtrDeque<T>(), mRecursiveMutex("mediaqueue"), - mEndOfStream(false) {} + mEndOfStream(false), + mEnablePreciseDuration(aEnablePreciseDuration) {} ~MediaQueue() { Reset(); } @@ -97,6 +109,7 @@ class MediaQueue : private nsRefPtrDeque<T> { AdjustTimeStampIfNeeded(aItem); } nsRefPtrDeque<T>::PushFront(aItem); + AddDurationToPreciseDuration(aItem); } inline void Push(T* aItem) { @@ -112,6 +125,7 @@ class MediaQueue : private nsRefPtrDeque<T> { MOZ_DIAGNOSTIC_ASSERT(item->GetEndTime() >= item->mTime); AdjustTimeStampIfNeeded(item); nsRefPtrDeque<T>::Push(dont_AddRef(item)); + AddDurationToPreciseDuration(item); mPushEvent.Notify(RefPtr<T>(item)); // Pushing new data after queue has ended means that the stream is active @@ -126,6 +140,7 @@ class MediaQueue : private nsRefPtrDeque<T> { RefPtr<T> rv = nsRefPtrDeque<T>::PopFront(); if (rv) { MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime); + SubtractDurationFromPreciseDuration(rv); mPopFrontEvent.Notify(RefPtr<T>(rv)); } return rv.forget(); @@ -133,7 +148,12 @@ class MediaQueue : private nsRefPtrDeque<T> { inline already_AddRefed<T> PopBack() { RecursiveMutexAutoLock lock(mRecursiveMutex); - return nsRefPtrDeque<T>::Pop(); + RefPtr<T> rv = nsRefPtrDeque<T>::Pop(); + if (rv) { + MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime); + SubtractDurationFromPreciseDuration(rv); + } + return rv.forget(); } inline RefPtr<T> PeekFront() const { @@ -151,6 +171,7 @@ class MediaQueue : private nsRefPtrDeque<T> { nsRefPtrDeque<T>::Erase(); SetOffset(media::TimeUnit::Zero()); mEndOfStream = false; + ResetPreciseDuration(); } bool AtEndOfStream() const { @@ -186,6 +207,12 @@ class MediaQueue : private nsRefPtrDeque<T> { return (last->GetEndTime() - first->mTime).ToMicroseconds(); } + // Return a precise duration if the feature is enabled. Otherwise, return -1. + int64_t PreciseDuration() const { + RecursiveMutexAutoLock lock(mRecursiveMutex); + return GetPreciseDuration(); + } + void LockedForEach(nsDequeFunctor<T>& aFunctor) const { RecursiveMutexAutoLock lock(mRecursiveMutex); nsRefPtrDeque<T>::ForEach(aFunctor); @@ -268,6 +295,59 @@ class MediaQueue : private nsRefPtrDeque<T> { // the media queue starts receiving looped data, which timestamp needs to be // modified. media::TimeUnit mOffset; + + inline void AddDurationToPreciseDuration(T* aItem) { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + media::TimeUnit> || + std::is_same_v<typename DurationTypeTrait<T>::type, + uint64_t>) { + mPreciseDuration += aItem->mDuration; + } + } + + inline void SubtractDurationFromPreciseDuration(T* aItem) { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + media::TimeUnit> || + std::is_same_v<typename DurationTypeTrait<T>::type, + uint64_t>) { + mPreciseDuration -= aItem->mDuration; + } + } + + inline void ResetPreciseDuration() { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + media::TimeUnit>) { + mPreciseDuration = media::TimeUnit::Zero(); + } else if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + uint64_t>) { + mPreciseDuration = 0; + } + } + + inline int64_t GetPreciseDuration() const { + if (mEnablePreciseDuration) { + if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + media::TimeUnit>) { + return mPreciseDuration.ToMicroseconds(); + } else if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type, + uint64_t>) { + return mPreciseDuration; + } + } + return -1; + } + + typename DurationTypeTrait<T>::type mPreciseDuration; + const bool mEnablePreciseDuration = false; }; } // namespace mozilla diff --git a/dom/media/MediaStreamTrack.cpp b/dom/media/MediaStreamTrack.cpp index 8ddb16d8e1..4235d96d13 100644 --- a/dom/media/MediaStreamTrack.cpp +++ b/dom/media/MediaStreamTrack.cpp @@ -324,7 +324,7 @@ void MediaStreamTrack::GetSettings(dom::MediaTrackSettings& aResult, } if (aResult.mFacingMode.WasPassed()) { aResult.mFacingMode.Value().AssignASCII( - VideoFacingModeEnumValues::GetString(VideoFacingModeEnum::User)); + GetEnumString(VideoFacingModeEnum::User)); } } diff --git a/dom/media/MediaTrackGraph.cpp b/dom/media/MediaTrackGraph.cpp index 157ad403d2..2af7aacb1f 100644 --- a/dom/media/MediaTrackGraph.cpp +++ b/dom/media/MediaTrackGraph.cpp @@ -1393,6 +1393,9 @@ void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { if (currentDeviceIndex == mOutputDevices.NoIndex) { // Outputs for this device have been removed. // Fall back to the primary output device. + LOG(LogLevel::Info, ("%p: No remaining outputs to device %p. " + "Switch to primary output device %p for AEC", + this, mOutputDeviceForAEC, PrimaryOutputDeviceID())); mOutputDeviceForAEC = PrimaryOutputDeviceID(); currentDeviceIndex = 0; MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC); @@ -1425,6 +1428,9 @@ void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { for (const auto& output : outputDeviceEntry.mTrackOutputs) { if (HasNonNullAudio(output)) { // Switch to this device. + LOG(LogLevel::Info, + ("%p: Switch output device for AEC from silent %p to non-null %p", + this, mOutputDeviceForAEC, outputDeviceEntry.mDeviceID)); mOutputDeviceForAEC = outputDeviceEntry.mDeviceID; return; } diff --git a/dom/media/PeerConnection.sys.mjs b/dom/media/PeerConnection.sys.mjs index 1bebde8159..00b4023c2f 100644 --- a/dom/media/PeerConnection.sys.mjs +++ b/dom/media/PeerConnection.sys.mjs @@ -24,7 +24,7 @@ const PC_COREQUEST_CID = Components.ID( "{74b2122d-65a8-4824-aa9e-3d664cb75dc2}" ); -function logMsg(msg, file, line, flag, winID) { +function logWebRTCMsg(msg, file, line, flag, win) { let scriptErrorClass = Cc["@mozilla.org/scripterror;1"]; let scriptError = scriptErrorClass.createInstance(Ci.nsIScriptError); scriptError.initWithWindowID( @@ -35,9 +35,14 @@ function logMsg(msg, file, line, flag, winID) { 0, flag, "content javascript", - winID + win.windowGlobalChild.innerWindowId ); Services.console.logMessage(scriptError); + if ( + Services.prefs.getBoolPref("media.peerconnection.treat_warnings_as_errors") + ) { + throw new win.TypeError(msg); + } } let setupPrototype = (_class, dict) => { @@ -100,7 +105,7 @@ export class GlobalPCList { if (this._list[winID] === undefined) { return; } - this._list[winID] = this._list[winID].filter(function (e, i, a) { + this._list[winID] = this._list[winID].filter(function (e) { return e.get() !== null; }); @@ -237,14 +242,12 @@ export class RTCSessionDescription { init(win) { this._win = win; this._winID = this._win.windowGlobalChild.innerWindowId; + this._legacyPref = Services.prefs.getBoolPref( + "media.peerconnection.description.legacy.enabled" + ); } __init({ type, sdp }) { - if (!type) { - throw new this._win.TypeError( - "Missing required 'type' member of RTCSessionDescriptionInit" - ); - } Object.assign(this, { _type: type, _sdp: sdp }); } @@ -252,6 +255,10 @@ export class RTCSessionDescription { return this._type; } set type(type) { + if (!this._legacyPref) { + // TODO: this throws even in sloppy mode. Remove in bug 1883992 + throw new this._win.TypeError("setting getter-only property type"); + } this.warn(); this._type = type; } @@ -260,6 +267,10 @@ export class RTCSessionDescription { return this._sdp; } set sdp(sdp) { + if (!this._legacyPref) { + // TODO: this throws even in sloppy mode. Remove in bug 1883992 + throw new this._win.TypeError("setting getter-only property sdp"); + } this.warn(); this._sdp = sdp; } @@ -267,23 +278,26 @@ export class RTCSessionDescription { warn() { if (!this._warned) { // Warn once per RTCSessionDescription about deprecated writable usage. - this.logWarning( - "RTCSessionDescription's members are readonly! " + - "Writing to them is deprecated and will break soon!" - ); + if (this._legacyPref) { + this.logMsg( + "RTCSessionDescription's members are readonly! " + + "Writing to them is deprecated and will break soon!", + Ci.nsIScriptError.warningFlag + ); + } else { + this.logMsg( + "RTCSessionDescription's members are readonly! " + + "Writing to them no longer works!", + Ci.nsIScriptError.errorFlag + ); + } this._warned = true; } } - logWarning(msg) { + logMsg(msg, flag) { let err = this._win.Error(); - logMsg( - msg, - err.fileName, - err.lineNumber, - Ci.nsIScriptError.warningFlag, - this._winID - ); + logWebRTCMsg(msg, err.fileName, err.lineNumber, flag, this._win); } } @@ -350,6 +364,13 @@ export class RTCPeerConnection { constructor() { this._pc = null; this._closed = false; + this._pendingLocalDescription = null; + this._pendingRemoteDescription = null; + this._currentLocalDescription = null; + this._currentRemoteDescription = null; + this._legacyPref = Services.prefs.getBoolPref( + "media.peerconnection.description.legacy.enabled" + ); // http://rtcweb-wg.github.io/jsep/#rfc.section.4.1.9 // canTrickle == null means unknown; when a remote description is received it @@ -723,7 +744,7 @@ export class RTCPeerConnection { } }; - var stunServers = 0; + let stunServers = 0; iceServers.forEach(({ urls, username, credential, credentialType }) => { if (!urls) { @@ -788,11 +809,7 @@ export class RTCPeerConnection { } if (stunServers >= 5) { this.logError( - "Using five or more STUN/TURN servers causes problems" - ); - } else if (stunServers > 2) { - this.logWarning( - "Using more than two STUN/TURN servers slows down discovery" + "Using five or more STUN/TURN servers slows down discovery" ); } }); @@ -861,7 +878,7 @@ export class RTCPeerConnection { } logMsg(msg, file, line, flag) { - return logMsg(msg, file, line, flag, this._winID); + return logWebRTCMsg(msg, file, line, flag, this._win); } getEH(type) { @@ -996,7 +1013,7 @@ export class RTCPeerConnection { return this._async(() => this._createAnswer(optionsOrOnSucc)); } - _createAnswer(options) { + _createAnswer() { this._checkClosed(); return this._chain(() => this._createAnAnswer()); } @@ -1192,11 +1209,6 @@ export class RTCPeerConnection { } _setRemoteDescription({ type, sdp }) { - if (!type) { - throw new this._win.TypeError( - "Missing required 'type' member of RTCSessionDescriptionInit" - ); - } if (type == "pranswer") { throw new this._win.DOMException( "pranswer not yet implemented", @@ -1541,24 +1553,36 @@ export class RTCPeerConnection { return this.pendingLocalDescription || this.currentLocalDescription; } + cacheDescription(name, type, sdp) { + if ( + !this[name] || + this[name].type != type || + this[name].sdp != sdp || + this._legacyPref + ) { + this[name] = sdp.length + ? new this._win.RTCSessionDescription({ type, sdp }) + : null; + } + return this[name]; + } + get currentLocalDescription() { this._checkClosed(); - const sdp = this._pc.currentLocalDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.currentOfferer ? "offer" : "answer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_currentLocalDescription", + this._pc.currentOfferer ? "offer" : "answer", + this._pc.currentLocalDescription + ); } get pendingLocalDescription() { this._checkClosed(); - const sdp = this._pc.pendingLocalDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.pendingOfferer ? "offer" : "answer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_pendingLocalDescription", + this._pc.pendingOfferer ? "offer" : "answer", + this._pc.pendingLocalDescription + ); } get remoteDescription() { @@ -1567,22 +1591,20 @@ export class RTCPeerConnection { get currentRemoteDescription() { this._checkClosed(); - const sdp = this._pc.currentRemoteDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.currentOfferer ? "answer" : "offer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_currentRemoteDescription", + this._pc.currentOfferer ? "answer" : "offer", + this._pc.currentRemoteDescription + ); } get pendingRemoteDescription() { this._checkClosed(); - const sdp = this._pc.pendingRemoteDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.pendingOfferer ? "answer" : "offer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_pendingRemoteDescription", + this._pc.pendingOfferer ? "answer" : "offer", + this._pc.pendingRemoteDescription + ); } get peerIdentity() { @@ -1918,8 +1940,7 @@ export class PeerConnectionObserver { switch (state) { case "IceConnectionState": - let connState = this._dompc._pc.iceConnectionState; - this.handleIceConnectionStateChange(connState); + this.handleIceConnectionStateChange(this._dompc._pc.iceConnectionState); break; case "IceGatheringState": diff --git a/dom/media/VideoFrameConverter.h b/dom/media/VideoFrameConverter.h index 36132c1b45..31b3104955 100644 --- a/dom/media/VideoFrameConverter.h +++ b/dom/media/VideoFrameConverter.h @@ -109,8 +109,8 @@ class VideoFrameConverter { // for processing so it can be immediately sent. mLastFrameQueuedForProcessing.mTime = time; - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod<StoreCopyPassByLRef<FrameToProcess>>( + MOZ_ALWAYS_SUCCEEDS( + mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>( "VideoFrameConverter::ProcessVideoFrame", this, &VideoFrameConverter::ProcessVideoFrame, mLastFrameQueuedForProcessing))); @@ -138,8 +138,8 @@ class VideoFrameConverter { mLastFrameQueuedForProcessing.mForceBlack = true; mLastFrameQueuedForProcessing.mImage = nullptr; - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod<StoreCopyPassByLRef<FrameToProcess>>( + MOZ_ALWAYS_SUCCEEDS( + mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>( "VideoFrameConverter::ProcessVideoFrame", this, &VideoFrameConverter::ProcessVideoFrame, mLastFrameQueuedForProcessing))); @@ -293,11 +293,10 @@ class VideoFrameConverter { return; } - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod<StoreCopyPassByLRef<FrameToProcess>>( - "VideoFrameConverter::ProcessVideoFrame", this, - &VideoFrameConverter::ProcessVideoFrame, - mLastFrameQueuedForProcessing))); + MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NewRunnableMethod<FrameToProcess>( + "VideoFrameConverter::ProcessVideoFrame", this, + &VideoFrameConverter::ProcessVideoFrame, + mLastFrameQueuedForProcessing))); } void ProcessVideoFrame(const FrameToProcess& aFrame) { @@ -364,7 +363,9 @@ class VideoFrameConverter { aFrame.mImage->AsPlanarYCbCrImage(); if (image) { dom::ImageUtils utils(image); - if (utils.GetFormat() == dom::ImageBitmapFormat::YUV420P && + Maybe<dom::ImageBitmapFormat> format = utils.GetFormat(); + if (format.isSome() && + format.value() == dom::ImageBitmapFormat::YUV420P && image->GetData()) { const layers::PlanarYCbCrData* data = image->GetData(); rtc::scoped_refptr<webrtc::I420BufferInterface> video_frame_buffer = diff --git a/dom/media/VideoUtils.cpp b/dom/media/VideoUtils.cpp index 279ce9b912..90057f7c83 100644 --- a/dom/media/VideoUtils.cpp +++ b/dom/media/VideoUtils.cpp @@ -4,7 +4,6 @@ #include "VideoUtils.h" -#include <functional> #include <stdint.h> #include "CubebUtils.h" @@ -19,7 +18,6 @@ #include "mozilla/StaticPrefs_accessibility.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/TaskQueue.h" -#include "mozilla/Telemetry.h" #include "nsCharSeparatedTokenizer.h" #include "nsContentTypeParser.h" #include "nsIConsoleService.h" @@ -29,7 +27,6 @@ #include "nsNetCID.h" #include "nsServiceManagerUtils.h" #include "nsThreadUtils.h" -#include "AudioStream.h" namespace mozilla { diff --git a/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js b/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js index 77ce4ddbc1..433e4fd51e 100644 --- a/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js +++ b/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js @@ -31,8 +31,8 @@ async function checkIsIframeVideoDocumentAutoplay(browser) { const iframe = content.document.createElement("iframe"); iframe.src = pageURL; content.document.body.appendChild(iframe); - const iframeLoaded = new Promise((resolve, reject) => { - iframe.addEventListener("load", e => resolve(), { once: true }); + const iframeLoaded = new Promise(resolve => { + iframe.addEventListener("load", () => resolve(), { once: true }); }); await iframeLoaded; return iframe.browsingContext; diff --git a/dom/media/autoplay/test/browser/head.js b/dom/media/autoplay/test/browser/head.js index c84850900a..6522104b7f 100644 --- a/dom/media/autoplay/test/browser/head.js +++ b/dom/media/autoplay/test/browser/head.js @@ -29,10 +29,10 @@ function loadAutoplayVideo(browser, args) { info("- create a new autoplay video -"); let video = content.document.createElement("video"); video.id = "v1"; - video.didPlayPromise = new Promise((resolve, reject) => { + video.didPlayPromise = new Promise(resolve => { video.addEventListener( "playing", - e => { + () => { video.didPlay = true; resolve(); }, @@ -40,7 +40,7 @@ function loadAutoplayVideo(browser, args) { ); video.addEventListener( "blocked", - e => { + () => { video.didPlay = false; resolve(); }, @@ -54,7 +54,7 @@ function loadAutoplayVideo(browser, args) { info("will call play() when reached loadedmetadata"); video.addEventListener( "loadedmetadata", - e => { + () => { video.play().then( () => { info("video play() resolved"); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html index e25b6401d1..9fcda2afe9 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html @@ -36,8 +36,8 @@ let x = eventNames.map( (eventName) => { - return new Promise(function (resolve, reject) { - window.addEventListener(eventName, async function (event) { + return new Promise(function (resolve) { + window.addEventListener(eventName, async function () { let p = await element.play().then(() => true, () => false); ok(p, "Expect to be activated already in " + eventName); resolve(); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html index b901df3324..fc8c47065e 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html @@ -91,7 +91,7 @@ input.blur(); } - async function testAutoplayKeyBlacklist(testCase, parent_window) { + async function testAutoplayKeyBlacklist() { let element = document.createElement("video"); element.preload = "auto"; element.src = "short.mp4"; diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html index 3594d0f236..dd358e6dc1 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html @@ -22,7 +22,7 @@ window.is = window.opener.is; window.info = window.opener.info; - async function testPlayBeforeLoadedMetata(testCase, parent_window) { + async function testPlayBeforeLoadedMetata(testCase) { info("testPlayBeforeLoadedMetata: " + testCase.resource); let element = document.createElement("video"); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html index 125ee156b6..f3b4e7da4d 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html @@ -22,8 +22,8 @@ window.is = window.opener.is; window.info = window.opener.info; - function testAutoplayUnmutePauses(testCase, parent_window) { - return new Promise(function (resolve, reject) { + function testAutoplayUnmutePauses(testCase) { + return new Promise(function (resolve) { info("testAutoplayUnmutePauses: " + testCase.property); diff --git a/dom/media/doctor/test/browser/browser_doctor_notification.js b/dom/media/doctor/test/browser/browser_doctor_notification.js index 5789622e23..6068304dfa 100644 --- a/dom/media/doctor/test/browser/browser_doctor_notification.js +++ b/dom/media/doctor/test/browser/browser_doctor_notification.js @@ -233,15 +233,11 @@ async function setFormatDiagnosticsReportForMimeType(tab, params) { async function setDecodeError(tab, params) { info(`start check for ${params.error}`); - await SpecialPowers.spawn( - tab.linkedBrowser, - [params], - async (params, shouldReportNotification) => { - const video = content.document.createElement("video"); - SpecialPowers.wrap(video).setDecodeError(params.error); - await content._waitForReport(params, params.shouldReportNotification); - } - ); + await SpecialPowers.spawn(tab.linkedBrowser, [params], async params => { + const video = content.document.createElement("video"); + SpecialPowers.wrap(video).setDecodeError(params.error); + await content._waitForReport(params, params.shouldReportNotification); + }); ok(true, `finished check for ${params.error}`); } diff --git a/dom/media/eme/EMEUtils.cpp b/dom/media/eme/EMEUtils.cpp index 294951e7e6..5a6b645df2 100644 --- a/dom/media/eme/EMEUtils.cpp +++ b/dom/media/eme/EMEUtils.cpp @@ -8,13 +8,13 @@ #include "jsfriendapi.h" #include "MediaData.h" +#include "KeySystemConfig.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/dom/KeySystemNames.h" #include "mozilla/dom/UnionTypes.h" #ifdef MOZ_WMF_CDM # include "mozilla/PMFCDM.h" -# include "KeySystemConfig.h" #endif namespace mozilla { @@ -143,22 +143,35 @@ const char* ToMediaKeyStatusStr(dom::MediaKeyStatus aStatus) { bool IsHardwareDecryptionSupported( const dom::MediaKeySystemConfiguration& aConfig) { - bool supportHardwareDecryption = false; for (const auto& capabilities : aConfig.mAudioCapabilities) { if (capabilities.mRobustness.EqualsLiteral("HW_SECURE_ALL")) { - supportHardwareDecryption = true; - break; + return true; } } for (const auto& capabilities : aConfig.mVideoCapabilities) { if (capabilities.mRobustness.EqualsLiteral("3000") || capabilities.mRobustness.EqualsLiteral("HW_SECURE_ALL") || capabilities.mRobustness.EqualsLiteral("HW_SECURE_DECODE")) { - supportHardwareDecryption = true; - break; + return true; + } + } + return false; +} + +bool IsHardwareDecryptionSupported(const KeySystemConfig& aConfig) { + for (const auto& robustness : aConfig.mAudioRobustness) { + if (robustness.EqualsLiteral("HW_SECURE_ALL")) { + return true; + } + } + for (const auto& robustness : aConfig.mVideoRobustness) { + if (robustness.EqualsLiteral("3000") || + robustness.EqualsLiteral("HW_SECURE_ALL") || + robustness.EqualsLiteral("HW_SECURE_DECODE")) { + return true; } } - return supportHardwareDecryption; + return false; } const char* EncryptionSchemeStr(const CryptoScheme& aScheme) { @@ -245,4 +258,16 @@ bool CheckIfHarewareDRMConfigExists( return foundHWDRMconfig; } +bool DoesKeySystemSupportHardwareDecryption(const nsAString& aKeySystem) { +#ifdef MOZ_WMF_CDM + if (aKeySystem.EqualsLiteral(kPlayReadyKeySystemHardware) || + aKeySystem.EqualsLiteral(kPlayReadyHardwareClearLeadKeySystemName) || + aKeySystem.EqualsLiteral(kWidevineExperimentKeySystemName) || + aKeySystem.EqualsLiteral(kWidevineExperiment2KeySystemName)) { + return true; + } +#endif + return false; +} + } // namespace mozilla diff --git a/dom/media/eme/EMEUtils.h b/dom/media/eme/EMEUtils.h index ca5c684a9f..3fbf22f359 100644 --- a/dom/media/eme/EMEUtils.h +++ b/dom/media/eme/EMEUtils.h @@ -89,6 +89,7 @@ const char* ToMediaKeyStatusStr(dom::MediaKeyStatus aStatus); // Return true if given config supports hardware decryption (SL3000 or L1). bool IsHardwareDecryptionSupported( const dom::MediaKeySystemConfiguration& aConfig); +bool IsHardwareDecryptionSupported(const KeySystemConfig& aConfig); const char* EncryptionSchemeStr(const CryptoScheme& aScheme); @@ -104,6 +105,8 @@ bool DoesKeySystemSupportClearLead(const nsAString& aKeySystem); bool CheckIfHarewareDRMConfigExists( const nsTArray<dom::MediaKeySystemConfiguration>& aConfigs); +bool DoesKeySystemSupportHardwareDecryption(const nsAString& aKeySystem); + } // namespace mozilla #endif // EME_LOG_H_ diff --git a/dom/media/eme/KeySystemConfig.cpp b/dom/media/eme/KeySystemConfig.cpp index b1b1f9ff63..0cb5da1a56 100644 --- a/dom/media/eme/KeySystemConfig.cpp +++ b/dom/media/eme/KeySystemConfig.cpp @@ -57,9 +57,10 @@ bool KeySystemConfig::Supports(const nsAString& aKeySystem) { {nsCString(kWidevineExperimentKeySystemName)}); } - if ((IsPlayReadyKeySystemAndSupported(aKeySystem) || - IsWMFClearKeySystemAndSupported(aKeySystem)) && - WMFCDMImpl::Supports(aKeySystem)) { + // PlayReady and WMF-based ClearKey are always installed, we don't need to + // download them. + if (IsPlayReadyKeySystemAndSupported(aKeySystem) || + IsWMFClearKeySystemAndSupported(aKeySystem)) { return true; } #endif @@ -69,7 +70,8 @@ bool KeySystemConfig::Supports(const nsAString& aKeySystem) { /* static */ bool KeySystemConfig::CreateKeySystemConfigs( - const nsAString& aKeySystem, nsTArray<KeySystemConfig>& aOutConfigs) { + const nsAString& aKeySystem, const DecryptionInfo aDecryption, + nsTArray<KeySystemConfig>& aOutConfigs) { if (!Supports(aKeySystem)) { return false; } @@ -103,10 +105,16 @@ bool KeySystemConfig::CreateKeySystemConfigs( config->mMP4.SetCanDecrypt(EME_CODEC_FLAC); config->mMP4.SetCanDecrypt(EME_CODEC_OPUS); config->mMP4.SetCanDecrypt(EME_CODEC_VP9); +#ifdef MOZ_AV1 + config->mMP4.SetCanDecrypt(EME_CODEC_AV1); +#endif config->mWebM.SetCanDecrypt(EME_CODEC_VORBIS); config->mWebM.SetCanDecrypt(EME_CODEC_OPUS); config->mWebM.SetCanDecrypt(EME_CODEC_VP8); config->mWebM.SetCanDecrypt(EME_CODEC_VP9); +#ifdef MOZ_AV1 + config->mWebM.SetCanDecrypt(EME_CODEC_AV1); +#endif if (StaticPrefs::media_clearkey_test_key_systems_enabled()) { // Add testing key systems. These offer the same capabilities as the @@ -156,6 +164,10 @@ bool KeySystemConfig::CreateKeySystemConfigs( &config->mMP4}, {nsCString(VIDEO_MP4), EME_CODEC_VP9, java::MediaDrmProxy::AVC, &config->mMP4}, +# ifdef MOZ_AV1 + {nsCString(VIDEO_MP4), EME_CODEC_AV1, java::MediaDrmProxy::AV1, + &config->mMP4}, +# endif {nsCString(AUDIO_MP4), EME_CODEC_AAC, java::MediaDrmProxy::AAC, &config->mMP4}, {nsCString(AUDIO_MP4), EME_CODEC_FLAC, java::MediaDrmProxy::FLAC, @@ -166,6 +178,10 @@ bool KeySystemConfig::CreateKeySystemConfigs( &config->mWebM}, {nsCString(VIDEO_WEBM), EME_CODEC_VP9, java::MediaDrmProxy::VP9, &config->mWebM}, +# ifdef MOZ_AV1 + {nsCString(VIDEO_WEBM), EME_CODEC_AV1, java::MediaDrmProxy::AV1, + &config->mWebM}, +# endif {nsCString(AUDIO_WEBM), EME_CODEC_VORBIS, java::MediaDrmProxy::VORBIS, &config->mWebM}, {nsCString(AUDIO_WEBM), EME_CODEC_OPUS, java::MediaDrmProxy::OPUS, @@ -198,10 +214,16 @@ bool KeySystemConfig::CreateKeySystemConfigs( config->mMP4.SetCanDecrypt(EME_CODEC_OPUS); config->mMP4.SetCanDecryptAndDecode(EME_CODEC_H264); config->mMP4.SetCanDecryptAndDecode(EME_CODEC_VP9); +# ifdef MOZ_AV1 + config->mMP4.SetCanDecryptAndDecode(EME_CODEC_AV1); +# endif config->mWebM.SetCanDecrypt(EME_CODEC_VORBIS); config->mWebM.SetCanDecrypt(EME_CODEC_OPUS); config->mWebM.SetCanDecryptAndDecode(EME_CODEC_VP8); config->mWebM.SetCanDecryptAndDecode(EME_CODEC_VP9); +# ifdef MOZ_AV1 + config->mWebM.SetCanDecryptAndDecode(EME_CODEC_AV1); +# endif #endif return true; } @@ -209,7 +231,8 @@ bool KeySystemConfig::CreateKeySystemConfigs( if (IsPlayReadyKeySystemAndSupported(aKeySystem) || IsWidevineExperimentKeySystemAndSupported(aKeySystem)) { RefPtr<WMFCDMImpl> cdm = MakeRefPtr<WMFCDMImpl>(aKeySystem); - return cdm->GetCapabilities(aOutConfigs); + return cdm->GetCapabilities(aDecryption == DecryptionInfo::Hardware, + aOutConfigs); } #endif return false; @@ -243,7 +266,9 @@ void KeySystemConfig::GetGMPKeySystemConfigs(dom::Promise* aPromise) { continue; } #endif - if (KeySystemConfig::CreateKeySystemConfigs(name, keySystemConfigs)) { + if (KeySystemConfig::CreateKeySystemConfigs( + name, KeySystemConfig::DecryptionInfo::Software, + keySystemConfigs)) { auto* info = cdmInfo.AppendElement(fallible); if (!info) { aPromise->MaybeReject(NS_ERROR_OUT_OF_MEMORY); diff --git a/dom/media/eme/KeySystemConfig.h b/dom/media/eme/KeySystemConfig.h index 8bd7d98217..cc35ba76de 100644 --- a/dom/media/eme/KeySystemConfig.h +++ b/dom/media/eme/KeySystemConfig.h @@ -36,6 +36,7 @@ struct KeySystemConfig { static constexpr auto EME_CODEC_VORBIS = "vorbis"_ns; static constexpr auto EME_CODEC_FLAC = "flac"_ns; static constexpr auto EME_CODEC_H264 = "h264"_ns; + static constexpr auto EME_CODEC_AV1 = "av1"_ns; static constexpr auto EME_CODEC_VP8 = "vp8"_ns; static constexpr auto EME_CODEC_VP9 = "vp9"_ns; static constexpr auto EME_CODEC_HEVC = "hevc"_ns; @@ -123,7 +124,13 @@ struct KeySystemConfig { // Return true if given key system is supported on the current device. static bool Supports(const nsAString& aKeySystem); + + enum class DecryptionInfo : uint8_t { + Software, + Hardware, + }; static bool CreateKeySystemConfigs(const nsAString& aKeySystem, + const DecryptionInfo aDecryption, nsTArray<KeySystemConfig>& aOutConfigs); static void GetGMPKeySystemConfigs(dom::Promise* aPromise); diff --git a/dom/media/eme/MediaKeySession.cpp b/dom/media/eme/MediaKeySession.cpp index 908df5f7c8..8a3a01dd5c 100644 --- a/dom/media/eme/MediaKeySession.cpp +++ b/dom/media/eme/MediaKeySession.cpp @@ -52,7 +52,7 @@ static const uint32_t MAX_CENC_INIT_DATA_LENGTH = 64 * 1024; MediaKeySession::MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, const nsAString& aKeySystem, MediaKeySessionType aSessionType, - ErrorResult& aRv) + bool aHardwareDecryption, ErrorResult& aRv) : DOMEventTargetHelper(aParent), mKeys(aKeys), mKeySystem(aKeySystem), @@ -61,7 +61,8 @@ MediaKeySession::MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, mIsClosed(false), mUninitialized(true), mKeyStatusMap(new MediaKeyStatusMap(aParent)), - mExpiration(JS::GenericNaN()) { + mExpiration(JS::GenericNaN()), + mHardwareDecryption(aHardwareDecryption) { EME_LOG("MediaKeySession[%p,''] ctor", this); MOZ_ASSERT(aParent); @@ -123,9 +124,8 @@ void MediaKeySession::UpdateKeyStatusMap() { nsPrintfCString("MediaKeySession[%p,'%s'] key statuses change {", this, NS_ConvertUTF16toUTF8(mSessionId).get())); for (const CDMCaps::KeyStatus& status : keyStatuses) { - message.AppendPrintf( - " (%s,%s)", ToHexString(status.mId).get(), - nsCString(MediaKeyStatusValues::GetString(status.mStatus)).get()); + message.AppendPrintf(" (%s,%s)", ToHexString(status.mId).get(), + GetEnumString(status.mStatus).get()); } message.AppendLiteral(" }"); // Use %s so we aren't exposing random strings to printf interpolation. @@ -250,8 +250,8 @@ already_AddRefed<Promise> MediaKeySession::GenerateRequest( // cdm implementation value does not support initDataType as an // Initialization Data Type, return a promise rejected with a // NotSupportedError. String comparison is case-sensitive. - if (!MediaKeySystemAccess::KeySystemSupportsInitDataType(mKeySystem, - aInitDataType)) { + if (!MediaKeySystemAccess::KeySystemSupportsInitDataType( + mKeySystem, aInitDataType, mHardwareDecryption)) { promise->MaybeRejectWithNotSupportedError( "Unsupported initDataType passed to MediaKeySession.generateRequest()"); EME_LOG( @@ -542,8 +542,7 @@ void MediaKeySession::DispatchKeyMessage(MediaKeyMessageType aMessageType, EME_LOG( "MediaKeySession[%p,'%s'] DispatchKeyMessage() type=%s message='%s'", this, NS_ConvertUTF16toUTF8(mSessionId).get(), - nsCString(MediaKeyMessageTypeValues::GetString(aMessageType)).get(), - ToHexString(aMessage).get()); + GetEnumString(aMessageType).get(), ToHexString(aMessage).get()); } RefPtr<MediaKeyMessageEvent> event( @@ -611,12 +610,8 @@ void MediaKeySession::SetOnmessage(EventHandlerNonNull* aCallback) { SetEventHandler(nsGkAtoms::onmessage, aCallback); } -nsCString ToCString(MediaKeySessionType aType) { - return nsCString(MediaKeySessionTypeValues::GetString(aType)); -} - nsString ToString(MediaKeySessionType aType) { - return NS_ConvertUTF8toUTF16(ToCString(aType)); + return NS_ConvertUTF8toUTF16(GetEnumString(aType)); } } // namespace mozilla::dom diff --git a/dom/media/eme/MediaKeySession.h b/dom/media/eme/MediaKeySession.h index e19488c311..7204f99eef 100644 --- a/dom/media/eme/MediaKeySession.h +++ b/dom/media/eme/MediaKeySession.h @@ -36,8 +36,6 @@ class ArrayBufferViewOrArrayBuffer; class MediaKeyError; class MediaKeyStatusMap; -nsCString ToCString(MediaKeySessionType aType); - nsString ToString(MediaKeySessionType aType); class MediaKeySession final : public DOMEventTargetHelper, @@ -49,7 +47,7 @@ class MediaKeySession final : public DOMEventTargetHelper, public: MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, const nsAString& aKeySystem, MediaKeySessionType aSessionType, - ErrorResult& aRv); + bool aHardwareDecryption, ErrorResult& aRv); void SetSessionId(const nsAString& aSessionId); @@ -134,6 +132,9 @@ class MediaKeySession final : public DOMEventTargetHelper, bool mUninitialized; RefPtr<MediaKeyStatusMap> mKeyStatusMap; double mExpiration; + + // True if this key session is related with hardware decryption. + bool mHardwareDecryption; }; } // namespace dom diff --git a/dom/media/eme/MediaKeySystemAccess.cpp b/dom/media/eme/MediaKeySystemAccess.cpp index b58ff76424..d498c2a773 100644 --- a/dom/media/eme/MediaKeySystemAccess.cpp +++ b/dom/media/eme/MediaKeySystemAccess.cpp @@ -214,6 +214,9 @@ static KeySystemConfig::EMECodecString ToEMEAPICodecString( if (IsH264CodecString(aCodec)) { return KeySystemConfig::EME_CODEC_H264; } + if (IsAV1CodecString(aCodec)) { + return KeySystemConfig::EME_CODEC_AV1; + } if (IsVP8CodecString(aCodec)) { return KeySystemConfig::EME_CODEC_VP8; } @@ -228,31 +231,53 @@ static KeySystemConfig::EMECodecString ToEMEAPICodecString( return ""_ns; } -static nsTArray<KeySystemConfig> GetSupportedKeySystems() { +static nsTArray<KeySystemConfig> GetSupportedKeySystems( + const nsAString& aKeySystem, bool aIsHardwareDecryption) { + using DecryptionInfo = KeySystemConfig::DecryptionInfo; nsTArray<KeySystemConfig> keySystemConfigs; - - const nsTArray<nsString> keySystemNames{ - NS_ConvertUTF8toUTF16(kClearKeyKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineKeySystemName), + if (IsWidevineKeySystem(aKeySystem) || IsClearkeyKeySystem(aKeySystem)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + aKeySystem, DecryptionInfo::Software, keySystemConfigs); + } #ifdef MOZ_WMF_CDM - NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), - NS_ConvertUTF8toUTF16(kPlayReadyKeySystemHardware), - NS_ConvertUTF8toUTF16(kPlayReadyHardwareClearLeadKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineExperimentKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineExperiment2KeySystemName), -#endif - }; - for (const auto& name : keySystemNames) { - Unused << KeySystemConfig::CreateKeySystemConfigs(name, keySystemConfigs); + if (IsPlayReadyKeySystem(aKeySystem)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), + DecryptionInfo::Software, keySystemConfigs); + if (aIsHardwareDecryption) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemHardware), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyHardwareClearLeadKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + } + } + // If key system is kWidevineKeySystemName but with hardware decryption + // requirement, then we need to check those experiement key systems which are + // used for hardware decryption. + if (IsWidevineExperimentKeySystem(aKeySystem) || + (IsWidevineKeySystem(aKeySystem) && aIsHardwareDecryption)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kWidevineExperimentKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kWidevineExperiment2KeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); } +#endif return keySystemConfigs; } static bool GetKeySystemConfigs( - const nsAString& aKeySystem, + const nsAString& aKeySystem, bool aIsHardwareDecryption, nsTArray<KeySystemConfig>& aOutKeySystemConfig) { bool foundConfigs = false; - for (auto& config : GetSupportedKeySystems()) { + for (auto& config : + GetSupportedKeySystems(aKeySystem, aIsHardwareDecryption)) { if (config.IsSameKeySystem(aKeySystem)) { aOutKeySystemConfig.AppendElement(std::move(config)); foundConfigs = true; @@ -263,9 +288,10 @@ static bool GetKeySystemConfigs( /* static */ bool MediaKeySystemAccess::KeySystemSupportsInitDataType( - const nsAString& aKeySystem, const nsAString& aInitDataType) { + const nsAString& aKeySystem, const nsAString& aInitDataType, + bool aIsHardwareDecryption) { nsTArray<KeySystemConfig> implementations; - GetKeySystemConfigs(aKeySystem, implementations); + GetKeySystemConfigs(aKeySystem, aIsHardwareDecryption, implementations); bool containInitType = false; for (const auto& config : implementations) { if (config.mInitDataTypes.Contains(aInitDataType)) { @@ -352,15 +378,13 @@ static bool SupportsEncryptionScheme( static bool ToSessionType(const nsAString& aSessionType, MediaKeySessionType& aOutType) { - if (aSessionType.Equals(ToString(MediaKeySessionType::Temporary))) { - aOutType = MediaKeySessionType::Temporary; - return true; - } - if (aSessionType.Equals(ToString(MediaKeySessionType::Persistent_license))) { - aOutType = MediaKeySessionType::Persistent_license; - return true; + Maybe<MediaKeySessionType> type = + StringToEnum<MediaKeySessionType>(aSessionType); + if (type.isNothing()) { + return false; } - return false; + aOutType = type.value(); + return true; } // 5.1.1 Is persistent session type? @@ -395,6 +419,7 @@ static CodecType GetCodecType(const KeySystemConfig::EMECodecString& aCodec) { return Audio; } if (aCodec.Equals(KeySystemConfig::EME_CODEC_H264) || + aCodec.Equals(KeySystemConfig::EME_CODEC_AV1) || aCodec.Equals(KeySystemConfig::EME_CODEC_VP8) || aCodec.Equals(KeySystemConfig::EME_CODEC_VP9) || aCodec.Equals(KeySystemConfig::EME_CODEC_HEVC)) { @@ -1040,7 +1065,11 @@ bool MediaKeySystemAccess::GetSupportedConfig( DecoderDoctorDiagnostics* aDiagnostics, bool aIsPrivateBrowsing, const std::function<void(const char*)>& aDeprecationLogFn) { nsTArray<KeySystemConfig> implementations; - if (!GetKeySystemConfigs(aKeySystem, implementations)) { + const bool isHardwareDecryptionRequest = + CheckIfHarewareDRMConfigExists(aConfigs) || + DoesKeySystemSupportHardwareDecryption(aKeySystem); + if (!GetKeySystemConfigs(aKeySystem, isHardwareDecryptionRequest, + implementations)) { return false; } for (const auto& implementation : implementations) { @@ -1082,7 +1111,7 @@ static nsCString ToCString(const nsString& aString) { static nsCString ToCString(const MediaKeysRequirement aValue) { nsCString str("'"); - str.AppendASCII(MediaKeysRequirementValues::GetString(aValue)); + str.AppendASCII(GetEnumString(aValue)); str.AppendLiteral("'"); return str; } diff --git a/dom/media/eme/MediaKeySystemAccess.h b/dom/media/eme/MediaKeySystemAccess.h index 954ff7adf3..18eec47008 100644 --- a/dom/media/eme/MediaKeySystemAccess.h +++ b/dom/media/eme/MediaKeySystemAccess.h @@ -67,7 +67,8 @@ class MediaKeySystemAccess final : public nsISupports, public nsWrapperCache { const std::function<void(const char*)>& aDeprecationLogFn); static bool KeySystemSupportsInitDataType(const nsAString& aKeySystem, - const nsAString& aInitDataType); + const nsAString& aInitDataType, + bool aIsHardwareDecryption); static nsCString ToCString( const Sequence<MediaKeySystemConfiguration>& aConfig); diff --git a/dom/media/eme/MediaKeySystemAccessManager.cpp b/dom/media/eme/MediaKeySystemAccessManager.cpp index 2bc12d57d7..8ebe7ceee7 100644 --- a/dom/media/eme/MediaKeySystemAccessManager.cpp +++ b/dom/media/eme/MediaKeySystemAccessManager.cpp @@ -412,8 +412,7 @@ void MediaKeySystemAccessManager::RequestMediaKeySystemAccess( "MediaKeySystemAccess::GetKeySystemStatus(%s) " "result=%s msg='%s'", NS_ConvertUTF16toUTF8(aRequest->mKeySystem).get(), - nsCString(MediaKeySystemStatusValues::GetString(status)).get(), - message.get()); + GetEnumString(status).get(), message.get()); LogToBrowserConsole(NS_ConvertUTF8toUTF16(msg)); EME_LOG("%s", msg.get()); diff --git a/dom/media/eme/MediaKeys.cpp b/dom/media/eme/MediaKeys.cpp index c4340885a3..9a3f4bba7a 100644 --- a/dom/media/eme/MediaKeys.cpp +++ b/dom/media/eme/MediaKeys.cpp @@ -23,7 +23,6 @@ #include "mozilla/dom/UnionTypes.h" #include "mozilla/dom/WindowContext.h" #include "mozilla/dom/WindowGlobalChild.h" -#include "nsContentCID.h" #include "nsContentTypeParser.h" #include "nsContentUtils.h" #include "nsIScriptObjectPrincipal.h" @@ -429,7 +428,8 @@ class MediaKeysGMPCrashHelper : public GMPCrashHelper { already_AddRefed<CDMProxy> MediaKeys::CreateCDMProxy() { const bool isHardwareDecryptionSupported = - IsHardwareDecryptionSupported(mConfig); + IsHardwareDecryptionSupported(mConfig) || + DoesKeySystemSupportHardwareDecryption(mKeySystem); EME_LOG("MediaKeys[%p]::CreateCDMProxy(), isHardwareDecryptionSupported=%d", this, isHardwareDecryptionSupported); RefPtr<CDMProxy> proxy; @@ -662,8 +662,12 @@ already_AddRefed<MediaKeySession> MediaKeys::CreateSession( EME_LOG("MediaKeys[%p] Creating session", this); - RefPtr<MediaKeySession> session = new MediaKeySession( - GetParentObject(), this, mKeySystem, aSessionType, aRv); + const bool isHardwareDecryption = + IsHardwareDecryptionSupported(mConfig) || + DoesKeySystemSupportHardwareDecryption(mKeySystem); + RefPtr<MediaKeySession> session = + new MediaKeySession(GetParentObject(), this, mKeySystem, aSessionType, + isHardwareDecryption, aRv); if (aRv.Failed()) { return nullptr; @@ -792,8 +796,7 @@ void MediaKeys::GetSessionsInfo(nsString& sessionsInfo) { sessionsInfo.AppendLiteral("(kid="); sessionsInfo.Append(keyID); sessionsInfo.AppendLiteral(" status="); - sessionsInfo.AppendASCII( - MediaKeyStatusValues::GetString(keyStatusMap->GetValueAtIndex(i))); + sessionsInfo.AppendASCII(GetEnumString(keyStatusMap->GetValueAtIndex(i))); sessionsInfo.AppendLiteral(")"); } sessionsInfo.AppendLiteral(")"); @@ -824,7 +827,7 @@ already_AddRefed<Promise> MediaKeys::GetStatusForPolicy( } EME_LOG("GetStatusForPolicy minHdcpVersion = %s.", - HDCPVersionValues::GetString(aPolicy.mMinHdcpVersion.Value()).data()); + GetEnumString(aPolicy.mMinHdcpVersion.Value()).get()); mProxy->GetStatusForPolicy(StorePromise(promise), aPolicy.mMinHdcpVersion.Value()); return promise.forget(); diff --git a/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp b/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp index e5431f50fd..f3cfb6c026 100644 --- a/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp +++ b/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp @@ -9,7 +9,6 @@ #include "nsString.h" #include "mozilla/dom/MediaKeys.h" #include "mozilla/dom/MediaKeySession.h" -#include "nsContentCID.h" #include "nsServiceManagerUtils.h" #include "MainThreadUtils.h" #include "mozilla/EMEUtils.h" diff --git a/dom/media/eme/mediafoundation/WMFCDMImpl.cpp b/dom/media/eme/mediafoundation/WMFCDMImpl.cpp index 1fe42aa8e2..add978f755 100644 --- a/dom/media/eme/mediafoundation/WMFCDMImpl.cpp +++ b/dom/media/eme/mediafoundation/WMFCDMImpl.cpp @@ -15,37 +15,8 @@ namespace mozilla { -/* static */ -bool WMFCDMImpl::Supports(const nsAString& aKeySystem) { - MOZ_ASSERT(NS_IsMainThread()); - if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { - return false; - } - - static std::map<nsString, bool> sSupports; - static bool sSetRunOnShutdown = false; - if (!sSetRunOnShutdown) { - GetMainThreadSerialEventTarget()->Dispatch( - NS_NewRunnableFunction("WMFCDMImpl::Supports", [&] { - RunOnShutdown([&] { sSupports.clear(); }, - ShutdownPhase::XPCOMShutdown); - })); - sSetRunOnShutdown = true; - } - - nsString key(aKeySystem); - if (const auto& s = sSupports.find(key); s != sSupports.end()) { - return s->second; - } - - RefPtr<WMFCDMImpl> cdm = MakeRefPtr<WMFCDMImpl>(aKeySystem); - nsTArray<KeySystemConfig> configs; - bool s = cdm->GetCapabilities(configs); - sSupports[key] = s; - return s; -} - -bool WMFCDMImpl::GetCapabilities(nsTArray<KeySystemConfig>& aOutConfigs) { +bool WMFCDMImpl::GetCapabilities(bool aIsHardwareDecryption, + nsTArray<KeySystemConfig>& aOutConfigs) { MOZ_ASSERT(NS_IsMainThread()); if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { return false; @@ -67,13 +38,14 @@ bool WMFCDMImpl::GetCapabilities(nsTArray<KeySystemConfig>& aOutConfigs) { auto keySystem = std::string{NS_ConvertUTF16toUTF8(mKeySystem).get()}; if (auto rv = sKeySystemConfigs.find(keySystem); rv != sKeySystemConfigs.end()) { - EME_LOG("Return cached capabilities for %s", keySystem.c_str()); for (const auto& config : rv->second) { - aOutConfigs.AppendElement(config); - EME_LOG("-- capabilities (%s)", - NS_ConvertUTF16toUTF8(config.GetDebugInfo()).get()); + if (IsHardwareDecryptionSupported(config) == aIsHardwareDecryption) { + EME_LOG("Return cached capabilities for %s (%s)", keySystem.c_str(), + NS_ConvertUTF16toUTF8(config.GetDebugInfo()).get()); + aOutConfigs.AppendElement(config); + return true; + } } - return true; } // Not cached result, ask the remote process. @@ -83,42 +55,33 @@ bool WMFCDMImpl::GetCapabilities(nsTArray<KeySystemConfig>& aOutConfigs) { mCDM = MakeRefPtr<MFCDMChild>(mKeySystem); } bool ok = false; - static const bool sIsHwSecure[2] = {false, true}; - for (const auto& isHWSecure : sIsHwSecure) { - media::Await( - do_AddRef(backgroundTaskQueue), mCDM->GetCapabilities(isHWSecure), - [&ok, &aOutConfigs, keySystem, - isHWSecure](const MFCDMCapabilitiesIPDL& capabilities) { - EME_LOG("capabilities: keySystem=%s (hw-secure=%d)", - keySystem.c_str(), isHWSecure); - for (const auto& v : capabilities.videoCapabilities()) { - EME_LOG("capabilities: video=%s", - NS_ConvertUTF16toUTF8(v.contentType()).get()); - } - for (const auto& a : capabilities.audioCapabilities()) { - EME_LOG("capabilities: audio=%s", - NS_ConvertUTF16toUTF8(a.contentType()).get()); - } - for (const auto& v : capabilities.encryptionSchemes()) { - EME_LOG("capabilities: encryptionScheme=%s", - EncryptionSchemeStr(v)); - } - KeySystemConfig* config = aOutConfigs.AppendElement(); - MFCDMCapabilitiesIPDLToKeySystemConfig(capabilities, *config); - sKeySystemConfigs[keySystem].AppendElement(*config); - // This is equal to "com.microsoft.playready.recommendation.3000", so - // we can store it directly without asking the remote process again. - if (keySystem.compare(kPlayReadyKeySystemName) == 0 && isHWSecure) { - config->mKeySystem.AssignLiteral(kPlayReadyKeySystemHardware); - sKeySystemConfigs["com.microsoft.playready.recommendation.3000"] - .AppendElement(*config); - } - ok = true; - }, - [](nsresult rv) { - EME_LOG("Fail to get key system capabilities. rv=%x", uint32_t(rv)); - }); - } + media::Await( + do_AddRef(backgroundTaskQueue), + mCDM->GetCapabilities(aIsHardwareDecryption), + [&ok, &aOutConfigs, keySystem, + aIsHardwareDecryption](const MFCDMCapabilitiesIPDL& capabilities) { + EME_LOG("capabilities: keySystem=%s (hw-secure=%d)", keySystem.c_str(), + aIsHardwareDecryption); + for (const auto& v : capabilities.videoCapabilities()) { + EME_LOG("capabilities: video=%s", + NS_ConvertUTF16toUTF8(v.contentType()).get()); + } + for (const auto& a : capabilities.audioCapabilities()) { + EME_LOG("capabilities: audio=%s", + NS_ConvertUTF16toUTF8(a.contentType()).get()); + } + for (const auto& v : capabilities.encryptionSchemes()) { + EME_LOG("capabilities: encryptionScheme=%s", EncryptionSchemeStr(v)); + } + KeySystemConfig* config = aOutConfigs.AppendElement(); + MFCDMCapabilitiesIPDLToKeySystemConfig(capabilities, *config); + sKeySystemConfigs[keySystem].AppendElement(*config); + ok = true; + }, + [](nsresult rv) { + EME_LOG("Fail to get key system capabilities. rv=%x", uint32_t(rv)); + }); + return ok; } diff --git a/dom/media/eme/mediafoundation/WMFCDMImpl.h b/dom/media/eme/mediafoundation/WMFCDMImpl.h index 452629ec84..b7e6308848 100644 --- a/dom/media/eme/mediafoundation/WMFCDMImpl.h +++ b/dom/media/eme/mediafoundation/WMFCDMImpl.h @@ -34,9 +34,9 @@ class WMFCDMImpl final { explicit WMFCDMImpl(const nsAString& aKeySystem) : mKeySystem(aKeySystem) {} - static bool Supports(const nsAString& aKeySystem); // TODO: make this async? - bool GetCapabilities(nsTArray<KeySystemConfig>& aOutConfigs); + bool GetCapabilities(bool aIsHardwareDecryption, + nsTArray<KeySystemConfig>& aOutConfigs); using InitPromise = GenericPromise; struct InitParams { diff --git a/dom/media/eme/mediafoundation/WMFCDMProxy.cpp b/dom/media/eme/mediafoundation/WMFCDMProxy.cpp index 21207ecc22..f7e05dfb6a 100644 --- a/dom/media/eme/mediafoundation/WMFCDMProxy.cpp +++ b/dom/media/eme/mediafoundation/WMFCDMProxy.cpp @@ -381,8 +381,7 @@ void WMFCDMProxy::GetStatusForPolicy(PromiseId aPromiseId, RETURN_IF_SHUTDOWN(); EME_LOG("WMFCDMProxy::GetStatusForPolicy(this=%p, pid=%" PRIu32 ", minHDCP=%s)", - this, aPromiseId, - dom::HDCPVersionValues::GetString(aMinHdcpVersion).data()); + this, aPromiseId, dom::GetEnumString(aMinHdcpVersion).get()); mCDM->GetStatusForPolicy(aPromiseId, aMinHdcpVersion) ->Then( mMainThread, __func__, diff --git a/dom/media/flac/FlacDecoder.cpp b/dom/media/flac/FlacDecoder.cpp index 2f205c9aae..07d9f10983 100644 --- a/dom/media/flac/FlacDecoder.cpp +++ b/dom/media/flac/FlacDecoder.cpp @@ -11,13 +11,7 @@ namespace mozilla { /* static */ -bool FlacDecoder::IsEnabled() { -#ifdef MOZ_FFVPX - return StaticPrefs::media_flac_enabled(); -#else - return false; -#endif -} +bool FlacDecoder::IsEnabled() { return StaticPrefs::media_flac_enabled(); } /* static */ bool FlacDecoder::IsSupportedType(const MediaContainerType& aContainerType) { diff --git a/dom/media/gmp/ChromiumCDMProxy.cpp b/dom/media/gmp/ChromiumCDMProxy.cpp index e8d871a113..566b386b0b 100644 --- a/dom/media/gmp/ChromiumCDMProxy.cpp +++ b/dom/media/gmp/ChromiumCDMProxy.cpp @@ -605,8 +605,7 @@ void ChromiumCDMProxy::GetStatusForPolicy( MOZ_ASSERT(NS_IsMainThread()); EME_LOG("ChromiumCDMProxy::GetStatusForPolicy(this=%p, pid=%" PRIu32 ") minHdcpVersion=%s", - this, aPromiseId, - dom::HDCPVersionValues::GetString(aMinHdcpVersion).data()); + this, aPromiseId, dom::GetEnumString(aMinHdcpVersion).get()); RefPtr<gmp::ChromiumCDMParent> cdm = GetCDMParent(); if (!cdm) { diff --git a/dom/media/gmp/GMPChild.cpp b/dom/media/gmp/GMPChild.cpp index 1eea8b108b..d543d46387 100644 --- a/dom/media/gmp/GMPChild.cpp +++ b/dom/media/gmp/GMPChild.cpp @@ -237,16 +237,15 @@ mozilla::ipc::IPCResult GMPChild::RecvPreloadLibs(const nsCString& aLibs) { bool GMPChild::GetUTF8LibPath(nsACString& aOutLibPath) { nsCOMPtr<nsIFile> libFile; -#define GMP_PATH_CRASH(explain) \ - do { \ - nsAutoString path; \ - if (!libFile || NS_FAILED(libFile->GetPath(path))) { \ - path = mPluginPath; \ - } \ - CrashReporter::AnnotateCrashReport( \ - CrashReporter::Annotation::GMPLibraryPath, \ - NS_ConvertUTF16toUTF8(path)); \ - MOZ_CRASH(explain); \ +#define GMP_PATH_CRASH(explain) \ + do { \ + nsAutoString path; \ + if (!libFile || NS_FAILED(libFile->GetPath(path))) { \ + path = mPluginPath; \ + } \ + CrashReporter::RecordAnnotationNSString( \ + CrashReporter::Annotation::GMPLibraryPath, path); \ + MOZ_CRASH(explain); \ } while (false) nsresult rv = NS_NewLocalFile(mPluginPath, true, getter_AddRefs(libFile)); @@ -514,7 +513,7 @@ mozilla::ipc::IPCResult GMPChild::RecvStartPlugin(const nsString& aAdapter) { nsAutoCString libPath; if (!GetUTF8LibPath(libPath)) { - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationNSCString( CrashReporter::Annotation::GMPLibraryPath, NS_ConvertUTF16toUTF8(mPluginPath)); @@ -556,7 +555,7 @@ mozilla::ipc::IPCResult GMPChild::RecvStartPlugin(const nsString& aAdapter) { NS_WARNING("Failed to load GMP"); #endif delete platformAPI; - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationNSCString( CrashReporter::Annotation::GMPLibraryPath, NS_ConvertUTF16toUTF8(mPluginPath)); @@ -727,21 +726,21 @@ mozilla::ipc::IPCResult GMPChild::RecvShutdown(ShutdownResolver&& aResolver) { } const bool isProfiling = profiler_is_active(); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - GrabShutdownProfileAndShutdown"_ns - : "Not profiling - GrabShutdownProfileAndShutdown"_ns); + isProfiling ? "Profiling - GrabShutdownProfileAndShutdown" + : "Not profiling - GrabShutdownProfileAndShutdown"); ProfileAndAdditionalInformation shutdownProfileAndAdditionalInformation = mProfilerController->GrabShutdownProfileAndShutdown(); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - Destroying ChildProfilerController"_ns - : "Not profiling - Destroying ChildProfilerController"_ns); + isProfiling ? "Profiling - Destroying ChildProfilerController" + : "Not profiling - Destroying ChildProfilerController"); mProfilerController = nullptr; - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - SendShutdownProfile (resovling)"_ns - : "Not profiling - SendShutdownProfile (resolving)"_ns); + isProfiling ? "Profiling - SendShutdownProfile (resovling)" + : "Not profiling - SendShutdownProfile (resolving)"); if (const size_t len = shutdownProfileAndAdditionalInformation.SizeOf(); len >= size_t(IPC::Channel::kMaximumMessageSize)) { shutdownProfileAndAdditionalInformation.mProfile = @@ -752,10 +751,10 @@ mozilla::ipc::IPCResult GMPChild::RecvShutdown(ShutdownResolver&& aResolver) { // Send the shutdown profile to the parent process through our own // message channel, which we know will survive for long enough. aResolver(shutdownProfileAndAdditionalInformation.mProfile); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - SendShutdownProfile (resolved)"_ns - : "Not profiling - SendShutdownProfile (resolved)"_ns); + isProfiling ? "Profiling - SendShutdownProfile (resolved)" + : "Not profiling - SendShutdownProfile (resolved)"); return IPC_OK(); } diff --git a/dom/media/gmp/GMPDiskStorage.cpp b/dom/media/gmp/GMPDiskStorage.cpp index 5f7a023b9b..c1c0c8215a 100644 --- a/dom/media/gmp/GMPDiskStorage.cpp +++ b/dom/media/gmp/GMPDiskStorage.cpp @@ -13,7 +13,6 @@ #include "mozilla/EndianUtils.h" #include "nsClassHashtable.h" #include "prio.h" -#include "nsContentCID.h" #include "nsServiceManagerUtils.h" namespace mozilla::gmp { diff --git a/dom/media/gmp/GMPLoader.cpp b/dom/media/gmp/GMPLoader.cpp index 9fa7756165..c0c9364c7b 100644 --- a/dom/media/gmp/GMPLoader.cpp +++ b/dom/media/gmp/GMPLoader.cpp @@ -84,7 +84,7 @@ class PassThroughGMPAdapter : public GMPAdapter { bool GMPLoader::Load(const char* aUTF8LibPath, uint32_t aUTF8LibPathLen, const GMPPlatformAPI* aPlatformAPI, GMPAdapter* aAdapter) { - CrashReporter::AutoAnnotateCrashReport autoLibPath( + CrashReporter::AutoRecordAnnotation autoLibPath( CrashReporter::Annotation::GMPLibraryPath, nsDependentCString(aUTF8LibPath)); diff --git a/dom/media/gmp/GMPParent.cpp b/dom/media/gmp/GMPParent.cpp index 115e9b7392..0830313be9 100644 --- a/dom/media/gmp/GMPParent.cpp +++ b/dom/media/gmp/GMPParent.cpp @@ -770,13 +770,15 @@ bool GMPParent::EnsureProcessLoaded() { void GMPParent::AddCrashAnnotations() { if (mCrashReporter) { - mCrashReporter->AddAnnotation(CrashReporter::Annotation::GMPPlugin, true); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginFilename, - NS_ConvertUTF16toUTF8(mName)); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginName, - mDisplayName); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginVersion, - mVersion); + mCrashReporter->AddAnnotationBool(CrashReporter::Annotation::GMPPlugin, + true); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginFilename, + NS_ConvertUTF16toUTF8(mName)); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginName, mDisplayName); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginVersion, mVersion); } } diff --git a/dom/media/gmp/moz.build b/dom/media/gmp/moz.build index a2b1be5ed8..3c34021506 100644 --- a/dom/media/gmp/moz.build +++ b/dom/media/gmp/moz.build @@ -126,7 +126,7 @@ PREPROCESSED_IPDL_SOURCES += [ "PGMPContent.ipdl", ] -if CONFIG["OS_TARGET"] in ["WINNT", "Darwin"]: +if CONFIG["TARGET_OS"] in ["WINNT", "OSX"]: DEFINES["SUPPORT_STORAGE_ID"] = 1 include("/ipc/chromium/chromium-config.mozbuild") diff --git a/dom/media/gmp/rlz/moz.build b/dom/media/gmp/rlz/moz.build index 8e2d9ea5d1..a874d54c65 100644 --- a/dom/media/gmp/rlz/moz.build +++ b/dom/media/gmp/rlz/moz.build @@ -9,19 +9,19 @@ FINAL_LIBRARY = 'xul' -if CONFIG['OS_TARGET'] in ['WINNT', 'Darwin']: +if CONFIG['TARGET_OS'] in ['WINNT', 'OSX']: UNIFIED_SOURCES += [ 'lib/crc8.cc', 'lib/machine_id.cc', 'lib/string_utils.cc', ] -if CONFIG['OS_TARGET'] == 'WINNT': +if CONFIG['TARGET_OS'] == 'WINNT': UNIFIED_SOURCES += [ 'win/lib/machine_id_win.cc', ] -if CONFIG['OS_TARGET'] == 'Darwin': +if CONFIG['TARGET_OS'] == 'OSX': UNIFIED_SOURCES += [ 'mac/lib/machine_id_mac.cc', ] diff --git a/dom/media/gtest/TestAudioSampleFormat.cpp b/dom/media/gtest/TestAudioSampleFormat.cpp new file mode 100644 index 0000000000..4737a73a23 --- /dev/null +++ b/dom/media/gtest/TestAudioSampleFormat.cpp @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AudioSampleFormat.h" +#include "gtest/gtest.h" +#include <type_traits> + +using namespace mozilla; + +template <typename T> +constexpr T LowestSample() { + if constexpr (std::is_integral_v<T>) { + return std::numeric_limits<T>::lowest(); + } else { + return -1.0f; + } +} + +// When converting a sample-type to another sample-type, this returns the +// maximum value possible in the destination format +template <typename Dest> +constexpr Dest HighestSample() { + if constexpr (std::is_integral_v<Dest>) { + return std::numeric_limits<Dest>::max(); + } else { + return +1.0f; + } +} + +// When converting a sample-type to another sample-type, this returns the +// maximum value expected in the destination format +template <typename Dest, typename Source> +constexpr Dest HighestSampleExpected() { + // When converting small integer samples to large integer sample, the higher + // bound isn't reached because of positive / negative integer assymetry. + if constexpr (std::is_same_v<Source, uint8_t> && + std::is_same_v<Dest, int16_t>) { + return 32512; // INT16_MAX - 2 << 7 + 1 + } else if constexpr (std::is_same_v<Source, uint8_t> && + std::is_same_v<Dest, int32_t>) { + return 2130706432; // INT32_MAX - (2 << 23) + 1 + } else if constexpr (std::is_same_v<Source, int16_t> && + std::is_same_v<Dest, int32_t>) { + return 2147418112; // INT32_MAX - UINT16_MAX + } + + if constexpr (std::is_integral_v<Dest>) { + return std::numeric_limits<Dest>::max(); + } else { + return +1.0f; + } +} + +template <typename Source, typename Dest> +void TestSampleTypePair() { + std::cout << __PRETTY_FUNCTION__ << std::endl; + + ASSERT_EQ(LowestSample<Dest>(), + ConvertAudioSample<Dest>(LowestSample<Source>())); + Dest expected = HighestSampleExpected<Dest, Source>(); + ASSERT_EQ(expected, ConvertAudioSample<Dest>(HighestSample<Source>())); + ASSERT_EQ(Bias<Dest>(), ConvertAudioSample<Dest>(Bias<Source>())); +} + +template <typename T> +void TestSampleType24bits() { + std::cout << __PRETTY_FUNCTION__ << std::endl; + + int32_t max_sample_24bits = (2 << 22) - 1; + int32_t min_sample_24bits = -(2 << 22); + int32_t silence_24bits = 0; + + ASSERT_EQ(LowestSample<T>(), Int24ToAudioSample<T>(min_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample<T>(min_sample_24bits), LowestSample<T>()); + if constexpr (std::is_same_v<T, int32_t>) { + // Quantization issue: 2147483392 + (2<<8 - 1) == INT32_MAX + // See comment on HighestSampleExpected above + const int32_t HIGHEST_FROM_24BITS = 2147483392; + ASSERT_EQ(HIGHEST_FROM_24BITS, Int24ToAudioSample<T>(max_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample<T>(max_sample_24bits), HIGHEST_FROM_24BITS); + } else { + ASSERT_EQ(HighestSample<T>(), Int24ToAudioSample<T>(max_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample<T>(max_sample_24bits), HighestSample<T>()); + } + ASSERT_EQ(Bias<T>(), Int24ToAudioSample<T>(silence_24bits)); + ASSERT_EQ(Int24ToAudioSample<T>(silence_24bits), Bias<T>()); +} + +TEST(AudioSampleFormat, Boundaries) +{ + TestSampleTypePair<uint8_t, uint8_t>(); + TestSampleTypePair<uint8_t, int16_t>(); + TestSampleTypePair<uint8_t, int32_t>(); + TestSampleTypePair<uint8_t, float>(); + TestSampleTypePair<int16_t, uint8_t>(); + TestSampleTypePair<int16_t, int16_t>(); + TestSampleTypePair<int16_t, int32_t>(); + TestSampleTypePair<int16_t, float>(); + TestSampleTypePair<int32_t, uint8_t>(); + TestSampleTypePair<int32_t, int16_t>(); + TestSampleTypePair<int32_t, int32_t>(); + TestSampleTypePair<int32_t, float>(); + TestSampleTypePair<float, uint8_t>(); + TestSampleTypePair<float, int16_t>(); + TestSampleTypePair<float, int32_t>(); + TestSampleTypePair<float, float>(); + + // Separately test 24-bit audio stored in 32-bits integers. + TestSampleType24bits<uint8_t>(); + TestSampleType24bits<int16_t>(); + TestSampleType24bits<int32_t>(); + TestSampleType24bits<float>(); +} diff --git a/dom/media/gtest/TestAudioSegment.cpp b/dom/media/gtest/TestAudioSegment.cpp index ee44839283..ea0e43f0cd 100644 --- a/dom/media/gtest/TestAudioSegment.cpp +++ b/dom/media/gtest/TestAudioSegment.cpp @@ -31,7 +31,7 @@ float GetLowValue<float>() { template <> int16_t GetLowValue<short>() { - return -INT16_MAX; + return INT16_MIN; } template <> @@ -62,7 +62,7 @@ const T* const* GetPlanarChannelArray(size_t aChannels, size_t aSize) { for (size_t c = 0; c < aChannels; c++) { channels[c] = new T[aSize]; for (size_t i = 0; i < aSize; i++) { - channels[c][i] = FloatToAudioSample<T>(1. / (c + 1)); + channels[c][i] = ConvertAudioSample<T>(1.f / static_cast<float>(c + 1)); } } return channels; @@ -104,7 +104,7 @@ const T* GetInterleavedChannelArray(size_t aChannels, size_t aSize) { T* samples = new T[sampleCount]; for (size_t i = 0; i < sampleCount; i++) { uint32_t channel = (i % aChannels) + 1; - samples[i] = FloatToAudioSample<T>(1. / channel); + samples[i] = ConvertAudioSample<T>(1.f / static_cast<float>(channel)); } return samples; } @@ -128,8 +128,9 @@ void TestInterleaveAndConvert() { uint32_t channelIndex = 0; for (size_t i = 0; i < arraySize * channels; i++) { - ASSERT_TRUE(FuzzyEqual( - dst[i], FloatToAudioSample<DstT>(1. / (channelIndex + 1)))); + ASSERT_TRUE( + FuzzyEqual(dst[i], ConvertAudioSample<DstT>( + 1.f / static_cast<float>(channelIndex + 1)))); channelIndex++; channelIndex %= channels; } @@ -151,8 +152,9 @@ void TestDeinterleaveAndConvert() { for (size_t channel = 0; channel < channels; channel++) { for (size_t i = 0; i < arraySize; i++) { - ASSERT_TRUE(FuzzyEqual(dst[channel][i], - FloatToAudioSample<DstT>(1. / (channel + 1)))); + ASSERT_TRUE(FuzzyEqual( + dst[channel][i], + ConvertAudioSample<DstT>(1.f / static_cast<float>(channel + 1)))); } } diff --git a/dom/media/gtest/TestAudioTrackGraph.cpp b/dom/media/gtest/TestAudioTrackGraph.cpp index 457c50e731..1bd255bed1 100644 --- a/dom/media/gtest/TestAudioTrackGraph.cpp +++ b/dom/media/gtest/TestAudioTrackGraph.cpp @@ -1462,7 +1462,7 @@ float rmsf32(AudioDataValue* aSamples, uint32_t aChannels, uint32_t aFrames) { for (uint32_t i = 0; i < aFrames; i++) { downmixed = 0.; for (uint32_t j = 0; j < aChannels; j++) { - downmixed += AudioSampleToFloat(aSamples[readIdx++]); + downmixed += ConvertAudioSample<float>(aSamples[readIdx++]); } rms += downmixed * downmixed; } diff --git a/dom/media/gtest/TestMediaDataDecoder.cpp b/dom/media/gtest/TestMediaDataDecoder.cpp index 79a92842b6..820b15b718 100644 --- a/dom/media/gtest/TestMediaDataDecoder.cpp +++ b/dom/media/gtest/TestMediaDataDecoder.cpp @@ -63,8 +63,7 @@ TEST(MediaDataDecoder, H264) } // Decoding AV1 via. ffvpx is supported on Linux only. -#if defined(MOZ_AV1) && defined(MOZ_WIDGET_GTK) && defined(MOZ_FFVPX) && \ - !defined(MOZ_FFVPX_AUDIOONLY) +#if defined(MOZ_AV1) && defined(MOZ_WIDGET_GTK) && !defined(MOZ_FFVPX_AUDIOONLY) TEST(MediaDataDecoder, AV1) { if (!MP4Decoder::IsSupportedType(MediaContainerType(MEDIAMIMETYPE(VIDEO_MP4)), diff --git a/dom/media/gtest/TestMediaDataEncoder.cpp b/dom/media/gtest/TestMediaDataEncoder.cpp index bdab94cfe5..27a6b7cd07 100644 --- a/dom/media/gtest/TestMediaDataEncoder.cpp +++ b/dom/media/gtest/TestMediaDataEncoder.cpp @@ -382,8 +382,7 @@ TEST_F(MediaDataEncoderTest, AndroidNotSupportedSize) { } #endif -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, H264AVCC) { RUN_IF_SUPPORTED(CodecType::H264, [this]() { // Encod frames in avcC format. @@ -508,8 +507,7 @@ TEST_F(MediaDataEncoderTest, VP8Duration) { }); } -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, VP8EncodeAfterDrain) { RUN_IF_SUPPORTED(CodecType::VP8, [this]() { RefPtr<MediaDataEncoder> e = CreateVP8Encoder(); @@ -673,8 +671,7 @@ TEST_F(MediaDataEncoderTest, VP9Duration) { }); } -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, VP9EncodeAfterDrain) { RUN_IF_SUPPORTED(CodecType::VP9, [this]() { RefPtr<MediaDataEncoder> e = CreateVP9Encoder(); diff --git a/dom/media/gtest/TestMediaQueue.cpp b/dom/media/gtest/TestMediaQueue.cpp index 5b049dc7fe..7176de069f 100644 --- a/dom/media/gtest/TestMediaQueue.cpp +++ b/dom/media/gtest/TestMediaQueue.cpp @@ -6,6 +6,7 @@ #include "MediaData.h" #include "MediaQueue.h" +#include "nsISupportsImpl.h" using namespace mozilla; using mozilla::media::TimeUnit; @@ -285,4 +286,19 @@ TEST(MediaQueue, TimestampAdjustmentForNotSupportDataType) EXPECT_EQ(data->GetEndTime(), TimeUnit::FromMicroseconds(10)); } +TEST(MediaQueue, PreciseDuration) +{ + MediaQueue<MediaData> queueOff; + queueOff.Push(CreateDataRawPtr(5, 10)); + queueOff.Push(CreateDataRawPtr(0, 5)); + EXPECT_EQ(queueOff.Duration(), 0); + EXPECT_EQ(queueOff.PreciseDuration(), -1); + + MediaQueue<MediaData> queueOn(true /* aEnablePreciseDuration */); + queueOn.Push(CreateDataRawPtr(5, 10)); + queueOn.Push(CreateDataRawPtr(0, 5)); + EXPECT_EQ(queueOn.Duration(), 0); + EXPECT_EQ(queueOn.PreciseDuration(), 10); +} + #undef EXPECT_EQUAL_SIZE_T diff --git a/dom/media/gtest/TestMediaUtils.cpp b/dom/media/gtest/TestMediaUtils.cpp index 33a32b7ea0..3708f43f01 100644 --- a/dom/media/gtest/TestMediaUtils.cpp +++ b/dom/media/gtest/TestMediaUtils.cpp @@ -15,7 +15,7 @@ using namespace mozilla::gtest; using namespace mozilla::media; // Spawning the death test child process aborts on Android. -#if !defined(ANDROID) +#if !defined(ANDROID) && defined(GTEST_HAS_DEATH_TEST) // Kept here for reference as it can be handy during development. # define DISABLE_CRASH_REPORTING \ diff --git a/dom/media/gtest/moz.build b/dom/media/gtest/moz.build index 581be004ef..df67aeef18 100644 --- a/dom/media/gtest/moz.build +++ b/dom/media/gtest/moz.build @@ -31,6 +31,7 @@ UNIFIED_SOURCES += [ "TestAudioMixer.cpp", "TestAudioPacketizer.cpp", "TestAudioRingBuffer.cpp", + "TestAudioSampleFormat.cpp", "TestAudioSegment.cpp", "TestAudioSinkWrapper.cpp", "TestAudioTrackEncoder.cpp", diff --git a/dom/media/ipc/MFCDMSerializers.h b/dom/media/ipc/MFCDMSerializers.h index 587f30ae09..64867f4dbb 100644 --- a/dom/media/ipc/MFCDMSerializers.h +++ b/dom/media/ipc/MFCDMSerializers.h @@ -5,9 +5,9 @@ #ifndef DOM_MEDIA_IPC_MFCDMSERIALIZERS_H_ #define DOM_MEDIA_IPC_MFCDMSERIALIZERS_H_ -#include "ipc/EnumSerializer.h" #include "MediaData.h" #include "mozilla/KeySystemConfig.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "mozilla/dom/MediaKeyMessageEventBinding.h" #include "mozilla/dom/MediaKeyStatusMapBinding.h" @@ -35,23 +35,17 @@ struct ParamTraits<mozilla::CryptoScheme> template <> struct ParamTraits<mozilla::dom::MediaKeyMessageType> - : public ContiguousEnumSerializer< - mozilla::dom::MediaKeyMessageType, - mozilla::dom::MediaKeyMessageType::License_request, - mozilla::dom::MediaKeyMessageType::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaKeyMessageType> {}; template <> struct ParamTraits<mozilla::dom::MediaKeyStatus> - : public ContiguousEnumSerializer<mozilla::dom::MediaKeyStatus, - mozilla::dom::MediaKeyStatus::Usable, - mozilla::dom::MediaKeyStatus::EndGuard_> { + : public mozilla::dom::WebIDLEnumSerializer<mozilla::dom::MediaKeyStatus> { }; template <> struct ParamTraits<mozilla::dom::HDCPVersion> - : public ContiguousEnumSerializer<mozilla::dom::HDCPVersion, - mozilla::dom::HDCPVersion::_1_0, - mozilla::dom::HDCPVersion::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer<mozilla::dom::HDCPVersion> {}; } // namespace IPC diff --git a/dom/media/ipc/MFMediaEngineParent.cpp b/dom/media/ipc/MFMediaEngineParent.cpp index 0238956781..5ed1b71160 100644 --- a/dom/media/ipc/MFMediaEngineParent.cpp +++ b/dom/media/ipc/MFMediaEngineParent.cpp @@ -113,7 +113,6 @@ void MFMediaEngineParent::DestroyEngineIfExists( mRequestSampleListener.DisconnectIfExists(); if (mDXGIDeviceManager) { mDXGIDeviceManager = nullptr; - wmf::MFUnlockDXGIDeviceManager(); } if (aError) { Unused << SendNotifyError(*aError); @@ -191,6 +190,9 @@ void MFMediaEngineParent::InitializeDXGIDeviceManager() { UINT deviceResetToken; RETURN_VOID_IF_FAILED( wmf::MFLockDXGIDeviceManager(&deviceResetToken, &mDXGIDeviceManager)); + if (!mDXGIDeviceManager) { + return; + } RETURN_VOID_IF_FAILED( mDXGIDeviceManager->ResetDevice(d3d11Device.get(), deviceResetToken)); LOG("Initialized DXGI manager"); diff --git a/dom/media/ipc/RDDProcessHost.cpp b/dom/media/ipc/RDDProcessHost.cpp index 0c0d35e7b5..d855b7804b 100644 --- a/dom/media/ipc/RDDProcessHost.cpp +++ b/dom/media/ipc/RDDProcessHost.cpp @@ -53,10 +53,6 @@ bool RDDProcessHost::Launch(StringVector aExtraOpts) { } mPrefSerializer->AddSharedPrefCmdLineArgs(*this, aExtraOpts); -#if defined(XP_WIN) && defined(MOZ_SANDBOX) - mSandboxLevel = Preferences::GetInt("security.sandbox.rdd.level"); -#endif - mLaunchPhase = LaunchPhase::Waiting; mLaunchTime = TimeStamp::Now(); diff --git a/dom/media/ipc/RDDProcessManager.cpp b/dom/media/ipc/RDDProcessManager.cpp index e7da3c3569..cb51b68282 100644 --- a/dom/media/ipc/RDDProcessManager.cpp +++ b/dom/media/ipc/RDDProcessManager.cpp @@ -170,8 +170,8 @@ RefPtr<GenericNonExclusivePromise> RDDProcessManager::LaunchRDDProcess() { } mQueuedPrefs.Clear(); - CrashReporter::AnnotateCrashReport( - CrashReporter::Annotation::RDDProcessStatus, "Running"_ns); + CrashReporter::RecordAnnotationCString( + CrashReporter::Annotation::RDDProcessStatus, "Running"); if (!CreateVideoBridge()) { mNumProcessAttempts++; @@ -271,8 +271,8 @@ void RDDProcessManager::DestroyProcess() { mRDDChild = nullptr; mQueuedPrefs.Clear(); - CrashReporter::AnnotateCrashReport( - CrashReporter::Annotation::RDDProcessStatus, "Destroyed"_ns); + CrashReporter::RecordAnnotationCString( + CrashReporter::Annotation::RDDProcessStatus, "Destroyed"); } bool RDDProcessManager::CreateContentBridge( diff --git a/dom/media/ipc/RemoteDecoderManagerChild.cpp b/dom/media/ipc/RemoteDecoderManagerChild.cpp index 48da254f39..b0b075e4d2 100644 --- a/dom/media/ipc/RemoteDecoderManagerChild.cpp +++ b/dom/media/ipc/RemoteDecoderManagerChild.cpp @@ -50,8 +50,8 @@ using namespace gfx; // Used so that we only ever attempt to check if the RDD/GPU/Utility processes // should be launched serially. Protects sLaunchPromise StaticMutex sLaunchMutex; -static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, - StaticRefPtr<GenericNonExclusivePromise>> +static EnumeratedArray<RemoteDecodeIn, StaticRefPtr<GenericNonExclusivePromise>, + size_t(RemoteDecodeIn::SENTINEL)> sLaunchPromises MOZ_GUARDED_BY(sLaunchMutex); // Only modified on the main-thread, read on any thread. While it could be read @@ -61,8 +61,8 @@ static StaticDataMutex<StaticRefPtr<nsIThread>> sRemoteDecoderManagerChildThread("sRemoteDecoderManagerChildThread"); // Only accessed from sRemoteDecoderManagerChildThread -static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, - StaticRefPtr<RemoteDecoderManagerChild>> +static EnumeratedArray<RemoteDecodeIn, StaticRefPtr<RemoteDecoderManagerChild>, + size_t(RemoteDecodeIn::SENTINEL)> sRemoteDecoderManagerChildForProcesses; static StaticAutoPtr<nsTArray<RefPtr<Runnable>>> sRecreateTasks; @@ -70,8 +70,8 @@ static StaticAutoPtr<nsTArray<RefPtr<Runnable>>> sRecreateTasks; // Used for protecting codec support information collected from different remote // processes. StaticMutex sProcessSupportedMutex; -static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, - Maybe<media::MediaCodecsSupported>> +static EnumeratedArray<RemoteDecodeIn, Maybe<media::MediaCodecsSupported>, + size_t(RemoteDecodeIn::SENTINEL)> sProcessSupported MOZ_GUARDED_BY(sProcessSupportedMutex); class ShutdownObserver final : public nsIObserver { @@ -311,6 +311,16 @@ RemoteDecoderManagerChild::CreateAudioDecoder( __func__); } + if (!aParams.mMediaEngineId && + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, + nsPrintfCString("%s only support for media engine playback", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + RefPtr<GenericNonExclusivePromise> launchPromise; if (StaticPrefs::media_utility_process_enabled() && (aLocation == RemoteDecodeIn::UtilityProcess_Generic || @@ -385,6 +395,16 @@ RemoteDecoderManagerChild::CreateVideoDecoder( __func__); } + if (!aParams.mMediaEngineId && + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, + nsPrintfCString("%s only support for media engine playback", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + MOZ_ASSERT(aLocation != RemoteDecodeIn::Unspecified); RefPtr<GenericNonExclusivePromise> p; diff --git a/dom/media/mediacapabilities/MediaCapabilities.cpp b/dom/media/mediacapabilities/MediaCapabilities.cpp index abc83dae4d..d3123f913b 100644 --- a/dom/media/mediacapabilities/MediaCapabilities.cpp +++ b/dom/media/mediacapabilities/MediaCapabilities.cpp @@ -45,21 +45,6 @@ static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { return nsCString(); } - nsCString hdrMetaType( - aConfig->mHdrMetadataType.WasPassed() - ? HdrMetadataTypeValues::GetString(aConfig->mHdrMetadataType.Value()) - : "?"); - - nsCString colorGamut( - aConfig->mColorGamut.WasPassed() - ? ColorGamutValues::GetString(aConfig->mColorGamut.Value()) - : "?"); - - nsCString transferFunction(aConfig->mTransferFunction.WasPassed() - ? TransferFunctionValues::GetString( - aConfig->mTransferFunction.Value()) - : "?"); - auto str = nsPrintfCString( "[contentType:%s width:%d height:%d bitrate:%" PRIu64 " framerate:%lf hasAlphaChannel:%s hdrMetadataType:%s colorGamut:%s " @@ -69,7 +54,15 @@ static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { aConfig->mHasAlphaChannel.WasPassed() ? aConfig->mHasAlphaChannel.Value() ? "true" : "false" : "?", - hdrMetaType.get(), colorGamut.get(), transferFunction.get(), + aConfig->mHdrMetadataType.WasPassed() + ? GetEnumString(aConfig->mHdrMetadataType.Value()).get() + : "?", + aConfig->mColorGamut.WasPassed() + ? GetEnumString(aConfig->mColorGamut.Value()).get() + : "?", + aConfig->mTransferFunction.WasPassed() + ? GetEnumString(aConfig->mTransferFunction.Value()).get() + : "?", aConfig->mScalabilityMode.WasPassed() ? NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.Value()).get() : "?"); diff --git a/dom/media/mediacontrol/ContentMediaController.cpp b/dom/media/mediacontrol/ContentMediaController.cpp index 1171785fe4..c0b466ff0f 100644 --- a/dom/media/mediacontrol/ContentMediaController.cpp +++ b/dom/media/mediacontrol/ContentMediaController.cpp @@ -286,8 +286,8 @@ void ContentMediaAgent::NotifyMediaFullScreenState(uint64_t aBrowsingContextId, } } -void ContentMediaAgent::UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) { +void ContentMediaAgent::UpdatePositionState( + uint64_t aBrowsingContextId, const Maybe<PositionState>& aState) { RefPtr<BrowsingContext> bc = GetBrowsingContextForAgent(aBrowsingContextId); if (!bc || bc->IsDiscarded()) { return; diff --git a/dom/media/mediacontrol/ContentMediaController.h b/dom/media/mediacontrol/ContentMediaController.h index 9e162dbb27..a58be24b9d 100644 --- a/dom/media/mediacontrol/ContentMediaController.h +++ b/dom/media/mediacontrol/ContentMediaController.h @@ -66,7 +66,7 @@ class ContentMediaAgent : public IMediaInfoUpdater { void NotifyMediaFullScreenState(uint64_t aBrowsingContextId, bool aIsInFullScreen) override; void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) override; + const Maybe<PositionState>& aState) override; // Use these methods to register/unregister `ContentMediaControlKeyReceiver` // in order to listen to media control key events. diff --git a/dom/media/mediacontrol/ContentPlaybackController.cpp b/dom/media/mediacontrol/ContentPlaybackController.cpp index ba48c0a5ce..fcc8e3ab58 100644 --- a/dom/media/mediacontrol/ContentPlaybackController.cpp +++ b/dom/media/mediacontrol/ContentPlaybackController.cpp @@ -168,8 +168,12 @@ void ContentMediaControlKeyHandler::HandleMediaControlAction( if (!aContext->GetDocShell()) { return; } + if (aAction.mKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Invalid media control key."); + return; + } ContentPlaybackController controller(aContext); - switch (aAction.mKey) { + switch (aAction.mKey.value()) { case MediaControlKey::Focus: controller.Focus(); return; @@ -179,6 +183,9 @@ void ContentMediaControlKeyHandler::HandleMediaControlAction( case MediaControlKey::Pause: controller.Pause(); return; + case MediaControlKey::Playpause: + MOZ_ASSERT_UNREACHABLE("Invalid media control key."); + return; case MediaControlKey::Stop: controller.Stop(); return; diff --git a/dom/media/mediacontrol/MediaControlKeyManager.cpp b/dom/media/mediacontrol/MediaControlKeyManager.cpp index 4cb562aa84..ba6ed3a524 100644 --- a/dom/media/mediacontrol/MediaControlKeyManager.cpp +++ b/dom/media/mediacontrol/MediaControlKeyManager.cpp @@ -184,10 +184,16 @@ void MediaControlKeyManager::SetEnablePictureInPictureMode(bool aIsEnabled) { } } -void MediaControlKeyManager::SetPositionState(const PositionState& aState) { - LOG_INFO("Set PositionState, duration=%f, playbackRate=%f, position=%f", - aState.mDuration, aState.mPlaybackRate, - aState.mLastReportedPlaybackPosition); +void MediaControlKeyManager::SetPositionState( + const Maybe<PositionState>& aState) { + if (aState) { + LOG_INFO("Set PositionState, duration=%f, playbackRate=%f, position=%f", + aState->mDuration, aState->mPlaybackRate, + aState->mLastReportedPlaybackPosition); + } else { + LOG_INFO("Set PositionState, Nothing"); + } + if (mEventSource && mEventSource->IsOpened()) { mEventSource->SetPositionState(aState); } diff --git a/dom/media/mediacontrol/MediaControlKeyManager.h b/dom/media/mediacontrol/MediaControlKeyManager.h index feb857e335..9b029f3bc9 100644 --- a/dom/media/mediacontrol/MediaControlKeyManager.h +++ b/dom/media/mediacontrol/MediaControlKeyManager.h @@ -41,7 +41,7 @@ class MediaControlKeyManager final : public MediaControlKeySource, void SetSupportedMediaKeys(const MediaKeysArray& aSupportedKeys) override; void SetEnableFullScreen(bool aIsEnabled) override; void SetEnablePictureInPictureMode(bool aIsEnabled) override; - void SetPositionState(const PositionState& aState) override; + void SetPositionState(const Maybe<PositionState>& aState) override; private: ~MediaControlKeyManager(); diff --git a/dom/media/mediacontrol/MediaControlKeySource.cpp b/dom/media/mediacontrol/MediaControlKeySource.cpp index 22756c860a..db372fc57b 100644 --- a/dom/media/mediacontrol/MediaControlKeySource.cpp +++ b/dom/media/mediacontrol/MediaControlKeySource.cpp @@ -36,7 +36,11 @@ void MediaControlKeyHandler::OnActionPerformed( return; } - switch (aAction.mKey) { + if (aAction.mKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Error : undefined media key!"); + return; + } + switch (aAction.mKey.value()) { case MediaControlKey::Focus: controller->Focus(); return; diff --git a/dom/media/mediacontrol/MediaControlKeySource.h b/dom/media/mediacontrol/MediaControlKeySource.h index f5d62a429e..4ab2c8b8be 100644 --- a/dom/media/mediacontrol/MediaControlKeySource.h +++ b/dom/media/mediacontrol/MediaControlKeySource.h @@ -28,10 +28,10 @@ struct SeekDetails { struct MediaControlAction { MediaControlAction() = default; - explicit MediaControlAction(MediaControlKey aKey) : mKey(aKey) {} + explicit MediaControlAction(MediaControlKey aKey) : mKey(Some(aKey)) {} MediaControlAction(MediaControlKey aKey, const SeekDetails& aDetails) - : mKey(aKey), mDetails(Some(aDetails)) {} - MediaControlKey mKey = MediaControlKey::EndGuard_; + : mKey(Some(aKey)), mDetails(Some(aDetails)) {} + Maybe<MediaControlKey> mKey; Maybe<SeekDetails> mDetails; }; @@ -109,7 +109,7 @@ class MediaControlKeySource { // to notify change to the embedded application. virtual void SetEnableFullScreen(bool aIsEnabled){}; virtual void SetEnablePictureInPictureMode(bool aIsEnabled){}; - virtual void SetPositionState(const PositionState& aState){}; + virtual void SetPositionState(const Maybe<PositionState>& aState){}; protected: virtual ~MediaControlKeySource() = default; diff --git a/dom/media/mediacontrol/MediaControlService.cpp b/dom/media/mediacontrol/MediaControlService.cpp index c321e080d2..f45ab4253d 100644 --- a/dom/media/mediacontrol/MediaControlService.cpp +++ b/dom/media/mediacontrol/MediaControlService.cpp @@ -157,6 +157,10 @@ void MediaControlService::NotifyMediaControlHasEverBeenUsed() { Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, u"Android"_ns, usedOnMediaControl); #endif +#ifdef MOZ_WIDGET_UIKIT + Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, + u"iOS"_ns, usedOnMediaControl); +#endif } void MediaControlService::NotifyMediaControlHasEverBeenEnabled() { @@ -182,6 +186,10 @@ void MediaControlService::NotifyMediaControlHasEverBeenEnabled() { Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, u"Android"_ns, enableOnMediaControl); #endif +#ifdef MOZ_WIDGET_UIKIT + Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, + u"iOS"_ns, enableOnMediaControl); +#endif } NS_IMETHODIMP @@ -510,7 +518,7 @@ void MediaControlService::ControllerManager::ConnectMainControllerEvents() { mSource->SetEnablePictureInPictureMode(aIsEnabled); }); mPositionChangedListener = mMainController->PositionChangedEvent().Connect( - AbstractThread::MainThread(), [this](const PositionState& aState) { + AbstractThread::MainThread(), [this](const Maybe<PositionState>& aState) { mSource->SetPositionState(aState); }); } diff --git a/dom/media/mediacontrol/MediaControlUtils.h b/dom/media/mediacontrol/MediaControlUtils.h index a327c2f3d8..e4e75e7c97 100644 --- a/dom/media/mediacontrol/MediaControlUtils.h +++ b/dom/media/mediacontrol/MediaControlUtils.h @@ -50,6 +50,14 @@ inline const char* ToMediaControlKeyStr(MediaControlKey aKey) { } } +inline const char* ToMediaControlKeyStr(const Maybe<MediaControlKey>& aKey) { + if (aKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Invalid action."); + return "Unknown"; + } + return ToMediaControlKeyStr(aKey.value()); +} + inline const char* ToMediaSessionActionStr(MediaSessionAction aAction) { switch (aAction) { case MediaSessionAction::Play: @@ -99,11 +107,6 @@ inline MediaControlKey ConvertMediaSessionActionToControlKey( } } -inline MediaSessionAction ConvertToMediaSessionAction(uint8_t aActionValue) { - MOZ_DIAGNOSTIC_ASSERT(aActionValue < uint8_t(MediaSessionAction::EndGuard_)); - return static_cast<MediaSessionAction>(aActionValue); -} - inline const char* ToMediaPlaybackStateStr(MediaPlaybackState aState) { switch (aState) { case MediaPlaybackState::eStarted: diff --git a/dom/media/mediacontrol/MediaController.cpp b/dom/media/mediacontrol/MediaController.cpp index bfb98f24c9..4290e952b0 100644 --- a/dom/media/mediacontrol/MediaController.cpp +++ b/dom/media/mediacontrol/MediaController.cpp @@ -186,9 +186,10 @@ bool MediaController::ShouldPropagateActionToAllContexts( // These three actions have default action handler for each frame, so we // need to propagate to all contexts. We would handle default handlers in // `ContentMediaController::HandleMediaKey`. - return aAction.mKey == MediaControlKey::Play || - aAction.mKey == MediaControlKey::Pause || - aAction.mKey == MediaControlKey::Stop; + return aAction.mKey.isSome() && + (aAction.mKey.value() == MediaControlKey::Play || + aAction.mKey.value() == MediaControlKey::Pause || + aAction.mKey.value() == MediaControlKey::Stop); } void MediaController::UpdateMediaControlActionToContentMediaIfNeeded( @@ -493,11 +494,16 @@ void MediaController::HandleSupportedMediaSessionActionsChanged( MediaController_Binding::ClearCachedSupportedKeysValue(this); } -void MediaController::HandlePositionStateChanged(const PositionState& aState) { +void MediaController::HandlePositionStateChanged( + const Maybe<PositionState>& aState) { + if (!aState) { + return; + } + PositionStateEventInit init; - init.mDuration = aState.mDuration; - init.mPlaybackRate = aState.mPlaybackRate; - init.mPosition = aState.mLastReportedPlaybackPosition; + init.mDuration = aState->mDuration; + init.mPlaybackRate = aState->mPlaybackRate; + init.mPosition = aState->mLastReportedPlaybackPosition; RefPtr<PositionStateEvent> event = PositionStateEvent::Constructor(this, u"positionstatechange"_ns, init); DispatchAsyncEvent(event.forget()); diff --git a/dom/media/mediacontrol/MediaController.h b/dom/media/mediacontrol/MediaController.h index 82b351ead7..f95ff6368e 100644 --- a/dom/media/mediacontrol/MediaController.h +++ b/dom/media/mediacontrol/MediaController.h @@ -159,7 +159,7 @@ class MediaController final : public DOMEventTargetHelper, void HandleSupportedMediaSessionActionsChanged( const nsTArray<MediaSessionAction>& aSupportedAction); - void HandlePositionStateChanged(const PositionState& aState); + void HandlePositionStateChanged(const Maybe<PositionState>& aState); void HandleMetadataChanged(const MediaMetadataBase& aMetadata); // This would register controller to the media control service that takes a diff --git a/dom/media/mediacontrol/MediaStatusManager.cpp b/dom/media/mediacontrol/MediaStatusManager.cpp index 4365e6b531..9187e56f25 100644 --- a/dom/media/mediacontrol/MediaStatusManager.cpp +++ b/dom/media/mediacontrol/MediaStatusManager.cpp @@ -154,6 +154,7 @@ void MediaStatusManager::SetActiveMediaSessionContextId( *mActiveMediaSessionContextId); mMetadataChangedEvent.Notify(GetCurrentMediaMetadata()); mSupportedActionsChangedEvent.Notify(GetSupportedActions()); + mPositionStateChangedEvent.Notify(GetCurrentPositionState()); if (StaticPrefs::media_mediacontrol_testingevents_enabled()) { if (nsCOMPtr<nsIObserverService> obs = services::GetObserverService()) { obs->NotifyObservers(nullptr, "active-media-session-changed", nullptr); @@ -170,6 +171,7 @@ void MediaStatusManager::ClearActiveMediaSessionContextIdIfNeeded() { StoreMediaSessionContextIdOnWindowContext(); mMetadataChangedEvent.Notify(GetCurrentMediaMetadata()); mSupportedActionsChangedEvent.Notify(GetSupportedActions()); + mPositionStateChangedEvent.Notify(GetCurrentPositionState()); if (StaticPrefs::media_mediacontrol_testingevents_enabled()) { if (nsCOMPtr<nsIObserverService> obs = services::GetObserverService()) { obs->NotifyObservers(nullptr, "active-media-session-changed", nullptr); @@ -362,8 +364,14 @@ void MediaStatusManager::DisableAction(uint64_t aBrowsingContextId, NotifySupportedKeysChangedIfNeeded(aBrowsingContextId); } -void MediaStatusManager::UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) { +void MediaStatusManager::UpdatePositionState( + uint64_t aBrowsingContextId, const Maybe<PositionState>& aState) { + auto info = mMediaSessionInfoMap.Lookup(aBrowsingContextId); + if (info) { + LOG("Update position state for context %" PRIu64, aBrowsingContextId); + info->mPositionState = aState; + } + // The position state comes from non-active media session which we don't care. if (!mActiveMediaSessionContextId || *mActiveMediaSessionContextId != aBrowsingContextId) { @@ -393,9 +401,8 @@ CopyableTArray<MediaSessionAction> MediaStatusManager::GetSupportedActions() MediaSessionInfo info = mMediaSessionInfoMap.Get(*mActiveMediaSessionContextId); - const uint8_t actionNums = uint8_t(MediaSessionAction::EndGuard_); - for (uint8_t actionValue = 0; actionValue < actionNums; actionValue++) { - MediaSessionAction action = ConvertToMediaSessionAction(actionValue); + for (MediaSessionAction action : + MakeWebIDLEnumeratedRange<MediaSessionAction>()) { if (info.IsActionSupported(action)) { supportedActions.AppendElement(action); } @@ -421,6 +428,16 @@ MediaMetadataBase MediaStatusManager::GetCurrentMediaMetadata() const { return CreateDefaultMetadata(); } +Maybe<PositionState> MediaStatusManager::GetCurrentPositionState() const { + if (mActiveMediaSessionContextId) { + auto info = mMediaSessionInfoMap.Lookup(*mActiveMediaSessionContextId); + if (info) { + return info->mPositionState; + } + } + return Nothing(); +} + void MediaStatusManager::FillMissingTitleAndArtworkIfNeeded( MediaMetadataBase& aMetadata) const { // If the metadata doesn't set its title and artwork properly, we would like diff --git a/dom/media/mediacontrol/MediaStatusManager.h b/dom/media/mediacontrol/MediaStatusManager.h index 24247d119d..a4216c8453 100644 --- a/dom/media/mediacontrol/MediaStatusManager.h +++ b/dom/media/mediacontrol/MediaStatusManager.h @@ -53,6 +53,7 @@ class MediaSessionInfo { Maybe<MediaMetadataBase> mMetadata; MediaSessionPlaybackState mDeclaredPlaybackState = MediaSessionPlaybackState::None; + Maybe<PositionState> mPositionState; // Use bitwise to store the supported actions. uint32_t mSupportedActions = 0; }; @@ -118,7 +119,7 @@ class IMediaInfoUpdater { // Use this method when media session update its position state. virtual void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) = 0; + const Maybe<PositionState>& aState) = 0; }; /** @@ -163,7 +164,7 @@ class MediaStatusManager : public IMediaInfoUpdater { void DisableAction(uint64_t aBrowsingContextId, MediaSessionAction aAction) override; void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) override; + const Maybe<PositionState>& aState) override; // Return active media session's metadata if active media session exists and // it has already set its metadata. Otherwise, return default media metadata @@ -180,7 +181,7 @@ class MediaStatusManager : public IMediaInfoUpdater { return mMetadataChangedEvent; } - MediaEventSource<PositionState>& PositionChangedEvent() { + MediaEventSource<Maybe<PositionState>>& PositionChangedEvent() { return mPositionStateChangedEvent; } @@ -246,6 +247,10 @@ class MediaStatusManager : public IMediaInfoUpdater { // media session doesn't exist, return 'None' instead. MediaSessionPlaybackState GetCurrentDeclaredPlaybackState() const; + // Return the active media session's position state. If the active media + // session doesn't exist or doesn't have any state, Nothing is returned. + Maybe<PositionState> GetCurrentPositionState() const; + // This state can match to the `guessed playback state` in the spec [1], it // indicates if we have any media element playing within the tab which this // controller belongs to. But currently we only take media elements into @@ -266,7 +271,7 @@ class MediaStatusManager : public IMediaInfoUpdater { MediaEventProducer<MediaMetadataBase> mMetadataChangedEvent; MediaEventProducer<nsTArray<MediaSessionAction>> mSupportedActionsChangedEvent; - MediaEventProducer<PositionState> mPositionStateChangedEvent; + MediaEventProducer<Maybe<PositionState>> mPositionStateChangedEvent; MediaEventProducer<MediaSessionPlaybackState> mPlaybackStateChangedEvent; MediaPlaybackStatus mPlaybackStatusDelegate; }; diff --git a/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js b/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js index 292c2f521f..a0c75bfa7c 100644 --- a/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js +++ b/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js @@ -184,7 +184,7 @@ function enableMediaFullScreenInIframe(tab) { } function waitUntilIframeMediaStartedPlaying(tab) { - return SpecialPowers.spawn(tab.linkedBrowser, [IFRAME_URL], async url => { + return SpecialPowers.spawn(tab.linkedBrowser, [IFRAME_URL], async () => { info(`check if media in iframe starts playing`); const iframe = content.document.getElementById("iframe"); iframe.contentWindow.postMessage("check-playing", "*"); diff --git a/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h b/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h index 5145eb7dbb..50ef254947 100644 --- a/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h +++ b/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h @@ -18,7 +18,7 @@ class MediaKeyListenerTest : public MediaControlKeyListener { void Clear() { mReceivedKey = mozilla::Nothing(); } void OnActionPerformed(const MediaControlAction& aAction) override { - mReceivedKey = mozilla::Some(aAction.mKey); + mReceivedKey = aAction.mKey; } bool IsResultEqualTo(MediaControlKey aResult) const { if (mReceivedKey) { diff --git a/dom/media/mediacontrol/tests/gtest/moz.build b/dom/media/mediacontrol/tests/gtest/moz.build index 7043bfcd5e..a8602f6718 100644 --- a/dom/media/mediacontrol/tests/gtest/moz.build +++ b/dom/media/mediacontrol/tests/gtest/moz.build @@ -11,7 +11,7 @@ UNIFIED_SOURCES += [ "TestMediaKeysEvent.cpp", ] -if CONFIG["MOZ_APPLEMEDIA"]: +if CONFIG["MOZ_APPLEMEDIA"] and CONFIG["TARGET_OS"] == "OSX": UNIFIED_SOURCES += ["TestMediaKeysEventMac.mm", "TestMediaKeysEventMediaCenter.mm"] include("/ipc/chromium/chromium-config.mozbuild") diff --git a/dom/media/mediasession/MediaSession.cpp b/dom/media/mediasession/MediaSession.cpp index e55fa28d96..9120aa5379 100644 --- a/dom/media/mediasession/MediaSession.cpp +++ b/dom/media/mediasession/MediaSession.cpp @@ -20,6 +20,29 @@ namespace mozilla::dom { +double PositionState::CurrentPlaybackPosition(TimeStamp aNow) const { + // https://w3c.github.io/mediasession/#current-playback-position + + // Set time elapsed to the system time in seconds minus the last position + // updated time. + auto timeElapsed = aNow - mPositionUpdatedTime; + // Mutliply time elapsed with actual playback rate. + timeElapsed = timeElapsed.MultDouble(mPlaybackRate); + // Set position to time elapsed added to last reported playback position. + auto position = timeElapsed.ToSeconds() + mLastReportedPlaybackPosition; + + // If position is less than zero, return zero. + if (position < 0.0) { + return 0.0; + } + // If position is greater than duration, return duration. + if (position > mDuration) { + return mDuration; + } + // Return position. + return position; +} + // We don't use NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE because we need to // unregister MediaSession from document's activity listeners. NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(MediaSession) @@ -138,6 +161,7 @@ void MediaSession::SetPositionState(const MediaPositionState& aState, // If the state is an empty dictionary then clear the position state. if (!aState.IsAnyMemberPresent()) { mPositionState.reset(); + NotifyPositionStateChanged(); return; } @@ -175,8 +199,8 @@ void MediaSession::SetPositionState(const MediaPositionState& aState, // Update the position state and last position updated time. MOZ_ASSERT(aState.mDuration.WasPassed()); - mPositionState = - Some(PositionState(aState.mDuration.Value(), playbackRate, position)); + mPositionState = Some(PositionState(aState.mDuration.Value(), playbackRate, + position, TimeStamp::Now())); NotifyPositionStateChanged(); } @@ -328,7 +352,7 @@ void MediaSession::NotifyPositionStateChanged() { RefPtr<BrowsingContext> currentBC = GetParentObject()->GetBrowsingContext(); MOZ_ASSERT(currentBC, "Update action after context destroyed!"); if (RefPtr<IMediaInfoUpdater> updater = ContentMediaAgent::Get(currentBC)) { - updater->UpdatePositionState(currentBC->Id(), *mPositionState); + updater->UpdatePositionState(currentBC->Id(), mPositionState); } } diff --git a/dom/media/mediasession/MediaSession.h b/dom/media/mediasession/MediaSession.h index db6864c842..6784fb531f 100644 --- a/dom/media/mediasession/MediaSession.h +++ b/dom/media/mediasession/MediaSession.h @@ -11,6 +11,7 @@ #include "mozilla/Attributes.h" #include "mozilla/dom/MediaSessionBinding.h" #include "mozilla/EnumeratedArray.h" +#include "mozilla/TimeStamp.h" #include "nsCycleCollectionParticipant.h" #include "nsIDocumentActivity.h" #include "nsWrapperCache.h" @@ -29,13 +30,21 @@ class MediaMetadata; struct PositionState { PositionState() = default; PositionState(double aDuration, double aPlaybackRate, - double aLastReportedTime) + double aLastReportedTime, TimeStamp aPositionUpdatedTime) : mDuration(aDuration), mPlaybackRate(aPlaybackRate), - mLastReportedPlaybackPosition(aLastReportedTime) {} - double mDuration; - double mPlaybackRate; - double mLastReportedPlaybackPosition; + mLastReportedPlaybackPosition(aLastReportedTime), + mPositionUpdatedTime(aPositionUpdatedTime) {} + + double mDuration = 0.0; + double mPlaybackRate = 0.0; + double mLastReportedPlaybackPosition = 0.0; + TimeStamp mPositionUpdatedTime; + + // Returns the playback position in seconds (from 0 to mDuration) + // at the current time (aNow). + // https://w3c.github.io/mediasession/#current-playback-position + double CurrentPlaybackPosition(TimeStamp aNow = TimeStamp::Now()) const; }; class MediaSession final : public nsIDocumentActivity, public nsWrapperCache { @@ -111,8 +120,7 @@ class MediaSession final : public nsIDocumentActivity, public nsWrapperCache { RefPtr<MediaMetadata> mMediaMetadata; - EnumeratedArray<MediaSessionAction, MediaSessionAction::EndGuard_, - RefPtr<MediaSessionActionHandler>> + EnumeratedArray<MediaSessionAction, RefPtr<MediaSessionActionHandler>> mActionHandlers; // This is used as is a hint for the user agent to determine whether the diff --git a/dom/media/mediasession/MediaSessionIPCUtils.h b/dom/media/mediasession/MediaSessionIPCUtils.h index c44f4b4553..610b123b6c 100644 --- a/dom/media/mediasession/MediaSessionIPCUtils.h +++ b/dom/media/mediasession/MediaSessionIPCUtils.h @@ -6,6 +6,7 @@ #define DOM_MEDIA_MEDIASESSION_MEDIASESSIONIPCUTILS_H_ #include "ipc/EnumSerializer.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "MediaMetadata.h" #include "mozilla/dom/MediaSession.h" #include "mozilla/dom/MediaSessionBinding.h" @@ -71,12 +72,14 @@ struct ParamTraits<mozilla::dom::PositionState> { WriteParam(aWriter, aParam.mDuration); WriteParam(aWriter, aParam.mPlaybackRate); WriteParam(aWriter, aParam.mLastReportedPlaybackPosition); + WriteParam(aWriter, aParam.mPositionUpdatedTime); } static bool Read(MessageReader* aReader, paramType* aResult) { if (!ReadParam(aReader, &(aResult->mDuration)) || !ReadParam(aReader, &(aResult->mPlaybackRate)) || - !ReadParam(aReader, &(aResult->mLastReportedPlaybackPosition))) { + !ReadParam(aReader, &(aResult->mLastReportedPlaybackPosition)) || + !ReadParam(aReader, &(aResult->mPositionUpdatedTime))) { return false; } return true; @@ -85,17 +88,13 @@ struct ParamTraits<mozilla::dom::PositionState> { template <> struct ParamTraits<mozilla::dom::MediaSessionPlaybackState> - : public ContiguousEnumSerializer< - mozilla::dom::MediaSessionPlaybackState, - mozilla::dom::MediaSessionPlaybackState::None, - mozilla::dom::MediaSessionPlaybackState::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaSessionPlaybackState> {}; template <> struct ParamTraits<mozilla::dom::MediaSessionAction> - : public ContiguousEnumSerializer< - mozilla::dom::MediaSessionAction, - mozilla::dom::MediaSessionAction::Play, - mozilla::dom::MediaSessionAction::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaSessionAction> {}; } // namespace IPC diff --git a/dom/media/mediasession/moz.build b/dom/media/mediasession/moz.build index 805bf67971..70f69fc14f 100644 --- a/dom/media/mediasession/moz.build +++ b/dom/media/mediasession/moz.build @@ -6,6 +6,10 @@ MOCHITEST_MANIFESTS += ["test/mochitest.toml"] +TEST_DIRS += [ + "test/gtest", +] + EXPORTS.mozilla.dom += [ "MediaMetadata.h", "MediaSession.h", diff --git a/dom/media/mediasession/test/gtest/TestPositionState.cpp b/dom/media/mediasession/test/gtest/TestPositionState.cpp new file mode 100644 index 0000000000..5fa38d0294 --- /dev/null +++ b/dom/media/mediasession/test/gtest/TestPositionState.cpp @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "MediaSession.h" + +using namespace mozilla; +using namespace mozilla::dom; + +struct CurrentPlaybackPositionFixture { + double mDuration = 0.0; + double mPlaybackRate = 0.0; + double mLastReportedTime = 0.0; + + TimeDuration mDelta; + double mExpectedPosition = -1; +}; + +class CurrentPlaybackPositionTest + : public testing::TestWithParam<CurrentPlaybackPositionFixture> {}; + +static const std::initializer_list<CurrentPlaybackPositionFixture> kFixtures = { + // position must be positive + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(-1.0), + 0.0, + }, + // no time elapsed + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(0.0), + 0.0, + }, + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(3.0), + 3.0, + }, + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(10.0), + 10.0, + }, + // position is clamped to the duration + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(20.0), + 10.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(-1.0), + 4.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(-6.0), + 0.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(6.0), + 10.0, + }, + // expected: 5s + 2 * 2s + { + 10.0, + 2.0, + 5.0, + TimeDuration::FromSeconds(2.0), + 9.0, + }, + // expected: 5s + 0.5 * 2s + { + 10.0, + 0.5, + 5.0, + TimeDuration::FromSeconds(2.0), + 6.0, + }, + { + 5.0, + 4.0, + 10.0, + TimeDuration::FromSeconds(20.0), + 5.0, + }, + // empty media (0s) + { + 0.0, + 4.0, + 5.0, + TimeDuration::FromSeconds(20.0), + 0.0, + }, +}; + +TEST_P(CurrentPlaybackPositionTest, Run) { + const auto& fixture = GetParam(); + PositionState state(fixture.mDuration, fixture.mPlaybackRate, + fixture.mLastReportedTime, TimeStamp::Now()); + + ASSERT_DOUBLE_EQ(state.CurrentPlaybackPosition(state.mPositionUpdatedTime + + fixture.mDelta), + fixture.mExpectedPosition); +} + +INSTANTIATE_TEST_SUITE_P(PositionState, CurrentPlaybackPositionTest, + testing::ValuesIn(kFixtures)); diff --git a/dom/media/mediasession/test/gtest/moz.build b/dom/media/mediasession/test/gtest/moz.build new file mode 100644 index 0000000000..fd2a9a6e30 --- /dev/null +++ b/dom/media/mediasession/test/gtest/moz.build @@ -0,0 +1,15 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "TestPositionState.cpp", +] + +LOCAL_INCLUDES += [ + "/dom/media/mediasession", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp index 24a74e261b..3a1308a9ca 100644 --- a/dom/media/mediasource/MediaSourceDecoder.cpp +++ b/dom/media/mediasource/MediaSourceDecoder.cpp @@ -57,8 +57,13 @@ MediaDecoderStateMachineBase* MediaSourceDecoder::CreateStateMachine( TrackingId::TrackAcrossProcesses::Yes); mReader = new MediaFormatReader(init, mDemuxer); #ifdef MOZ_WMF_MEDIA_ENGINE - // TODO : Only for testing development for now. In the future this should be - // used for encrypted content only. + // Our main purpose is to only using this state machine for encrypted playback + // (unless explicitly set the pref to allow non-encrypted playback), but we + // can't determine if playback is encrypted or not at the moment. Therefore, + // we will handle that in ExternalEngineStateMachine, and report special + // errors, such as NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR or + // NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR, to switch the state + // machine if necessary. if (StaticPrefs::media_wmf_media_engine_enabled() && !aDisableExternalEngine) { return new ExternalEngineStateMachine(this, mReader); @@ -319,12 +324,13 @@ bool MediaSourceDecoder::CanPlayThroughImpl() { } // If we have data up to the mediasource's duration or 3s ahead, we can // assume that we can play without interruption. - dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers(); - TimeUnit bufferedEnd = sourceBuffers->GetHighestBufferedEndTime(); + TimeIntervals buffered = GetBuffered(); + buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); TimeUnit timeAhead = std::min(duration, currentPosition + TimeUnit::FromSeconds(3)); TimeInterval interval(currentPosition, timeAhead); - return bufferedEnd >= timeAhead; + return buffered.ToMicrosecondResolution().ContainsWithStrictEnd( + ClampIntervalToEnd(interval)); } TimeInterval MediaSourceDecoder::ClampIntervalToEnd( @@ -366,6 +372,23 @@ bool MediaSourceDecoder::HadCrossOriginRedirects() { return false; } +#ifdef MOZ_WMF_MEDIA_ENGINE +void MediaSourceDecoder::MetadataLoaded( + UniquePtr<MediaInfo> aInfo, UniquePtr<MetadataTags> aTags, + MediaDecoderEventVisibility aEventVisibility) { + // If the previous state machine has loaded the metadata, then we don't need + // to load it again. This can happen when the media format or key system is + // not supported by previous state machine. + if (mFiredMetadataLoaded && mStateMachineRecreated) { + MSE_DEBUG( + "Metadata already loaded and being informed by previous state machine"); + return; + } + MediaDecoder::MetadataLoaded(std::move(aInfo), std::move(aTags), + aEventVisibility); +} +#endif + #undef MSE_DEBUG #undef MSE_DEBUGV diff --git a/dom/media/mediasource/MediaSourceDecoder.h b/dom/media/mediasource/MediaSourceDecoder.h index ff312cb6cf..2ebba67a17 100644 --- a/dom/media/mediasource/MediaSourceDecoder.h +++ b/dom/media/mediasource/MediaSourceDecoder.h @@ -85,6 +85,11 @@ class MediaSourceDecoder : public MediaDecoder, media::TimeInterval ClampIntervalToEnd(const media::TimeInterval& aInterval); bool CanPlayThroughImpl() override; +#ifdef MOZ_WMF_MEDIA_ENGINE + void MetadataLoaded(UniquePtr<MediaInfo> aInfo, UniquePtr<MetadataTags> aTags, + MediaDecoderEventVisibility aEventVisibility) override; +#endif + RefPtr<nsIPrincipal> mPrincipal; // The owning MediaSource holds a strong reference to this decoder, and diff --git a/dom/media/mediasource/test/mochitest.toml b/dom/media/mediasource/test/mochitest.toml index b9ed95b2bb..7e60af8929 100644 --- a/dom/media/mediasource/test/mochitest.toml +++ b/dom/media/mediasource/test/mochitest.toml @@ -270,3 +270,5 @@ skip-if = [ ["test_WaitingOnMissingData_mp4.html"] ["test_WaitingToEndedTransition_mp4.html"] + +["test_BufferedSeekCanPlayThrough.html"] diff --git a/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html b/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html new file mode 100644 index 0000000000..5c789a7b6e --- /dev/null +++ b/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html @@ -0,0 +1,46 @@ +<!DOCTYPE html> +<html><head> +<meta http-equiv="content-type" content="text/html; charset=windows-1252"> + <title>MSE: Don't get stuck buffering for too long when we have frames to show</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="mediasource.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<pre id="test"><script class="testbody" type="text/javascript"> + +SimpleTest.waitForExplicitFinish(); + +runWithMSE(async (ms, v) => { + logEvents(v); + await once(ms, "sourceopen"); + ok(true, "Receive a sourceopen event"); + ms.addEventListener("sourceopen", () => ok(false, "No more sourceopen")); + const sb = ms.addSourceBuffer("video/mp4"); + ok(sb, "Create a SourceBuffer"); + sb.addEventListener("error", e => { + ok(false, "Got Error: " + e); + SimpleTest.finish(); + }); + + // Load just the beginning of the media, and the end. Verify + // that canplaythrough isn't fired, and waiting is fired. + await fetchAndLoad(sb, "bipbop/bipbop_video", ["init"], ".mp4"); + await fetchAndLoad(sb, "bipbop/bipbop_video", ["1"], ".m4s"); + await fetchAndLoad(sb, "bipbop/bipbop_video", ["9"], ".m4s"); + // Slighly before the end of the first segment. + v.currentTime = v.buffered.end(0) - 0.1; + v.onseeked = function() { + is(v.readyState, HTMLMediaElement.HAVE_FUTURE_DATA, + "readyState is HAVE_FUTURE_DATA after seeking close to a large gap"); + SimpleTest.finish(); + } + v.oncanplaythrough = function() { + ok(false, "Should not have received canplaythrough"); + SimpleTest.finish(); + } +}); +</script> +</pre> +</body> +</html> diff --git a/dom/media/mediasource/test/test_EndOfStream.html b/dom/media/mediasource/test/test_EndOfStream.html index b926869f1f..bcaf2ee54d 100644 --- a/dom/media/mediasource/test/test_EndOfStream.html +++ b/dom/media/mediasource/test/test_EndOfStream.html @@ -12,7 +12,7 @@ SimpleTest.waitForExplicitFinish(); -runWithMSE(async (ms, v) => { +runWithMSE(async (ms) => { await once(ms, "sourceopen"); const sb = ms.addSourceBuffer("video/webm"); diff --git a/dom/media/mediasource/test/test_EndOfStream_mp4.html b/dom/media/mediasource/test/test_EndOfStream_mp4.html index 9319b80390..140641565d 100644 --- a/dom/media/mediasource/test/test_EndOfStream_mp4.html +++ b/dom/media/mediasource/test/test_EndOfStream_mp4.html @@ -12,7 +12,7 @@ SimpleTest.waitForExplicitFinish(); -runWithMSE(async (ms, v) => { +runWithMSE(async (ms) => { await once(ms, "sourceopen"); const sb = ms.addSourceBuffer("video/mp4"); diff --git a/dom/media/mediasource/test/test_ExperimentalAsync.html b/dom/media/mediasource/test/test_ExperimentalAsync.html index 6617716f26..e64a9befeb 100644 --- a/dom/media/mediasource/test/test_ExperimentalAsync.html +++ b/dom/media/mediasource/test/test_ExperimentalAsync.html @@ -61,7 +61,7 @@ runWithMSE(async function(ms, el) { await once(el, "seeked"); dump("dump: seeked to " + seekTime); is(el.currentTime, seekTime, "correctly seeked to " + seekTime); - await audiosb.appendBufferAsync(audioBuffer).catch(async function(ex2) { + await audiosb.appendBufferAsync(audioBuffer).catch(async function() { ok(false, "Shouldn't throw another time when data can be evicted"); dump(JSON.stringify(await SpecialPowers.wrap(el).mozRequestDebugInfo())); SimpleTest.finish(); @@ -73,7 +73,7 @@ runWithMSE(async function(ms, el) { await audiosb.removeAsync(ms.duration + 1, Infinity).catch(async function(ex4) { ok(true, "remove promise got rejected with start > duration"); is(ex4.name, "TypeError"); - await audiosb.removeAsync(0, Infinity).catch(function(ex5) { + await audiosb.removeAsync(0, Infinity).catch(function() { ok(false, "shouldn't throw"); }); ok(true, "remove succeeded"); diff --git a/dom/media/mediasource/test/test_SetModeThrows.html b/dom/media/mediasource/test/test_SetModeThrows.html index c715854b41..c81cb1b70f 100644 --- a/dom/media/mediasource/test/test_SetModeThrows.html +++ b/dom/media/mediasource/test/test_SetModeThrows.html @@ -13,7 +13,7 @@ SimpleTest.waitForExplicitFinish(); // MSE supports setting mode now. make sure it does not throw. -runWithMSE(function(ms, v) { +runWithMSE(function(ms) { ms.addEventListener("sourceopen", () => { const sb = ms.addSourceBuffer("video/webm"); diff --git a/dom/media/metrics.yaml b/dom/media/metrics.yaml index 46801ebaf4..fe2ed5ff6a 100644 --- a/dom/media/metrics.yaml +++ b/dom/media/metrics.yaml @@ -96,3 +96,47 @@ media.audio: notification_emails: - media-alerts@mozilla.com expires: never + +media.playback: + first_frame_loaded: + type: event + description: + The time that the media pipeline takes to load the first video frame. + metadata: + tags: + - 'Core :: Audio/Video: Playback' + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1882205 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1882205 + data_sensitivity: + - technical + notification_emails: + - media-alerts@mozilla.com + extra_keys: + first_frame_loaded_time: + description: + How long (in milliseconds) does the our media pipeline take to load + the first video frame. + type: quantity + playback_type: + description: + The type of the playback. The value could be one of following + (1) Non-MSE playback + (2) MSE playback + (3) EME playback + // Following are Windows-only + (4) Non-MSE media-engine playback + (5) MSE media-engine playback + (6) EME media-engine playback + type: string + video_codec: + description: The video codec used for playback + type: string + resolution: + description: The video resolution used for playback + type: string + key_system: + description: The key system used for the EME playback if exists + type: string + expires: never diff --git a/dom/media/nsIMediaManager.idl b/dom/media/nsIMediaManager.idl index 9cc39d04f4..68334b2e6a 100644 --- a/dom/media/nsIMediaManager.idl +++ b/dom/media/nsIMediaManager.idl @@ -8,11 +8,6 @@ interface nsIArray; interface nsIDOMWindow; interface nsIMediaDevice; -%{C++ -#define NS_MEDIAMANAGERSERVICE_CID {0xabc622ea, 0x9655, 0x4123, {0x80, 0xd9, 0x22, 0x62, 0x1b, 0xdd, 0x54, 0x65}} -#define MEDIAMANAGERSERVICE_CONTRACTID "@mozilla.org/mediaManagerService;1" -%} - [scriptable, builtinclass, uuid(24b23e01-33fd-401f-ba25-6e52658750b0)] interface nsIMediaManagerService : nsISupports { diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp index 2964527e07..00f46385e2 100644 --- a/dom/media/platforms/PDMFactory.cpp +++ b/dom/media/platforms/PDMFactory.cpp @@ -45,9 +45,6 @@ # include "mozilla/CDMProxy.h" # endif #endif -#ifdef MOZ_FFVPX -# include "FFVPXRuntimeLinker.h" -#endif #ifdef MOZ_FFMPEG # include "FFmpegRuntimeLinker.h" #endif @@ -60,6 +57,7 @@ #ifdef MOZ_OMX # include "OmxDecoderModule.h" #endif +#include "FFVPXRuntimeLinker.h" #include <functional> @@ -99,14 +97,12 @@ class PDMInitializer final { #ifdef MOZ_APPLEMEDIA AppleDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX - FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_rdd_ffmpeg_enabled()) { FFmpegRuntimeLinker::Init(); } #endif + FFVPXRuntimeLinker::Init(); } static void InitUtilityPDMs() { @@ -127,11 +123,9 @@ class PDMInitializer final { AppleDecoderModule::Init(); } #endif -#ifdef MOZ_FFVPX if (kind == ipc::SandboxingKind::GENERIC_UTILITY) { FFVPXRuntimeLinker::Init(); } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_utility_ffmpeg_enabled() && kind == ipc::SandboxingKind::GENERIC_UTILITY) { @@ -160,9 +154,7 @@ class PDMInitializer final { #ifdef MOZ_OMX OmxDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG FFmpegRuntimeLinker::Init(); #endif @@ -183,9 +175,7 @@ class PDMInitializer final { #ifdef MOZ_OMX OmxDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG FFmpegRuntimeLinker::Init(); #endif @@ -547,12 +537,7 @@ void PDMFactory::CreateRddPDMs() { CreateAndStartupPDM<AppleDecoderModule>(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_rdd_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && StaticPrefs::media_rdd_ffmpeg_enabled() && @@ -580,12 +565,9 @@ void PDMFactory::CreateUtilityPDMs() { } #endif if (aKind == ipc::SandboxingKind::GENERIC_UTILITY) { -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_utility_ffvpx_enabled()) { + if (StaticPrefs::media_utility_ffvpx_enabled()) { StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && StaticPrefs::media_utility_ffmpeg_enabled() && @@ -667,11 +649,7 @@ void PDMFactory::CreateContentPDMs() { CreateAndStartupPDM<OmxDecoderModule>(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && !StartupPDM(FFmpegRuntimeLinker::CreateDecoder())) { @@ -719,11 +697,7 @@ void PDMFactory::CreateDefaultPDMs() { CreateAndStartupPDM<OmxDecoderModule>(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && !StartupPDM(FFmpegRuntimeLinker::CreateDecoder())) { @@ -898,9 +872,6 @@ DecodeSupportSet PDMFactory::SupportsMimeType( /* static */ bool PDMFactory::AllDecodersAreRemote() { return StaticPrefs::media_rdd_process_enabled() && -#if defined(MOZ_FFVPX) - StaticPrefs::media_rdd_ffvpx_enabled() && -#endif StaticPrefs::media_rdd_opus_enabled() && StaticPrefs::media_rdd_theora_enabled() && StaticPrefs::media_rdd_vorbis_enabled() && diff --git a/dom/media/platforms/PEMFactory.cpp b/dom/media/platforms/PEMFactory.cpp index 9647c5b079..a5b42914eb 100644 --- a/dom/media/platforms/PEMFactory.cpp +++ b/dom/media/platforms/PEMFactory.cpp @@ -20,13 +20,12 @@ # include "WMFEncoderModule.h" #endif -#ifdef MOZ_FFVPX -# include "FFVPXRuntimeLinker.h" -#endif #ifdef MOZ_FFMPEG # include "FFmpegRuntimeLinker.h" #endif +#include "FFVPXRuntimeLinker.h" + #include "mozilla/StaticPrefs_media.h" #include "mozilla/gfx/gfxVars.h" @@ -56,15 +55,12 @@ PEMFactory::PEMFactory() { mCurrentPEMs.AppendElement(new WMFEncoderModule()); #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_ffmpeg_encoder_enabled()) { + if (StaticPrefs::media_ffmpeg_encoder_enabled()) { if (RefPtr<PlatformEncoderModule> pem = FFVPXRuntimeLinker::CreateEncoder()) { mCurrentPEMs.AppendElement(pem); } } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.cpp b/dom/media/platforms/agnostic/bytestreams/Adts.cpp index 5f31904d9c..71c9f15308 100644 --- a/dom/media/platforms/agnostic/bytestreams/Adts.cpp +++ b/dom/media/platforms/agnostic/bytestreams/Adts.cpp @@ -4,37 +4,56 @@ #include "Adts.h" #include "MediaData.h" +#include "PlatformDecoderModule.h" #include "mozilla/Array.h" #include "mozilla/ArrayUtils.h" +#include "mozilla/Logging.h" +#include "ADTSDemuxer.h" + +extern mozilla::LazyLogModule gMediaDemuxerLog; +#define LOG(msg, ...) \ + MOZ_LOG(gMediaDemuxerLog, LogLevel::Debug, msg, ##__VA_ARGS__) +#define ADTSLOG(msg, ...) \ + DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Debug, msg, ##__VA_ARGS__) +#define ADTSLOGV(msg, ...) \ + DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Verbose, msg, ##__VA_ARGS__) namespace mozilla { +namespace ADTS { static const int kADTSHeaderSize = 7; -int8_t Adts::GetFrequencyIndex(uint32_t aSamplesPerSecond) { - static const uint32_t freq_lookup[] = {96000, 88200, 64000, 48000, 44100, - 32000, 24000, 22050, 16000, 12000, - 11025, 8000, 7350, 0}; +constexpr std::array FREQ_LOOKUP{96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350, 0}; - int8_t i = 0; - while (freq_lookup[i] && aSamplesPerSecond < freq_lookup[i]) { - i++; - } +Result<uint8_t, bool> GetFrequencyIndex(uint32_t aSamplesPerSecond) { + auto found = + std::find(FREQ_LOOKUP.begin(), FREQ_LOOKUP.end(), aSamplesPerSecond); - if (!freq_lookup[i]) { - return -1; + if (found == FREQ_LOOKUP.end()) { + return Err(false); } - return i; + return std::distance(FREQ_LOOKUP.begin(), found); } -bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, - int8_t aProfile, MediaRawData* aSample) { +bool ConvertSample(uint16_t aChannelCount, uint8_t aFrequencyIndex, + uint8_t aProfile, MediaRawData* aSample) { size_t newSize = aSample->Size() + kADTSHeaderSize; + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Converting sample to ADTS format: newSize: %zu, ch: %u, " + "profile: %u, freq index: %d", + newSize, aChannelCount, aProfile, aFrequencyIndex)); + // ADTS header uses 13 bits for packet size. - if (newSize >= (1 << 13) || aChannelCount > 15 || aFrequencyIndex < 0 || - aProfile < 1 || aProfile > 4) { + if (newSize >= (1 << 13) || aChannelCount > 15 || aProfile < 1 || + aProfile > 4 || aFrequencyIndex >= FREQ_LOOKUP.size()) { + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Couldn't convert sample to ADTS format: newSize: %zu, ch: %u, " + "profile: %u, freq index: %d", + newSize, aChannelCount, aProfile, aFrequencyIndex)); return false; } @@ -66,7 +85,36 @@ bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, return true; } -bool Adts::RevertSample(MediaRawData* aSample) { +bool StripHeader(MediaRawData* aSample) { + if (aSample->Size() < kADTSHeaderSize) { + return false; + } + + FrameHeader header; + auto data = Span{aSample->Data(), aSample->Size()}; + MOZ_ASSERT(FrameHeader::MatchesSync(data), + "Don't attempt to strip the ADTS header of a raw AAC packet."); + + bool crcPresent = header.mHaveCrc; + + LOG(("Stripping ADTS, crc %spresent", crcPresent ? "" : "not ")); + + size_t toStrip = crcPresent ? kADTSHeaderSize + 2 : kADTSHeaderSize; + + UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter()); + writer->PopFront(toStrip); + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() > 0 && + writer->mCrypto.mPlainSizes[0] >= kADTSHeaderSize) { + writer->mCrypto.mPlainSizes[0] -= kADTSHeaderSize; + } + } + + return true; +} + +bool RevertSample(MediaRawData* aSample) { if (aSample->Size() < kADTSHeaderSize) { return false; } @@ -91,4 +139,156 @@ bool Adts::RevertSample(MediaRawData* aSample) { return true; } -} // namespace mozilla + +bool FrameHeader::MatchesSync(const Span<const uint8_t>& aData) { + return aData.Length() >= 2 && aData[0] == 0xFF && (aData[1] & 0xF6) == 0xF0; +} + +FrameHeader::FrameHeader() { Reset(); } + +// Header size +uint64_t FrameHeader::HeaderSize() const { return (mHaveCrc) ? 9 : 7; } + +bool FrameHeader::IsValid() const { return mFrameLength > 0; } + +// Resets the state to allow for a new parsing session. +void FrameHeader::Reset() { PodZero(this); } + +// Returns whether the byte creates a valid sequence up to this point. +bool FrameHeader::Parse(const Span<const uint8_t>& aData) { + if (!MatchesSync(aData)) { + return false; + } + + // AAC has 1024 samples per frame per channel. + mSamples = 1024; + + mHaveCrc = !(aData[1] & 0x01); + mObjectType = ((aData[2] & 0xC0) >> 6) + 1; + mSamplingIndex = (aData[2] & 0x3C) >> 2; + mChannelConfig = (aData[2] & 0x01) << 2 | (aData[3] & 0xC0) >> 6; + mFrameLength = + static_cast<uint32_t>((aData[3] & 0x03) << 11 | (aData[4] & 0xFF) << 3 | + (aData[5] & 0xE0) >> 5); + mNumAACFrames = (aData[6] & 0x03) + 1; + + static const uint32_t SAMPLE_RATES[] = {96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350}; + if (mSamplingIndex >= ArrayLength(SAMPLE_RATES)) { + LOG(("ADTS: Init() failure: invalid sample-rate index value: %" PRIu32 ".", + mSamplingIndex)); + // This marks the header as invalid. + mFrameLength = 0; + return false; + } + mSampleRate = SAMPLE_RATES[mSamplingIndex]; + + MOZ_ASSERT(mChannelConfig < 8); + mChannels = (mChannelConfig == 7) ? 8 : mChannelConfig; + + return true; +} + +Frame::Frame() : mOffset(0), mHeader() {} +uint64_t Frame::Offset() const { return mOffset; } +size_t Frame::Length() const { + // TODO: If fields are zero'd when invalid, this check wouldn't be + // necessary. + if (!mHeader.IsValid()) { + return 0; + } + + return mHeader.mFrameLength; +} + +// Returns the offset to the start of frame's raw data. +uint64_t Frame::PayloadOffset() const { return mOffset + mHeader.HeaderSize(); } + +// Returns the length of the frame's raw data (excluding the header) in bytes. +size_t Frame::PayloadLength() const { + // TODO: If fields are zero'd when invalid, this check wouldn't be + // necessary. + if (!mHeader.IsValid()) { + return 0; + } + + return mHeader.mFrameLength - mHeader.HeaderSize(); +} + +// Returns the parsed frame header. +const FrameHeader& Frame::Header() const { return mHeader; } + +bool Frame::IsValid() const { return mHeader.IsValid(); } + +// Resets the frame header and data. +void Frame::Reset() { + mHeader.Reset(); + mOffset = 0; +} + +// Returns whether the valid +bool Frame::Parse(uint64_t aOffset, const uint8_t* aStart, + const uint8_t* aEnd) { + MOZ_ASSERT(aStart && aEnd && aStart <= aEnd); + + bool found = false; + const uint8_t* ptr = aStart; + // Require at least 7 bytes of data at the end of the buffer for the minimum + // ADTS frame header. + while (ptr < aEnd - 7 && !found) { + found = mHeader.Parse(Span(ptr, aEnd)); + ptr++; + } + + mOffset = aOffset + (static_cast<size_t>(ptr - aStart)) - 1u; + + return found; +} + +const Frame& FrameParser::CurrentFrame() { return mFrame; } + +const Frame& FrameParser::FirstFrame() const { return mFirstFrame; } + +void FrameParser::Reset() { + EndFrameSession(); + mFirstFrame.Reset(); +} + +void FrameParser::EndFrameSession() { mFrame.Reset(); } + +bool FrameParser::Parse(uint64_t aOffset, const uint8_t* aStart, + const uint8_t* aEnd) { + const bool found = mFrame.Parse(aOffset, aStart, aEnd); + + if (mFrame.Length() && !mFirstFrame.Length()) { + mFirstFrame = mFrame; + } + + return found; +} + +// Initialize the AAC AudioSpecificConfig. +// Only handles two-byte version for AAC-LC. +void InitAudioSpecificConfig(const ADTS::Frame& frame, + MediaByteBuffer* aBuffer) { + const ADTS::FrameHeader& header = frame.Header(); + MOZ_ASSERT(header.IsValid()); + + int audioObjectType = header.mObjectType; + int samplingFrequencyIndex = header.mSamplingIndex; + int channelConfig = header.mChannelConfig; + + uint8_t asc[2]; + asc[0] = (audioObjectType & 0x1F) << 3 | (samplingFrequencyIndex & 0x0E) >> 1; + asc[1] = (samplingFrequencyIndex & 0x01) << 7 | (channelConfig & 0x0F) << 3; + + aBuffer->AppendElements(asc, 2); +} + +}; // namespace ADTS +}; // namespace mozilla + +#undef LOG +#undef ADTSLOG +#undef ADTSLOGV diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.h b/dom/media/platforms/agnostic/bytestreams/Adts.h index c2b6b558b6..e6d20806ab 100644 --- a/dom/media/platforms/agnostic/bytestreams/Adts.h +++ b/dom/media/platforms/agnostic/bytestreams/Adts.h @@ -6,17 +6,124 @@ #define ADTS_H_ #include <stdint.h> +#include "MediaData.h" +#include "mozilla/Result.h" namespace mozilla { class MediaRawData; -class Adts { +namespace ADTS { + +// adts::FrameHeader - Holds the ADTS frame header and its parsing +// state. +// +// ADTS Frame Structure +// +// 11111111 1111BCCD EEFFFFGH HHIJKLMM MMMMMMMM MMMOOOOO OOOOOOPP(QQQQQQQQ +// QQQQQQQQ) +// +// Header consists of 7 or 9 bytes(without or with CRC). +// Letter Length(bits) Description +// { sync } 12 syncword 0xFFF, all bits must be 1 +// B 1 MPEG Version: 0 for MPEG-4, 1 for MPEG-2 +// C 2 Layer: always 0 +// D 1 protection absent, Warning, set to 1 if there is no +// CRC and 0 if there is CRC +// E 2 profile, the MPEG-4 Audio Object Type minus 1 +// F 4 MPEG-4 Sampling Frequency Index (15 is forbidden) +// H 3 MPEG-4 Channel Configuration (in the case of 0, the +// channel configuration is sent via an in-band PCE) +// M 13 frame length, this value must include 7 or 9 bytes of +// header length: FrameLength = +// (ProtectionAbsent == 1 ? 7 : 9) + size(AACFrame) +// O 11 Buffer fullness +// P 2 Number of AAC frames(RDBs) in ADTS frame minus 1, for +// maximum compatibility always use 1 AAC frame per ADTS +// frame +// Q 16 CRC if protection absent is 0 +class FrameHeader { public: - static int8_t GetFrequencyIndex(uint32_t aSamplesPerSecond); - static bool ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, - int8_t aProfile, mozilla::MediaRawData* aSample); - static bool RevertSample(MediaRawData* aSample); + uint32_t mFrameLength{}; + uint32_t mSampleRate{}; + uint32_t mSamples{}; + uint32_t mChannels{}; + uint8_t mObjectType{}; + uint8_t mSamplingIndex{}; + uint8_t mChannelConfig{}; + uint8_t mNumAACFrames{}; + bool mHaveCrc{}; + + // Returns whether aPtr matches a valid ADTS header sync marker + static bool MatchesSync(const Span<const uint8_t>& aData); + FrameHeader(); + // Header size + uint64_t HeaderSize() const; + bool IsValid() const; + // Resets the state to allow for a new parsing session. + void Reset(); + + // Returns whether the byte creates a valid sequence up to this point. + bool Parse(const Span<const uint8_t>& aData); }; +class Frame { + public: + Frame(); + + uint64_t Offset() const; + size_t Length() const; + // Returns the offset to the start of frame's raw data. + uint64_t PayloadOffset() const; + + size_t PayloadLength() const; + // Returns the parsed frame header. + const FrameHeader& Header() const; + bool IsValid() const; + // Resets the frame header and data. + void Reset(); + // Returns whether the valid + bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd); + + private: + // The offset to the start of the header. + uint64_t mOffset; + // The currently parsed frame header. + FrameHeader mHeader; +}; + +class FrameParser { + public: + // Returns the currently parsed frame. Reset via Reset or EndFrameSession. + const Frame& CurrentFrame(); + // Returns the first parsed frame. Reset via Reset. + const Frame& FirstFrame() const; + // Resets the parser. Don't use between frames as first frame data is reset. + void Reset(); + // Clear the last parsed frame to allow for next frame parsing, i.e.: + // - sets PrevFrame to CurrentFrame + // - resets the CurrentFrame + // - resets ID3Header if no valid header was parsed yet + void EndFrameSession(); + // Parses contents of given ByteReader for a valid frame header and returns + // true if one was found. After returning, the variable passed to + // 'aBytesToSkip' holds the amount of bytes to be skipped (if any) in order to + // jump across a large ID3v2 tag spanning multiple buffers. + bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd); + + private: + // We keep the first parsed frame around for static info access, the + // previously parsed frame for debugging and the currently parsed frame. + Frame mFirstFrame; + Frame mFrame; +}; + +// Extract the audiospecificconfig from an ADTS header +void InitAudioSpecificConfig(const Frame& aFrame, MediaByteBuffer* aBuffer); +bool StripHeader(MediaRawData* aSample); +Result<uint8_t, bool> GetFrequencyIndex(uint32_t aSamplesPerSecond); +bool ConvertSample(uint16_t aChannelCount, uint8_t aFrequencyIndex, + uint8_t aProfile, mozilla::MediaRawData* aSample); +bool RevertSample(MediaRawData* aSample); +} // namespace ADTS } // namespace mozilla #endif diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp index 086936dcc6..4721ddefc3 100644 --- a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp @@ -256,21 +256,21 @@ static Result<Ok, nsresult> FindStartCodeInternal(BufferReader& aBr) { while (aBr.Remaining() >= 6) { uint32_t x32; MOZ_TRY_VAR(x32, aBr.PeekU32()); - if ((x32 - 0x01010101) & (~x32) & 0x80808080) { - if ((x32 >> 8) == 0x000001) { + if ((x32 - 0x01010101) & (~x32) & 0x80808080) { // Has 0x00 byte(s). + if ((x32 >> 8) == 0x000001) { // 0x000001?? return Ok(); } - if (x32 == 0x000001) { + if ((x32 & 0xffffff) == 0x000001) { // 0x??000001 mozilla::Unused << aBr.Read(1); return Ok(); } - if ((x32 & 0xff) == 0) { + if ((x32 & 0xff) == 0) { // 0x??????00 const uint8_t* p = aBr.Peek(1); - if ((x32 & 0xff00) == 0 && p[4] == 1) { + if ((x32 & 0xff00) == 0 && p[4] == 1) { // 0x????0000,01 mozilla::Unused << aBr.Read(2); return Ok(); } - if (p[4] == 0 && p[5] == 1) { + if (p[4] == 0 && p[5] == 1) { // 0x??????00,00,01 mozilla::Unused << aBr.Read(3); return Ok(); } diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp index e71632e6d3..4c74fa8723 100644 --- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp @@ -11,6 +11,9 @@ #include "GMPVideoDecoder.h" #include "MP4Decoder.h" #include "VPXDecoder.h" +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif namespace mozilla { @@ -45,6 +48,21 @@ static uint32_t ToCDMH264Profile(uint8_t aProfile) { return cdm::VideoCodecProfile::kUnknownVideoCodecProfile; } +#ifdef MOZ_AV1 +static uint32_t ToCDMAV1Profile(uint8_t aProfile) { + switch (aProfile) { + case 0: + return cdm::VideoCodecProfile::kAv1ProfileMain; + case 1: + return cdm::VideoCodecProfile::kAv1ProfileHigh; + case 2: + return cdm::VideoCodecProfile::kAv1ProfilePro; + default: + return cdm::VideoCodecProfile::kUnknownVideoCodecProfile; + } +} +#endif + RefPtr<MediaDataDecoder::InitPromise> ChromiumCDMVideoDecoder::Init() { if (!mCDMParent) { // Must have failed to get the CDMParent from the ChromiumCDMProxy @@ -60,6 +78,16 @@ RefPtr<MediaDataDecoder::InitPromise> ChromiumCDMVideoDecoder::Init() { ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0)); config.mExtraData() = mConfig.mExtraData->Clone(); mConvertToAnnexB = true; +#ifdef MOZ_AV1 + } else if (AOMDecoder::IsAV1(mConfig.mMimeType)) { + AOMDecoder::AV1SequenceInfo seqInfo; + MediaResult seqHdrResult; + AOMDecoder::TryReadAV1CBox(mConfig.mExtraData, seqInfo, seqHdrResult); + config.mCodec() = cdm::VideoCodec::kCodecAv1; + config.mProfile() = NS_SUCCEEDED(seqHdrResult.Code()) + ? ToCDMAV1Profile(seqInfo.mProfile) + : cdm::VideoCodecProfile::kUnknownVideoCodecProfile; +#endif } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { config.mCodec() = cdm::VideoCodec::kCodecVp8; config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded; @@ -105,9 +133,16 @@ nsCString ChromiumCDMVideoDecoder::GetDescriptionName() const { nsCString ChromiumCDMVideoDecoder::GetCodecName() const { if (MP4Decoder::IsH264(mConfig.mMimeType)) { return "h264"_ns; - } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + } +#ifdef MOZ_AV1 + if (AOMDecoder::IsAV1(mConfig.mMimeType)) { + return "av1"_ns; + } +#endif + if (VPXDecoder::IsVP8(mConfig.mMimeType)) { return "vp8"_ns; - } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + } + if (VPXDecoder::IsVP9(mConfig.mMimeType)) { return "vp9"_ns; } return "unknown"_ns; diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp index c143172073..a06dd30f89 100644 --- a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp +++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp @@ -28,7 +28,7 @@ namespace mozilla { -typedef MozPromiseRequestHolder<DecryptPromise> DecryptPromiseRequestHolder; +using DecryptPromiseRequestHolder = MozPromiseRequestHolder<DecryptPromise>; DDLoggedTypeDeclNameAndBase(EMEDecryptor, MediaDataDecoder); @@ -45,7 +45,7 @@ class ADTSSampleConverter { // doesn't care what is set. , mProfile(aInfo.mProfile < 1 || aInfo.mProfile > 4 ? 2 : aInfo.mProfile), - mFrequencyIndex(Adts::GetFrequencyIndex(aInfo.mRate)) { + mFrequencyIndex(ADTS::GetFrequencyIndex(aInfo.mRate).unwrapOr(255)) { EME_LOG("ADTSSampleConvertor(): aInfo.mProfile=%" PRIi8 " aInfo.mExtendedProfile=%" PRIi8, aInfo.mProfile, aInfo.mExtendedProfile); @@ -56,17 +56,17 @@ class ADTSSampleConverter { } } bool Convert(MediaRawData* aSample) const { - return Adts::ConvertSample(mNumChannels, mFrequencyIndex, mProfile, + return ADTS::ConvertSample(mNumChannels, mFrequencyIndex, mProfile, aSample); } bool Revert(MediaRawData* aSample) const { - return Adts::RevertSample(aSample); + return ADTS::RevertSample(aSample); } private: const uint32_t mNumChannels; const uint8_t mProfile; - const uint8_t mFrequencyIndex; + const uint8_t mFrequencyIndex{}; }; class EMEDecryptor final : public MediaDataDecoder, @@ -124,7 +124,7 @@ class EMEDecryptor final : public MediaDataDecoder, mThroughputLimiter->Throttle(aSample) ->Then( mThread, __func__, - [self](RefPtr<MediaRawData> aSample) { + [self](const RefPtr<MediaRawData>& aSample) { self->mThrottleRequest.Complete(); self->AttemptDecode(aSample); }, @@ -223,7 +223,7 @@ class EMEDecryptor final : public MediaDataDecoder, mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); mThroughputLimiter->Flush(); for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { - auto holder = iter.UserData(); + auto* holder = iter.UserData(); holder->DisconnectIfExists(); iter.Remove(); } @@ -240,7 +240,7 @@ class EMEDecryptor final : public MediaDataDecoder, MOZ_ASSERT(mDecodePromise.IsEmpty() && !mDecodeRequest.Exists(), "Must wait for decoding to complete"); for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { - auto holder = iter.UserData(); + auto* holder = iter.UserData(); holder->DisconnectIfExists(); iter.Remove(); } @@ -323,7 +323,7 @@ RefPtr<MediaDataDecoder::DecodePromise> EMEMediaDataDecoderProxy::Decode( mSamplesWaitingForKey->WaitIfKeyNotUsable(sample) ->Then( mThread, __func__, - [self, this](RefPtr<MediaRawData> aSample) { + [self, this](const RefPtr<MediaRawData>& aSample) { mKeyRequest.Complete(); MediaDataDecoderProxy::Decode(aSample) diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp index f01c7e94e4..e9c41be1f0 100644 --- a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp @@ -6,6 +6,9 @@ #include "GMPDecoderModule.h" +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif #include "DecoderDoctorDiagnostics.h" #include "GMPService.h" #include "GMPUtils.h" @@ -43,6 +46,9 @@ static already_AddRefed<MediaDataDecoderProxy> CreateDecoderWrapper( already_AddRefed<MediaDataDecoder> GMPDecoderModule::CreateVideoDecoder( const CreateDecoderParams& aParams) { if (!MP4Decoder::IsH264(aParams.mConfig.mMimeType) && +#ifdef MOZ_AV1 + !AOMDecoder::IsAV1(aParams.mConfig.mMimeType) && +#endif !VPXDecoder::IsVP8(aParams.mConfig.mMimeType) && !VPXDecoder::IsVP9(aParams.mConfig.mMimeType)) { return nullptr; @@ -63,6 +69,10 @@ media::DecodeSupportSet GMPDecoderModule::SupportsMimeType( AutoTArray<nsCString, 2> tags; if (MP4Decoder::IsH264(aMimeType)) { tags.AppendElement("h264"_ns); +#ifdef MOZ_AV1 + } else if (AOMDecoder::IsAV1(aMimeType)) { + tags.AppendElement("av1"_ns); +#endif } else if (VPXDecoder::IsVP9(aMimeType)) { tags.AppendElement("vp9"_ns); } else if (VPXDecoder::IsVP8(aMimeType)) { diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index ed64b62d60..3065ac0c27 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -14,6 +14,9 @@ #include "mozilla/SyncRunnable.h" #include "mozilla/UniquePtr.h" #include "nsTArray.h" +#include "ADTSDemuxer.h" + +#include <array> #define LOG(...) DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__) #define LOGEX(_this, ...) \ @@ -62,6 +65,7 @@ AppleATDecoder::~AppleATDecoder() { RefPtr<MediaDataDecoder::InitPromise> AppleATDecoder::Init() { if (!mFormatID) { + LOG("AppleATDecoder::Init failure: unknown format ID"); return InitPromise::CreateAndReject( MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Non recognised format")), @@ -85,6 +89,7 @@ RefPtr<MediaDataDecoder::FlushPromise> AppleATDecoder::Flush() { } } if (mErrored) { + LOG("Flush error"); mParsedFramesForAACMagicCookie = 0; mMagicCookie.Clear(); ProcessShutdown(); @@ -188,18 +193,28 @@ RefPtr<MediaDataDecoder::DecodePromise> AppleATDecoder::Decode( MediaResult rv = NS_OK; if (!mConverter) { + LOG("Lazily initing the decoder"); rv = SetupDecoder(aSample); if (rv != NS_OK && rv != NS_ERROR_NOT_INITIALIZED) { + LOG("Decoder not initialized"); return DecodePromise::CreateAndReject(rv, __func__); } } + if (mIsADTS) { + bool rv = ADTS::StripHeader(aSample); + if (!rv) { + LOG("Stripping the ADTS header in AppleATDecoder failed"); + } + } + mQueuedSamples.AppendElement(aSample); if (rv == NS_OK) { for (size_t i = 0; i < mQueuedSamples.Length(); i++) { rv = DecodeSample(mQueuedSamples[i]); if (NS_FAILED(rv)) { + LOG("Decoding error"); mErrored = true; return DecodePromise::CreateAndReject(rv, __func__); } @@ -277,7 +292,7 @@ MediaResult AppleATDecoder::DecodeSample(MediaRawData* aSample) { } size_t numFrames = outputData.Length() / channels; - int rate = mOutputFormat.mSampleRate; + int rate = AssertedCast<int>(mOutputFormat.mSampleRate); media::TimeUnit duration(numFrames, rate); if (!duration.IsValid()) { NS_WARNING("Invalid count of accumulated audio samples"); @@ -340,8 +355,8 @@ MediaResult AppleATDecoder::GetInputAudioDescription( aDesc.mChannelsPerFrame = mConfig.mChannels; aDesc.mSampleRate = mConfig.mRate; UInt32 inputFormatSize = sizeof(aDesc); - OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, - &inputFormatSize, &aDesc); + OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, + nullptr, &inputFormatSize, &aDesc); if (NS_WARN_IF(rv)) { return MediaResult( NS_ERROR_FAILURE, @@ -419,7 +434,7 @@ nsresult AppleATDecoder::SetupChannelLayout() { UInt32 propertySize; UInt32 size; OSStatus status = AudioConverterGetPropertyInfo( - mConverter, kAudioConverterOutputChannelLayout, &propertySize, NULL); + mConverter, kAudioConverterOutputChannelLayout, &propertySize, nullptr); if (status || !propertySize) { LOG("Couldn't get channel layout property (%s)", FourCC2Str(status)); return NS_ERROR_FAILURE; @@ -504,15 +519,36 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) { MOZ_ASSERT(mThread->IsOnCurrentThread()); static const uint32_t MAX_FRAMES = 2; + bool isADTS = + ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()}); + + if (isADTS) { + ADTS::FrameParser parser; + if (!parser.Parse(0, aSample->Data(), aSample->Data() + aSample->Size())) { + LOG("ADTS frame parsing error"); + return NS_ERROR_NOT_INITIALIZED; + } + + AudioCodecSpecificBinaryBlob blob; + ADTS::InitAudioSpecificConfig(parser.FirstFrame(), blob.mBinaryBlob); + mConfig.mCodecSpecificConfig = AudioCodecSpecificVariant{std::move(blob)}; + mConfig.mProfile = mConfig.mExtendedProfile = + parser.FirstFrame().Header().mObjectType; + mIsADTS = true; + } + if (mFormatID == kAudioFormatMPEG4AAC && mConfig.mExtendedProfile == 2 && mParsedFramesForAACMagicCookie < MAX_FRAMES) { + LOG("Attempting to get implicit AAC magic cookie"); // Check for implicit SBR signalling if stream is AAC-LC // This will provide us with an updated magic cookie for use with // GetInputAudioDescription. if (NS_SUCCEEDED(GetImplicitAACMagicCookie(aSample)) && - !mMagicCookie.Length()) { + !mMagicCookie.Length() && !isADTS) { // nothing found yet, will try again later + LOG("Getting implicit AAC magic cookie failed"); mParsedFramesForAACMagicCookie++; + LOG("Not initialized -- need magic cookie"); return NS_ERROR_NOT_INITIALIZED; } // An error occurred, fallback to using default stream description @@ -538,6 +574,7 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) { MediaResult rv = GetInputAudioDescription(inputFormat, magicCookie); if (NS_FAILED(rv)) { + LOG("GetInputAudioDescription failure"); return rv; } // Fill in the output format manually. @@ -617,28 +654,41 @@ static void _SampleCallback(void* aSBR, UInt32 aNumBytes, UInt32 aNumPackets, const void* aData, AudioStreamPacketDescription* aPackets) {} -nsresult AppleATDecoder::GetImplicitAACMagicCookie( - const MediaRawData* aSample) { +nsresult AppleATDecoder::GetImplicitAACMagicCookie(MediaRawData* aSample) { MOZ_ASSERT(mThread->IsOnCurrentThread()); - // Prepend ADTS header to AAC audio. - RefPtr<MediaRawData> adtssample(aSample->Clone()); - if (!adtssample) { - return NS_ERROR_OUT_OF_MEMORY; - } - int8_t frequency_index = Adts::GetFrequencyIndex(mConfig.mRate); + bool isADTS = + ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()}); - bool rv = Adts::ConvertSample(mConfig.mChannels, frequency_index, - mConfig.mProfile, adtssample); - if (!rv) { - NS_WARNING("Failed to apply ADTS header"); - return NS_ERROR_FAILURE; + RefPtr<MediaRawData> adtssample = aSample; + + if (!isADTS) { + // Prepend ADTS header to AAC audio. + adtssample = aSample->Clone(); + if (!adtssample) { + return NS_ERROR_OUT_OF_MEMORY; + } + auto frequency_index = ADTS::GetFrequencyIndex(mConfig.mRate); + + if (frequency_index.isErr()) { + LOG("%d isn't a valid rate for AAC", mConfig.mRate); + return NS_ERROR_FAILURE; + } + + // Arbitrarily pick main profile if not specified + int profile = mConfig.mProfile ? mConfig.mProfile : 1; + bool rv = ADTS::ConvertSample(mConfig.mChannels, frequency_index.unwrap(), + profile, adtssample); + if (!rv) { + LOG("Failed to apply ADTS header"); + return NS_ERROR_FAILURE; + } } if (!mStream) { OSStatus rv = AudioFileStreamOpen(this, _MetadataCallback, _SampleCallback, kAudioFileAAC_ADTSType, &mStream); if (rv) { - NS_WARNING("Couldn't open AudioFileStream"); + LOG("Couldn't open AudioFileStream"); return NS_ERROR_FAILURE; } } @@ -646,7 +696,7 @@ nsresult AppleATDecoder::GetImplicitAACMagicCookie( OSStatus status = AudioFileStreamParseBytes( mStream, adtssample->Size(), adtssample->Data(), 0 /* discontinuity */); if (status) { - NS_WARNING("Couldn't parse sample"); + LOG("Couldn't parse sample"); } if (status || mFileStreamError || mMagicCookie.Length()) { diff --git a/dom/media/platforms/apple/AppleATDecoder.h b/dom/media/platforms/apple/AppleATDecoder.h index d7aba2aacb..392b39993f 100644 --- a/dom/media/platforms/apple/AppleATDecoder.h +++ b/dom/media/platforms/apple/AppleATDecoder.h @@ -38,7 +38,7 @@ class AppleATDecoder final : public MediaDataDecoder, nsCString GetCodecName() const override; // Callbacks also need access to the config. - const AudioInfo mConfig; + AudioInfo mConfig; // Use to extract magic cookie for HE-AAC detection. nsTArray<uint8_t> mMagicCookie; @@ -67,11 +67,12 @@ class AppleATDecoder final : public MediaDataDecoder, // Setup AudioConverter once all information required has been gathered. // Will return NS_ERROR_NOT_INITIALIZED if more data is required. MediaResult SetupDecoder(MediaRawData* aSample); - nsresult GetImplicitAACMagicCookie(const MediaRawData* aSample); + nsresult GetImplicitAACMagicCookie(MediaRawData* aSample); nsresult SetupChannelLayout(); uint32_t mParsedFramesForAACMagicCookie; uint32_t mEncoderDelay = 0; uint64_t mTotalMediaFrames = 0; + bool mIsADTS = false; bool mErrored; }; diff --git a/dom/media/platforms/apple/AppleDecoderModule.cpp b/dom/media/platforms/apple/AppleDecoderModule.cpp index 520685fff6..c54593a495 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.cpp +++ b/dom/media/platforms/apple/AppleDecoderModule.cpp @@ -124,8 +124,7 @@ DecodeSupportSet AppleDecoderModule::Supports( case MediaCodec::VP8: [[fallthrough]]; case MediaCodec::VP9: - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_rdd_vpx_enabled() && + if (StaticPrefs::media_rdd_vpx_enabled() && StaticPrefs::media_utility_ffvpx_enabled()) { dss += DecodeSupport::SoftwareDecode; } @@ -233,6 +232,7 @@ bool AppleDecoderModule::CanCreateHWDecoder(MediaCodec aCodec) { /* static */ bool AppleDecoderModule::RegisterSupplementalVP9Decoder() { +#ifdef XP_MACOSX static bool sRegisterIfAvailable = []() { if (__builtin_available(macos 11.0, *)) { VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9); @@ -241,6 +241,9 @@ bool AppleDecoderModule::RegisterSupplementalVP9Decoder() { return false; }(); return sRegisterIfAvailable; +#else // iOS + return false; +#endif } /* static */ diff --git a/dom/media/platforms/apple/AppleVTDecoder.cpp b/dom/media/platforms/apple/AppleVTDecoder.cpp index aae9c1fc9b..ae34c2d142 100644 --- a/dom/media/platforms/apple/AppleVTDecoder.cpp +++ b/dom/media/platforms/apple/AppleVTDecoder.cpp @@ -7,7 +7,7 @@ #include "AppleVTDecoder.h" #include <CoreVideo/CVPixelBufferIOSurface.h> -#include <IOSurface/IOSurface.h> +#include <IOSurface/IOSurfaceRef.h> #include <limits> #include "AppleDecoderModule.h" @@ -486,7 +486,6 @@ void AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage, // Unlock the returned image data. CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); } else { -#ifndef MOZ_WIDGET_UIKIT // Set pixel buffer properties on aImage before we extract its surface. // This ensures that we can use defined enums to set values instead // of later setting magic CFSTR values on the surface itself. @@ -535,9 +534,6 @@ void AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage, info.mDisplay, aFrameRef.byte_offset, aFrameRef.composition_timestamp, aFrameRef.duration, image.forget(), aFrameRef.is_sync_point, aFrameRef.decode_timestamp); -#else - MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); -#endif } if (!data) { @@ -719,7 +715,6 @@ CFDictionaryRef AppleVTDecoder::CreateOutputConfiguration() { &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); } -#ifndef MOZ_WIDGET_UIKIT // Output format type: bool is10Bit = (gfx::BitDepthForColorDepth(mColorDepth) == 10); @@ -754,9 +749,6 @@ CFDictionaryRef AppleVTDecoder::CreateOutputConfiguration() { return CFDictionaryCreate( kCFAllocatorDefault, outputKeys, outputValues, ArrayLength(outputKeys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); -#else - MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); -#endif } } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp index 43041f81ea..1e8e488e25 100644 --- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp @@ -164,7 +164,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int16_t* data = reinterpret_cast<int16_t**>(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(*data++); + *tmp++ = ConvertAudioSample<float>(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_S16P) { @@ -174,7 +174,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int16_t** data = reinterpret_cast<int16_t**>(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(data[channel][frame]); + *tmp++ = ConvertAudioSample<float>(data[channel][frame]); } } } else if (aFrame->format == AV_SAMPLE_FMT_S32) { @@ -183,7 +183,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int32_t* data = reinterpret_cast<int32_t**>(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(*data++); + *tmp++ = ConvertAudioSample<float>(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_S32P) { @@ -193,7 +193,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int32_t** data = reinterpret_cast<int32_t**>(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(data[channel][frame]); + *tmp++ = ConvertAudioSample<float>(data[channel][frame]); } } } else if (aFrame->format == AV_SAMPLE_FMT_U8) { @@ -202,7 +202,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, uint8_t* data = reinterpret_cast<uint8_t**>(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = UInt8bitToAudioSample<AudioDataValue>(*data++); + *tmp++ = ConvertAudioSample<float>(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_U8P) { @@ -212,7 +212,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, uint8_t** data = reinterpret_cast<uint8_t**>(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = UInt8bitToAudioSample<AudioDataValue>(data[channel][frame]); + *tmp++ = ConvertAudioSample<float>(data[channel][frame]); } } } diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp index 4a30f2dd2d..bfb3105a57 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp @@ -177,9 +177,6 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC(av_packet_alloc, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) AV_FUNC(av_packet_unref, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) AV_FUNC(av_packet_free, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) - AV_FUNC_OPTION(av_rdft_init, AV_FUNC_AVCODEC_ALL) - AV_FUNC_OPTION(av_rdft_calc, AV_FUNC_AVCODEC_ALL) - AV_FUNC_OPTION(av_rdft_end, AV_FUNC_AVCODEC_ALL) AV_FUNC(avcodec_descriptor_get, AV_FUNC_AVCODEC_ALL) AV_FUNC(av_log_set_level, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_malloc, AV_FUNC_AVUTIL_ALL) @@ -254,6 +251,10 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC_AVUTIL_59 | AV_FUNC_AVUTIL_60) #endif + + AV_FUNC_OPTION(av_tx_init, AV_FUNC_AVUTIL_ALL) + AV_FUNC_OPTION(av_tx_uninit, AV_FUNC_AVUTIL_ALL) + #undef AV_FUNC #undef AV_FUNC_OPTION diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h index 98ab2f7930..eacbba286a 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h @@ -5,9 +5,9 @@ #ifndef __FFmpegLibWrapper_h__ #define __FFmpegLibWrapper_h__ -#include "FFmpegRDFTTypes.h" // for AvRdftInitFn, etc. #include "mozilla/Attributes.h" #include "mozilla/Types.h" +#include "ffvpx/tx.h" struct AVCodec; struct AVCodecContext; @@ -148,11 +148,6 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper { int (*avcodec_send_frame)(AVCodecContext* avctx, const AVFrame* frame); int (*avcodec_receive_frame)(AVCodecContext* avctx, AVFrame* frame); - // libavcodec optional - AvRdftInitFn av_rdft_init; - AvRdftCalcFn av_rdft_calc; - AvRdftEndFn av_rdft_end; - // libavutil void (*av_log_set_level)(int level); void* (*av_malloc)(size_t size); @@ -216,6 +211,10 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper { void* (*vaGetDisplayDRM)(int fd); #endif + // Only ever used with ffvpx + decltype(::av_tx_init)* av_tx_init; + decltype(::av_tx_uninit)* av_tx_uninit; + PRLibrary* mAVCodecLib; PRLibrary* mAVUtilLib; #ifdef MOZ_WIDGET_GTK diff --git a/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h b/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h deleted file mode 100644 index cb3e2476fb..0000000000 --- a/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -#ifndef FFmpegRDFTTypes_h -#define FFmpegRDFTTypes_h - -struct RDFTContext; - -typedef float FFTSample; - -enum RDFTransformType { - DFT_R2C, - IDFT_C2R, - IDFT_R2C, - DFT_C2R, -}; - -extern "C" { - -typedef RDFTContext* (*AvRdftInitFn)(int nbits, enum RDFTransformType trans); -typedef void (*AvRdftCalcFn)(RDFTContext* s, FFTSample* data); -typedef void (*AvRdftEndFn)(RDFTContext* s); -} - -struct FFmpegRDFTFuncs { - AvRdftInitFn init; - AvRdftCalcFn calc; - AvRdftEndFn end; -}; - -#endif // FFmpegRDFTTypes_h diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp index dcc3d9a88d..a3cfdf1b1d 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp @@ -504,128 +504,64 @@ MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { mCodecContext->flags |= AV_CODEC_FLAG_FRAME_DURATION; #endif mCodecContext->gop_size = static_cast<int>(mConfig.mKeyframeInterval); - // TODO (bug 1872871): Move the following extra settings to some helpers - // instead. + if (mConfig.mUsage == MediaDataEncoder::Usage::Realtime) { mLib->av_opt_set(mCodecContext->priv_data, "deadline", "realtime", 0); // Explicitly ask encoder do not keep in flight at any one time for // lookahead purposes. mLib->av_opt_set(mCodecContext->priv_data, "lag-in-frames", "0", 0); } - // Apply SVC settings. - if (Maybe<VPXSVCSetting> svc = - GetVPXSVCSetting(mConfig.mScalabilityMode, mConfig.mBitrate)) { - // For libvpx. - if (mCodecName == "libvpx" || mCodecName == "libvpx-vp9") { - // Show a warning if mScalabilityMode mismatches mNumTemporalLayers - if (mConfig.mCodecSpecific) { - if (mConfig.mCodecSpecific->is<VP8Specific>() || - mConfig.mCodecSpecific->is<VP9Specific>()) { - const uint8_t numTemporalLayers = - mConfig.mCodecSpecific->is<VP8Specific>() - ? mConfig.mCodecSpecific->as<VP8Specific>().mNumTemporalLayers - : mConfig.mCodecSpecific->as<VP9Specific>() - .mNumTemporalLayers; - if (numTemporalLayers != svc->mNumberLayers) { - FFMPEGV_LOG( - "Force using %zu layers defined in scalability mode instead of " - "the %u layers defined in VP8/9Specific", - svc->mNumberLayers, numTemporalLayers); - } - } - } - // Set ts_layering_mode. - nsPrintfCString parameters("ts_layering_mode=%u", svc->mLayeringMode); - // Set ts_target_bitrate. - parameters.Append(":ts_target_bitrate="); - for (size_t i = 0; i < svc->mTargetBitrates.Length(); ++i) { - if (i > 0) { - parameters.Append(","); - } - parameters.AppendPrintf("%d", svc->mTargetBitrates[i]); - } - // TODO: Set ts_number_layers, ts_periodicity, ts_layer_id and - // ts_rate_decimator if they are different from the preset values in - // ts_layering_mode. - - // Set parameters into ts-parameters. - mLib->av_opt_set(mCodecContext->priv_data, "ts-parameters", - parameters.get(), 0); - - // FFmpegVideoEncoder would be reset after Drain(), so mSVCInfo should be - // reset() before emplace(). - mSVCInfo.reset(); - mSVCInfo.emplace(std::move(svc->mLayerIds)); - - // TODO: layer settings should be changed dynamically when the frame's - // color space changed. - } else { - FFMPEGV_LOG("SVC setting is not implemented for %s codec", - mCodecName.get()); - } + if (Maybe<SVCSettings> settings = GetSVCSettings()) { + SVCSettings s = settings.extract(); + mLib->av_opt_set(mCodecContext->priv_data, s.mSettingKeyValue.first.get(), + s.mSettingKeyValue.second.get(), 0); + + // FFmpegVideoEncoder is reset after Drain(), so mSVCInfo should be reset() + // before emplace(). + mSVCInfo.reset(); + mSVCInfo.emplace(std::move(s.mTemporalLayerIds)); + + // TODO: layer settings should be changed dynamically when the frame's + // color space changed. } - // Apply codec specific settings. - nsAutoCString codecSpecificLog; - if (mConfig.mCodecSpecific) { - if (mConfig.mCodecSpecific->is<H264Specific>()) { - // For libx264. - if (mCodecName == "libx264") { - codecSpecificLog.Append(", H264:"); - - const H264Specific& specific = - mConfig.mCodecSpecific->as<H264Specific>(); - - // Set profile. - Maybe<H264Setting> profile = GetH264Profile(specific.mProfile); - if (!profile) { - FFMPEGV_LOG("failed to get h264 profile"); - return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, - RESULT_DETAIL("H264 profile is unknown")); - } - codecSpecificLog.Append( - nsPrintfCString(" profile - %d", profile->mValue)); - mCodecContext->profile = profile->mValue; - if (!profile->mString.IsEmpty()) { - codecSpecificLog.AppendPrintf(" (%s)", profile->mString.get()); - mLib->av_opt_set(mCodecContext->priv_data, "profile", - profile->mString.get(), 0); - } - - // Set level. - Maybe<H264Setting> level = GetH264Level(specific.mLevel); - if (!level) { - FFMPEGV_LOG("failed to get h264 level"); - return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, - RESULT_DETAIL("H264 level is unknown")); - } - codecSpecificLog.AppendPrintf(", level %d (%s)", level->mValue, - level->mString.get()); - mCodecContext->level = level->mValue; - MOZ_ASSERT(!level->mString.IsEmpty()); - mLib->av_opt_set(mCodecContext->priv_data, "level", - level->mString.get(), 0); - - // Set format: libx264's default format is annexb - if (specific.mFormat == H264BitStreamFormat::AVC) { - codecSpecificLog.Append(", AVCC"); - mLib->av_opt_set(mCodecContext->priv_data, "x264-params", "annexb=0", - 0); - // mCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER - // if we don't want to append SPS/PPS data in all keyframe - // (LIBAVCODEC_VERSION_MAJOR >= 57 only). - } else { - codecSpecificLog.Append(", AnnexB"); - // Set annexb explicitly even if it's default format. - mLib->av_opt_set(mCodecContext->priv_data, "x264-params", "annexb=1", - 0); - } - } else { - FFMPEGV_LOG("H264 settings is not implemented for codec %s ", - mCodecName.get()); + + nsAutoCString h264Log; + if (mConfig.mCodecSpecific && mConfig.mCodecSpecific->is<H264Specific>()) { + // TODO: Set profile, level, avcc/annexb for openh264 and others. + if (mCodecName == "libx264") { + const H264Specific& h264Specific = + mConfig.mCodecSpecific->as<H264Specific>(); + H264Settings s = GetH264Settings(h264Specific); + mCodecContext->profile = s.mProfile; + mCodecContext->level = s.mLevel; + for (const auto& pair : s.mSettingKeyValuePairs) { + mLib->av_opt_set(mCodecContext->priv_data, pair.first.get(), + pair.second.get(), 0); } + + // Log the settings. + // When using profile other than EXTENDED, the profile string is in the + // first element of mSettingKeyValuePairs, while EXTENDED profile has no + // profile string. + + MOZ_ASSERT_IF( + s.mSettingKeyValuePairs.Length() != 3, + h264Specific.mProfile == H264_PROFILE::H264_PROFILE_EXTENDED); + const char* profileStr = s.mSettingKeyValuePairs.Length() == 3 + ? s.mSettingKeyValuePairs[0].second.get() + : "extended"; + const char* levelStr = s.mSettingKeyValuePairs.Length() == 3 + ? s.mSettingKeyValuePairs[1].second.get() + : s.mSettingKeyValuePairs[0].second.get(); + const char* formatStr = + h264Specific.mFormat == H264BitStreamFormat::AVC ? "AVCC" : "AnnexB"; + h264Log.AppendPrintf(", H264: profile - %d (%s), level %d (%s), %s", + mCodecContext->profile, profileStr, + mCodecContext->level, levelStr, formatStr); } } + // TODO: keyint_min, max_b_frame? // - if mConfig.mDenoising is set: av_opt_set_int(mCodecContext->priv_data, // "noise_sensitivity", x, 0), where the x is from 0(disabled) to 6. @@ -657,7 +593,7 @@ MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { static_cast<int64_t>(mCodecContext->bit_rate), mCodecContext->width, mCodecContext->height, mCodecContext->time_base.num, mCodecContext->time_base.den, - codecSpecificLog.IsEmpty() ? "" : codecSpecificLog.get()); + h264Log.IsEmpty() ? "" : h264Log.get()); return MediaResult(NS_OK); } @@ -1152,4 +1088,99 @@ void FFmpegVideoEncoder<LIBAV_VER>::ForceEnablingFFmpegDebugLogs() { #endif // DEBUG } +Maybe<FFmpegVideoEncoder<LIBAV_VER>::SVCSettings> +FFmpegVideoEncoder<LIBAV_VER>::GetSVCSettings() { + MOZ_ASSERT(!mCodecName.IsEmpty()); + + // TODO: Add support for AV1 and H264. + if (mCodecName != "libvpx" && mCodecName != "libvpx-vp9") { + FFMPEGV_LOG("SVC setting is not implemented for %s codec", + mCodecName.get()); + return Nothing(); + } + + Maybe<VPXSVCSetting> svc = + GetVPXSVCSetting(mConfig.mScalabilityMode, mConfig.mBitrate); + if (!svc) { + FFMPEGV_LOG("No SVC settings obtained. Skip"); + return Nothing(); + } + + // Check if the number of temporal layers in codec specific settings matches + // the number of layers for the given scalability mode. + + auto GetNumTemporalLayers = [&]() -> uint8_t { + uint8_t layers = 0; + if (mConfig.mCodecSpecific) { + if (mConfig.mCodecSpecific->is<VP8Specific>()) { + layers = mConfig.mCodecSpecific->as<VP8Specific>().mNumTemporalLayers; + MOZ_ASSERT(layers > 0); + } else if (mConfig.mCodecSpecific->is<VP9Specific>()) { + layers = mConfig.mCodecSpecific->as<VP9Specific>().mNumTemporalLayers; + MOZ_ASSERT(layers > 0); + } + } + return layers; + }; + + DebugOnly<uint8_t> numTemporalLayers = GetNumTemporalLayers(); + MOZ_ASSERT_IF(numTemporalLayers > 0, numTemporalLayers == svc->mNumberLayers); + + // Form an SVC setting string for libvpx. + + nsPrintfCString parameters("ts_layering_mode=%u", svc->mLayeringMode); + parameters.Append(":ts_target_bitrate="); + for (size_t i = 0; i < svc->mTargetBitrates.Length(); ++i) { + if (i > 0) { + parameters.Append(","); + } + parameters.AppendPrintf("%d", svc->mTargetBitrates[i]); + } + + // TODO: Set ts_number_layers, ts_periodicity, ts_layer_id and + // ts_rate_decimator if they are different from the preset values in + // ts_layering_mode. + + return Some( + SVCSettings{std::move(svc->mLayerIds), + std::make_pair("ts-parameters"_ns, std::move(parameters))}); +} + +FFmpegVideoEncoder<LIBAV_VER>::H264Settings FFmpegVideoEncoder< + LIBAV_VER>::GetH264Settings(const H264Specific& aH264Specific) { + MOZ_ASSERT(mCodecName == "libx264", + "GetH264Settings is libx264-only for now"); + + nsTArray<std::pair<nsCString, nsCString>> keyValuePairs; + + Maybe<H264Setting> profile = GetH264Profile(aH264Specific.mProfile); + MOZ_RELEASE_ASSERT(profile.isSome()); + if (!profile->mString.IsEmpty()) { + keyValuePairs.AppendElement(std::make_pair("profile"_ns, profile->mString)); + } else { + MOZ_RELEASE_ASSERT(aH264Specific.mProfile == + H264_PROFILE::H264_PROFILE_EXTENDED); + } + + Maybe<H264Setting> level = GetH264Level(aH264Specific.mLevel); + MOZ_RELEASE_ASSERT(level.isSome()); + MOZ_RELEASE_ASSERT(!level->mString.IsEmpty()); + keyValuePairs.AppendElement(std::make_pair("level"_ns, level->mString)); + + // Set format: libx264's default format is annexb. + if (aH264Specific.mFormat == H264BitStreamFormat::AVC) { + keyValuePairs.AppendElement(std::make_pair("x264-params"_ns, "annexb=0")); + // mCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER + // if we don't want to append SPS/PPS data in all keyframe + // (LIBAVCODEC_VERSION_MAJOR >= 57 only). + } else { + // Set annexb explicitly even if it's default format. + keyValuePairs.AppendElement(std::make_pair("x264-params"_ns, "annexb=1")); + } + + return H264Settings{.mProfile = profile->mValue, + .mLevel = level->mValue, + .mSettingKeyValuePairs = std::move(keyValuePairs)}; +} + } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h index 1bcdd3eaf9..07c433ddd7 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h @@ -75,6 +75,19 @@ class FFmpegVideoEncoder<LIBAV_VER> final : public MediaDataEncoder { Result<already_AddRefed<MediaByteBuffer>, nsresult> GetExtraData( AVPacket* aPacket); void ForceEnablingFFmpegDebugLogs(); + struct SVCSettings { + nsTArray<uint8_t> mTemporalLayerIds; + // A key-value pair for av_opt_set. + std::pair<nsCString, nsCString> mSettingKeyValue; + }; + Maybe<SVCSettings> GetSVCSettings(); + struct H264Settings { + int mProfile; + int mLevel; + // A list of key-value pairs for av_opt_set. + nsTArray<std::pair<nsCString, nsCString>> mSettingKeyValuePairs; + }; + H264Settings GetH264Settings(const H264Specific& aH264Specific); // This refers to a static FFmpegLibWrapper, so raw pointer is adequate. const FFmpegLibWrapper* mLib; diff --git a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp index ba9ca4834e..dfc8244f1d 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp +++ b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp @@ -145,19 +145,13 @@ already_AddRefed<PlatformEncoderModule> FFVPXRuntimeLinker::CreateEncoder() { } /* static */ -void FFVPXRuntimeLinker::GetRDFTFuncs(FFmpegRDFTFuncs* aOutFuncs) { +void FFVPXRuntimeLinker::GetFFTFuncs(FFmpegFFTFuncs* aOutFuncs) { []() MOZ_NO_THREAD_SAFETY_ANALYSIS { MOZ_ASSERT(sLinkStatus != LinkStatus_INIT); }(); - if (sFFVPXLib.av_rdft_init && sFFVPXLib.av_rdft_calc && - sFFVPXLib.av_rdft_end) { - aOutFuncs->init = sFFVPXLib.av_rdft_init; - aOutFuncs->calc = sFFVPXLib.av_rdft_calc; - aOutFuncs->end = sFFVPXLib.av_rdft_end; - } else { - NS_WARNING("RDFT functions expected but not found"); - *aOutFuncs = FFmpegRDFTFuncs(); // zero - } + MOZ_ASSERT(sFFVPXLib.av_tx_init && sFFVPXLib.av_tx_uninit); + aOutFuncs->init = sFFVPXLib.av_tx_init; + aOutFuncs->uninit = sFFVPXLib.av_tx_uninit; } } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h index e52f108272..dccd37c7da 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h +++ b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h @@ -11,8 +11,12 @@ #include "PlatformEncoderModule.h" #include "mozilla/StaticMutex.h" #include "mozilla/ThreadSafety.h" +#include "ffvpx/tx.h" -struct FFmpegRDFTFuncs; +struct FFmpegFFTFuncs { + decltype(av_tx_init)* init; + decltype(av_tx_uninit)* uninit; +}; namespace mozilla { @@ -23,7 +27,7 @@ class FFVPXRuntimeLinker { static already_AddRefed<PlatformEncoderModule> CreateEncoder(); // Call (on any thread) after Init(). - static void GetRDFTFuncs(FFmpegRDFTFuncs* aOutFuncs); + static void GetFFTFuncs(FFmpegFFTFuncs* aOutFuncs); private: // Provide critical-section for Init() and sLinkStatus. diff --git a/dom/media/platforms/moz.build b/dom/media/platforms/moz.build index 34acf4e8d1..6f71c5cc12 100644 --- a/dom/media/platforms/moz.build +++ b/dom/media/platforms/moz.build @@ -47,19 +47,13 @@ DIRS += ["agnostic/bytestreams", "agnostic/eme", "agnostic/gmp", "omx"] if CONFIG["MOZ_WMF"]: DIRS += ["wmf"] -if CONFIG["MOZ_FFVPX"] or CONFIG["MOZ_FFMPEG"]: - # common code to either FFmpeg or FFVPX - EXPORTS += [ - "ffmpeg/FFmpegRDFTTypes.h", - ] - UNIFIED_SOURCES += [ - "ffmpeg/FFmpegLibWrapper.cpp", - ] +UNIFIED_SOURCES += [ + "ffmpeg/FFmpegLibWrapper.cpp", +] -if CONFIG["MOZ_FFVPX"]: - DIRS += [ - "ffmpeg/ffvpx", - ] +DIRS += [ + "ffmpeg/ffvpx", +] if CONFIG["MOZ_FFMPEG"]: DIRS += [ diff --git a/dom/media/platforms/wmf/MFCDMSession.cpp b/dom/media/platforms/wmf/MFCDMSession.cpp index cec783cbc6..b797898abb 100644 --- a/dom/media/platforms/wmf/MFCDMSession.cpp +++ b/dom/media/platforms/wmf/MFCDMSession.cpp @@ -304,8 +304,7 @@ void MFCDMSession::OnSessionKeyMessage( case MF_MEDIAKEYSESSION_MESSAGETYPE_INDIVIDUALIZATION_REQUEST: return dom::MediaKeyMessageType::Individualization_request; default: - MOZ_ASSERT_UNREACHABLE("Unknown session message type"); - return dom::MediaKeyMessageType::EndGuard_; + MOZ_CRASH("Unknown session message type"); } }; LOG("Notify 'keymessage' for %s", NS_ConvertUTF16toUTF8(*mSessionId).get()); diff --git a/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp index 4acf26e041..969f817882 100644 --- a/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp @@ -93,7 +93,7 @@ HRESULT MFMediaEngineAudioStream::CreateMediaType(const TrackInfo& aInfo, bool MFMediaEngineAudioStream::HasEnoughRawData() const { // If more than this much raw audio is queued, we'll hold off request more // audio. - return mRawDataQueueForFeedingEngine.Duration() >= + return mRawDataQueueForFeedingEngine.PreciseDuration() >= StaticPrefs::media_wmf_media_engine_raw_data_threshold_audio(); } diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp index 5b99fb0f2c..e291ab6a54 100644 --- a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp @@ -6,6 +6,7 @@ #include "MFTDecoder.h" #include "VideoUtils.h" +#include "mozilla/gfx/gfxVars.h" #include "mozilla/MFMediaEngineParent.h" #include "mozilla/MFMediaEngineUtils.h" #include "mozilla/RemoteDecoderManagerChild.h" @@ -99,6 +100,11 @@ media::DecodeSupportSet MFMediaEngineDecoderModule::SupportInternal( if (!StaticPrefs::media_wmf_media_engine_enabled()) { return media::DecodeSupportSet{}; } + // Only support hardware decoding. + if (!gfx::gfxVars::CanUseHardwareVideoDecoding() && + !StaticPrefs::media_wmf_media_engine_bypass_gfx_blocklist()) { + return media::DecodeSupportSet{}; + } bool supports = false; WMFStreamType type = GetStreamTypeFromMimeType(aParams.MimeType()); if (type != WMFStreamType::Unknown) { @@ -107,13 +113,11 @@ media::DecodeSupportSet MFMediaEngineDecoderModule::SupportInternal( MOZ_LOG(sPDMLog, LogLevel::Debug, ("MFMediaEngine decoder %s requested type '%s'", supports ? "supports" : "rejects", aParams.MimeType().get())); - // We only support HEVC hardware decoding. - if (supports && type == WMFStreamType::HEVC) { - return media::DecodeSupport::HardwareDecode; + if (!supports) { + return media::DecodeSupportSet{}; } - // TODO : find a way to report accurate result. - return supports ? media::DecodeSupport::SoftwareDecode - : media::DecodeSupportSet{}; + return StreamTypeIsVideo(type) ? media::DecodeSupport::HardwareDecode + : media::DecodeSupport::SoftwareDecode; } static bool CreateMFTDecoderOnMTA(const WMFStreamType& aType) { diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h index c23b9010cc..1c8de5a161 100644 --- a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h @@ -10,6 +10,8 @@ namespace mozilla { +// MFMediaEngineDecoderModule is used for the media engine playback, which only +// supports hardware decoding. class MFMediaEngineDecoderModule final : public PlatformDecoderModule { public: static void Init(); diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.cpp b/dom/media/platforms/wmf/MFMediaEngineStream.cpp index 6dce37ee35..70ffa50142 100644 --- a/dom/media/platforms/wmf/MFMediaEngineStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineStream.cpp @@ -107,7 +107,11 @@ MFMediaEngineStreamWrapper::NeedsConversion() const { } MFMediaEngineStream::MFMediaEngineStream() - : mIsShutdown(false), mIsSelected(false), mReceivedEOS(false) { + : mIsShutdown(false), + mIsSelected(false), + mRawDataQueueForFeedingEngine(true /* aEnablePreciseDuration */), + mRawDataQueueForGeneratingOutput(true /* aEnablePreciseDuration */), + mReceivedEOS(false) { MOZ_COUNT_CTOR(MFMediaEngineStream); } @@ -282,17 +286,8 @@ void MFMediaEngineStream::ReplySampleRequestIfPossible() { while (!mSampleRequestTokens.empty()) { mSampleRequestTokens.pop(); } - - SLOG("Notify end events"); - MOZ_ASSERT(mRawDataQueueForFeedingEngine.GetSize() == 0); MOZ_ASSERT(mSampleRequestTokens.empty()); - RETURN_VOID_IF_FAILED(mMediaEventQueue->QueueEventParamUnk( - MEEndOfStream, GUID_NULL, S_OK, nullptr)); - mEndedEvent.Notify(TrackType()); - PROFILER_MARKER_TEXT( - "MFMediaEngineStream:NotifyEnd", MEDIA_PLAYBACK, {}, - nsPrintfCString("stream=%s, id=%" PRIu64, GetDescriptionName().get(), - mStreamId)); + NotifyEndEvent(); return; } @@ -318,6 +313,18 @@ void MFMediaEngineStream::ReplySampleRequestIfPossible() { MEMediaSample, GUID_NULL, S_OK, inputSample.Get())); } +void MFMediaEngineStream::NotifyEndEvent() { + AssertOnTaskQueue(); + SLOG("Notify end event"); + MOZ_ASSERT(mRawDataQueueForFeedingEngine.GetSize() == 0); + RETURN_VOID_IF_FAILED(mMediaEventQueue->QueueEventParamUnk( + MEEndOfStream, GUID_NULL, S_OK, nullptr)); + mEndedEvent.Notify(TrackType()); + PROFILER_MARKER_TEXT("MFMediaEngineStream:NotifyEnd", MEDIA_PLAYBACK, {}, + nsPrintfCString("stream=%s, id=%" PRIu64, + GetDescriptionName().get(), mStreamId)); +} + bool MFMediaEngineStream::ShouldServeSamples() const { AssertOnTaskQueue(); return mParentSource && @@ -486,7 +493,7 @@ void MFMediaEngineStream::NotifyNewData(MediaRawData* aSample) { "], queue size=%zu, queue duration=%" PRId64, aSample->mTime.ToMicroseconds(), aSample->GetEndTime().ToMicroseconds(), mRawDataQueueForFeedingEngine.GetSize(), - mRawDataQueueForFeedingEngine.Duration()); + mRawDataQueueForFeedingEngine.PreciseDuration()); if (mReceivedEOS) { SLOG("Receive a new data, cancel old EOS flag"); mReceivedEOS = false; @@ -501,7 +508,7 @@ void MFMediaEngineStream::SendRequestSampleEvent(bool aIsEnough) { AssertOnTaskQueue(); SLOGV("data is %s, queue duration=%" PRId64, aIsEnough ? "enough" : "not enough", - mRawDataQueueForFeedingEngine.Duration()); + mRawDataQueueForFeedingEngine.PreciseDuration()); mParentSource->mRequestSampleEvent.Notify( SampleRequest{TrackType(), aIsEnough}); } diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.h b/dom/media/platforms/wmf/MFMediaEngineStream.h index aa3bf7e65d..e11d900498 100644 --- a/dom/media/platforms/wmf/MFMediaEngineStream.h +++ b/dom/media/platforms/wmf/MFMediaEngineStream.h @@ -84,7 +84,7 @@ class MFMediaEngineStream // Return the type of the track, the result should be either audio or video. virtual TrackInfo::TrackType TrackType() = 0; - RefPtr<MediaDataDecoder::FlushPromise> Flush(); + virtual RefPtr<MediaDataDecoder::FlushPromise> Flush(); MediaEventProducer<TrackInfo::TrackType>& EndedEvent() { return mEndedEvent; } @@ -93,7 +93,7 @@ class MFMediaEngineStream virtual MFMediaEngineVideoStream* AsVideoStream() { return nullptr; } - RefPtr<MediaDataDecoder::DecodePromise> OutputData( + virtual RefPtr<MediaDataDecoder::DecodePromise> OutputData( RefPtr<MediaRawData> aSample); virtual RefPtr<MediaDataDecoder::DecodePromise> Drain(); @@ -133,11 +133,13 @@ class MFMediaEngineStream // should uses `mRawDataQueueForGeneratingOutput` to generate output. virtual already_AddRefed<MediaData> OutputDataInternal() = 0; - void SendRequestSampleEvent(bool aIsEnough); + virtual void SendRequestSampleEvent(bool aIsEnough); HRESULT AddEncryptAttributes(IMFSample* aSample, const CryptoSample& aCryptoConfig); + void NotifyEndEvent(); + void AssertOnTaskQueue() const; void AssertOnMFThreadPool() const; diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp index ca043478f0..0fedcd31b9 100644 --- a/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp @@ -49,7 +49,7 @@ void MFMediaEngineVideoStream::SetKnowsCompositor( this]() { mKnowsCompositor = knowCompositor; LOG("Set SetKnowsCompositor=%p", mKnowsCompositor.get()); - ResolvePendingDrainPromiseIfNeeded(); + ResolvePendingPromisesIfNeeded(); })); } @@ -74,7 +74,7 @@ void MFMediaEngineVideoStream::SetDCompSurfaceHandle(HANDLE aDCompSurfaceHandle, } } LOG("Set DCompSurfaceHandle, handle=%p", mDCompSurfaceHandle); - ResolvePendingDrainPromiseIfNeeded(); + ResolvePendingPromisesIfNeeded(); })); } @@ -209,7 +209,7 @@ HRESULT MFMediaEngineVideoStream::CreateMediaType(const TrackInfo& aInfo, bool MFMediaEngineVideoStream::HasEnoughRawData() const { // If more than this much raw video is queued, we'll hold off request more // video. - return mRawDataQueueForFeedingEngine.Duration() >= + return mRawDataQueueForFeedingEngine.PreciseDuration() >= StaticPrefs::media_wmf_media_engine_raw_data_threshold_video(); } @@ -240,6 +240,32 @@ bool MFMediaEngineVideoStream::IsDCompImageReady() { return true; } +RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::OutputData( + RefPtr<MediaRawData> aSample) { + if (IsShutdown()) { + return MediaDataDecoder::DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_FAILURE, + RESULT_DETAIL("MFMediaEngineStream is shutdown")), + __func__); + } + AssertOnTaskQueue(); + NotifyNewData(aSample); + MediaDataDecoder::DecodedData outputs; + if (RefPtr<MediaData> outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + if (ShouldDelayVideoDecodeBeforeDcompReady()) { + LOG("Dcomp isn't ready and we already have enough video data. We will send " + "them back together at one when Dcomp is ready"); + return mVideoDecodeBeforeDcompPromise.Ensure(__func__); + } + return MediaDataDecoder::DecodePromise::CreateAndResolve(std::move(outputs), + __func__); +} + already_AddRefed<MediaData> MFMediaEngineVideoStream::OutputDataInternal() { AssertOnTaskQueue(); if (mRawDataQueueForGeneratingOutput.GetSize() == 0 || !IsDCompImageReady()) { @@ -261,28 +287,62 @@ RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::Drain() { MediaDataDecoder::DecodedData outputs; if (!IsDCompImageReady()) { LOGV("Waiting for dcomp image for draining"); + // A workaround for a special case where we have sent all input data to the + // media engine, and waiting for an output. Sometime media engine would + // never return the first frame to us, unless we notify it the end event, + // which happens on the case where the video only contains one frame. If we + // don't send end event to the media engine, the drain promise would be + // pending forever. + if (!mSampleRequestTokens.empty() && + mRawDataQueueForFeedingEngine.GetSize() == 0) { + NotifyEndEvent(); + } return mPendingDrainPromise.Ensure(__func__); } return MFMediaEngineStream::Drain(); } -void MFMediaEngineVideoStream::ResolvePendingDrainPromiseIfNeeded() { +RefPtr<MediaDataDecoder::FlushPromise> MFMediaEngineVideoStream::Flush() { + AssertOnTaskQueue(); + auto promise = MFMediaEngineStream::Flush(); + mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); + return promise; +} + +void MFMediaEngineVideoStream::ResolvePendingPromisesIfNeeded() { AssertOnTaskQueue(); - if (mPendingDrainPromise.IsEmpty()) { - return; - } if (!IsDCompImageReady()) { return; } - MediaDataDecoder::DecodedData outputs; - while (RefPtr<MediaData> outputData = OutputDataInternal()) { - outputs.AppendElement(outputData); - LOGV("Output data [%" PRId64 ",%" PRId64 "]", - outputData->mTime.ToMicroseconds(), - outputData->GetEndTime().ToMicroseconds()); + + // Resolve decoding promise first, then drain promise + if (!mVideoDecodeBeforeDcompPromise.IsEmpty()) { + MediaDataDecoder::DecodedData outputs; + while (RefPtr<MediaData> outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + mVideoDecodeBeforeDcompPromise.Resolve(std::move(outputs), __func__); + LOG("Resolved video decode before Dcomp promise"); + } + + // This drain promise could return no data, if all data has been processed in + // the decoding promise. + if (!mPendingDrainPromise.IsEmpty()) { + MediaDataDecoder::DecodedData outputs; + while (RefPtr<MediaData> outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + mPendingDrainPromise.Resolve(std::move(outputs), __func__); + LOG("Resolved pending drain promise"); } - mPendingDrainPromise.Resolve(std::move(outputs), __func__); - LOG("Resolved pending drain promise"); } MediaDataDecoder::ConversionRequired MFMediaEngineVideoStream::NeedsConversion() @@ -336,6 +396,20 @@ void MFMediaEngineVideoStream::UpdateConfig(const VideoInfo& aInfo) { void MFMediaEngineVideoStream::ShutdownCleanUpOnTaskQueue() { AssertOnTaskQueue(); mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); +} + +void MFMediaEngineVideoStream::SendRequestSampleEvent(bool aIsEnough) { + AssertOnTaskQueue(); + MFMediaEngineStream::SendRequestSampleEvent(aIsEnough); + // We need more data to be sent in, we should resolve the promise to allow + // more input data to be sent. + if (!aIsEnough && !mVideoDecodeBeforeDcompPromise.IsEmpty()) { + LOG("Resolved pending input promise to allow more input be sent in"); + mVideoDecodeBeforeDcompPromise.Resolve(MediaDataDecoder::DecodedData{}, + __func__); + } } bool MFMediaEngineVideoStream::IsEnded() const { @@ -352,6 +426,10 @@ bool MFMediaEngineVideoStream::IsEnded() const { bool MFMediaEngineVideoStream::IsEncrypted() const { return mIsEncrypted; } +bool MFMediaEngineVideoStream::ShouldDelayVideoDecodeBeforeDcompReady() { + return HasEnoughRawData() && !IsDCompImageReady(); +} + nsCString MFMediaEngineVideoStream::GetCodecName() const { switch (mStreamType) { case WMFStreamType::H264: diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.h b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h index df17c264e4..51fbe4876b 100644 --- a/dom/media/platforms/wmf/MFMediaEngineVideoStream.h +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h @@ -19,6 +19,7 @@ class DcompSurfaceImage; } // namespace layers class MFMediaSource; +class MediaRawData; class MFMediaEngineVideoStream final : public MFMediaEngineStream { public: @@ -50,8 +51,13 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { // change happens during playback. void SetConfig(const TrackInfo& aConfig); + RefPtr<MediaDataDecoder::DecodePromise> OutputData( + RefPtr<MediaRawData> aSample) override; + RefPtr<MediaDataDecoder::DecodePromise> Drain() override; + RefPtr<MediaDataDecoder::FlushPromise> Flush() override; + bool IsEncrypted() const override; private: @@ -66,12 +72,25 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { bool IsDCompImageReady(); - void ResolvePendingDrainPromiseIfNeeded(); + // Those promises are used to handle decode/drain which happens before the + // Dcomp surface is ready. + void ResolvePendingPromisesIfNeeded(); void ShutdownCleanUpOnTaskQueue() override; bool IsEnded() const override; + // Before Dcomp surface is ready, we can't return any video data due to + // lacking of the image, which should only happen on the beginning of the + // video playback. In that situation, once we have enough video raw data, we + // can stop delaying the decode promise by waiting the Dcomp surface and + // resolveing the promise when Dcomp surface is ready. Doing so helps to keep + // the decode promise pending, so that the MFR won't keep sending more input + // data, which we actually don't need that many. + bool ShouldDelayVideoDecodeBeforeDcompReady(); + + void SendRequestSampleEvent(bool aIsEnough) override; + // Task queue only members. HANDLE mDCompSurfaceHandle; bool mNeedRecreateImage; @@ -98,6 +117,12 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { // have dcomp image. MozPromiseHolder<MediaDataDecoder::DecodePromise> mPendingDrainPromise; + // The promise used to return all video output which are requested before the + // Dcomp surface is ready. This should only be used once in entire playback, + // typically happening around the beginning of the playback. + MozPromiseHolder<MediaDataDecoder::DecodePromise> + mVideoDecodeBeforeDcompPromise; + // Set when `CreateMediaType()` is called. bool mIsEncrypted = false; }; diff --git a/dom/media/platforms/wmf/MFMediaSource.h b/dom/media/platforms/wmf/MFMediaSource.h index 735d53579e..0e44ef12aa 100644 --- a/dom/media/platforms/wmf/MFMediaSource.h +++ b/dom/media/platforms/wmf/MFMediaSource.h @@ -132,8 +132,6 @@ class MFMediaSource : public Microsoft::WRL::RuntimeClass< void AssertOnManagerThread() const; void AssertOnMFThreadPool() const; - void NotifyEndOfStreamInternal(TrackInfo::TrackType aType); - bool IsSeekable() const; // A thread-safe event queue. diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp index 6ebcf9a80a..63db5efae8 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp @@ -55,6 +55,9 @@ WMFAudioMFTManager::WMFAudioMFTManager(const AudioInfo& aConfig) audioSpecConfig = audioCodecSpecificBinaryBlob->Elements(); configLength = audioCodecSpecificBinaryBlob->Length(); } + // If no extradata has been provided, assume this is ADTS. Otherwise, + // assume raw AAC packets. + mIsADTS = !configLength; AACAudioSpecificConfigToUserData(aConfig.mExtendedProfile, audioSpecConfig, configLength, mUserData); } @@ -104,7 +107,8 @@ bool WMFAudioMFTManager::Init() { NS_ENSURE_TRUE(SUCCEEDED(hr), false); if (mStreamType == WMFStreamType::AAC) { - hr = inputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0x0); // Raw AAC packet + UINT32 payloadType = mIsADTS ? 1 : 0; + hr = inputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, payloadType); NS_ENSURE_TRUE(SUCCEEDED(hr), false); hr = inputType->SetBlob(MF_MT_USER_DATA, mUserData.Elements(), @@ -144,7 +148,8 @@ WMFAudioMFTManager::Input(MediaRawData* aSample) { nsCString WMFAudioMFTManager::GetCodecName() const { if (mStreamType == WMFStreamType::AAC) { return "aac"_ns; - } else if (mStreamType == WMFStreamType::MP3) { + } + if (mStreamType == WMFStreamType::MP3) { return "mp3"_ns; } return "unknown"_ns; @@ -177,8 +182,8 @@ WMFAudioMFTManager::UpdateOutputType() { } HRESULT -WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutData) { - aOutData = nullptr; +WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutput) { + aOutput = nullptr; RefPtr<IMFSample> sample; HRESULT hr; int typeChangeCount = 0; @@ -242,8 +247,8 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutData) { NS_ENSURE_TRUE(SUCCEEDED(hr), hr); // Output is made of floats. - int32_t numSamples = currentLength / sizeof(float); - int32_t numFrames = numSamples / mAudioChannels; + uint32_t numSamples = currentLength / sizeof(float); + uint32_t numFrames = numSamples / mAudioChannels; MOZ_ASSERT(numFrames >= 0); MOZ_ASSERT(numSamples >= 0); if (numFrames == 0) { @@ -275,10 +280,10 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr<MediaData>& aOutData) { return MF_E_TRANSFORM_NEED_MORE_INPUT; } - aOutData = new AudioData(aStreamOffset, pts, std::move(audioData), - mAudioChannels, mAudioRate, mChannelsMap); - MOZ_DIAGNOSTIC_ASSERT(duration == aOutData->mDuration, "must be equal"); - mLastOutputDuration = aOutData->mDuration; + aOutput = new AudioData(aStreamOffset, pts, std::move(audioData), + mAudioChannels, mAudioRate, mChannelsMap); + MOZ_DIAGNOSTIC_ASSERT(duration == aOutput->mDuration, "must be equal"); + mLastOutputDuration = aOutput->mDuration; #ifdef LOG_SAMPLE_DECODE LOG("Decoded audio sample! timestamp=%lld duration=%lld currentLength=%u", diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.h b/dom/media/platforms/wmf/WMFAudioMFTManager.h index b5dc379396..f772593545 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.h +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.h @@ -58,6 +58,7 @@ class WMFAudioMFTManager : public MFTManager { media::TimeUnit mLastOutputDuration = media::TimeUnit::Zero(); bool mFirstFrame = true; + bool mIsADTS = false; uint64_t mTotalMediaFrames = 0; uint32_t mEncoderDelay = 0; diff --git a/dom/media/platforms/wmf/WMFMediaDataEncoder.h b/dom/media/platforms/wmf/WMFMediaDataEncoder.h index 13848b47ad..31a63c8347 100644 --- a/dom/media/platforms/wmf/WMFMediaDataEncoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataEncoder.h @@ -202,7 +202,9 @@ class WMFMediaDataEncoder final : public MediaDataEncoder { MOZ_ASSERT(mEncoder); const layers::PlanarYCbCrImage* image = aData->mImage->AsPlanarYCbCrImage(); - MOZ_ASSERT(image); + // TODO: Take care non planar Y-Cb-Cr image (Bug 1881647). + NS_ENSURE_TRUE(image, nullptr); + const layers::PlanarYCbCrData* yuv = image->GetData(); auto ySize = yuv->YDataSize(); auto cbcrSize = yuv->CbCrDataSize(); @@ -223,6 +225,7 @@ class WMFMediaDataEncoder final : public MediaDataEncoder { LockBuffer lockBuffer(buffer); NS_ENSURE_TRUE(SUCCEEDED(lockBuffer.Result()), nullptr); + // TODO: Take care non I420 image (Bug 1881647). bool ok = libyuv::I420ToNV12( yuv->mYChannel, yuv->mYStride, yuv->mCbChannel, yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride, diff --git a/dom/media/platforms/wmf/WMFUtils.cpp b/dom/media/platforms/wmf/WMFUtils.cpp index d096979919..dda9df808e 100644 --- a/dom/media/platforms/wmf/WMFUtils.cpp +++ b/dom/media/platforms/wmf/WMFUtils.cpp @@ -177,7 +177,8 @@ Maybe<gfx::YUVColorSpace> GetYUVColorSpace(IMFMediaType* aType) { } int32_t MFOffsetToInt32(const MFOffset& aOffset) { - return int32_t(aOffset.value + (aOffset.fract / 65536.0f)); + return AssertedCast<int32_t>(AssertedCast<float>(aOffset.value) + + (AssertedCast<float>(aOffset.fract) / 65536.0f)); } TimeUnit GetSampleDuration(IMFSample* aSample) { @@ -204,7 +205,7 @@ GetPictureRegion(IMFMediaType* aMediaType, gfx::IntRect& aOutPictureRegion) { // Determine if "pan and scan" is enabled for this media. If it is, we // only display a region of the video frame, not the entire frame. BOOL panScan = - MFGetAttributeUINT32(aMediaType, MF_MT_PAN_SCAN_ENABLED, FALSE); + !!MFGetAttributeUINT32(aMediaType, MF_MT_PAN_SCAN_ENABLED, FALSE); // If pan and scan mode is enabled. Try to get the display region. HRESULT hr = E_FAIL; @@ -300,11 +301,14 @@ const char* MFTMessageTypeToStr(MFT_MESSAGE_TYPE aMsg) { GUID AudioMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { if (aMimeType.EqualsLiteral("audio/mpeg")) { return MFAudioFormat_MP3; - } else if (MP4Decoder::IsAAC(aMimeType)) { + } + if (MP4Decoder::IsAAC(aMimeType)) { return MFAudioFormat_AAC; - } else if (aMimeType.EqualsLiteral("audio/vorbis")) { + } + if (aMimeType.EqualsLiteral("audio/vorbis")) { return MFAudioFormat_Vorbis; - } else if (aMimeType.EqualsLiteral("audio/opus")) { + } + if (aMimeType.EqualsLiteral("audio/opus")) { return MFAudioFormat_Opus; } NS_WARNING("Unsupport audio mimetype"); @@ -314,17 +318,19 @@ GUID AudioMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { GUID VideoMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { if (MP4Decoder::IsH264(aMimeType)) { return MFVideoFormat_H264; - } else if (VPXDecoder::IsVP8(aMimeType)) { + } + if (VPXDecoder::IsVP8(aMimeType)) { return MFVideoFormat_VP80; - } else if (VPXDecoder::IsVP9(aMimeType)) { + } + if (VPXDecoder::IsVP9(aMimeType)) { return MFVideoFormat_VP90; } #ifdef MOZ_AV1 - else if (AOMDecoder::IsAV1(aMimeType)) { + if (AOMDecoder::IsAV1(aMimeType)) { return MFVideoFormat_AV1; } #endif - else if (MP4Decoder::IsHEVC(aMimeType)) { + if (MP4Decoder::IsHEVC(aMimeType)) { return MFVideoFormat_HEVC; } NS_WARNING("Unsupport video mimetype"); @@ -368,7 +374,9 @@ void AACAudioSpecificConfigToUserData(uint8_t aAACProfileLevelIndication, // the rest can be all 0x00. BYTE heeInfo[heeInfoLen] = {0}; WORD* w = (WORD*)heeInfo; - w[0] = 0x0; // Payload type raw AAC packet + // If extradata has been provided, assume raw AAC packets (0). Otherwise, + // assume ADTS (1) + w[0] = aConfigLength ? 0 : 1; w[1] = aAACProfileLevelIndication; aOutUserData.AppendElements(heeInfo, heeInfoLen); @@ -377,10 +385,10 @@ void AACAudioSpecificConfigToUserData(uint8_t aAACProfileLevelIndication, // The AudioSpecificConfig is TTTTTFFF|FCCCCGGG // (T=ObjectType, F=Frequency, C=Channel, G=GASpecificConfig) // If frequency = 0xf, then the frequency is explicitly defined on 24 bits. - int8_t frequency = + uint8_t frequency = (aAudioSpecConfig[0] & 0x7) << 1 | (aAudioSpecConfig[1] & 0x80) >> 7; - int8_t channels = (aAudioSpecConfig[1] & 0x78) >> 3; - int8_t gasc = aAudioSpecConfig[1] & 0x7; + uint8_t channels = (aAudioSpecConfig[1] & 0x78) >> 3; + uint8_t gasc = aAudioSpecConfig[1] & 0x7; if (frequency != 0xf && channels && !gasc) { // We enter this condition if the AudioSpecificConfig should theorically // be 2 bytes long but it's not. diff --git a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java index cc54009a7b..b4cc65ec9c 100644 --- a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java +++ b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java @@ -17,6 +17,7 @@ import android.content.Context; import android.util.Log; import android.view.Surface; import android.view.WindowManager; +import androidx.annotation.NonNull; import java.util.concurrent.CountDownLatch; @@ -37,10 +38,8 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand private final String deviceName; private volatile long native_capturer; // |VideoCaptureAndroid*| in C++. - private Context context; + private final Context context; private CameraVideoCapturer cameraVideoCapturer; - private EglBase eglBase; - private SurfaceTextureHelper surfaceTextureHelper; // This class is recreated everytime we start/stop capture, so we // can safely create the CountDownLatches here. @@ -48,8 +47,16 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand private boolean capturerStartedSucceeded = false; private final CountDownLatch capturerStopped = new CountDownLatch(1); - @WebRTCJNITarget - public VideoCaptureAndroid(String deviceName) { + @WebRTCJNITarget + public static VideoCaptureAndroid create(@NonNull final String deviceName) { + final Context context = GetContext(); + return new VideoCaptureAndroid(context, deviceName, + Camera2Enumerator.isSupported(context) + ? new Camera2Enumerator(context) + : new Camera1Enumerator()); + } + + private VideoCaptureAndroid(@NonNull final Context context, @NonNull final String deviceName, @NonNull final CameraEnumerator enumerator) { // Remove the camera facing information from the name. String[] parts = deviceName.split("Facing (front|back):"); if (parts.length == 2) { @@ -58,20 +65,14 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand Log.e(TAG, "VideoCaptureAndroid: Expected facing mode as part of name: " + deviceName); this.deviceName = deviceName; } - this.context = GetContext(); + this.context = context; - CameraEnumerator enumerator; - if (Camera2Enumerator.isSupported(context)) { - enumerator = new Camera2Enumerator(context); - } else { - enumerator = new Camera1Enumerator(); - } try { cameraVideoCapturer = enumerator.createCapturer(this.deviceName, this); - eglBase = EglBase.create(); - surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureAndroidSurfaceTextureHelper", eglBase.getEglBaseContext()); + final EglBase eglBase = EglBase.create(); + final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureAndroidSurfaceTextureHelper", eglBase.getEglBaseContext()); cameraVideoCapturer.initialize(surfaceTextureHelper, context, this); - } catch (java.lang.IllegalArgumentException e) { + } catch (java.lang.RuntimeException e) { Log.e(TAG, "VideoCaptureAndroid: Exception while creating capturer: " + e); } } @@ -80,6 +81,10 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand @WebRTCJNITarget private static native Context GetContext(); + public boolean canCapture() { + return cameraVideoCapturer != null; + } + // Called by native code. Returns true if capturer is started. // // Note that this actually opens the camera, and Camera callbacks run on the diff --git a/dom/media/systemservices/android_video_capture/video_capture_android.cc b/dom/media/systemservices/android_video_capture/video_capture_android.cc index da0715db27..cd5a73ceb6 100644 --- a/dom/media/systemservices/android_video_capture/video_capture_android.cc +++ b/dom/media/systemservices/android_video_capture/video_capture_android.cc @@ -177,12 +177,14 @@ int32_t VideoCaptureAndroid::Init(const char* deviceUniqueIdUTF8) { AttachThreadScoped ats(g_jvm_capture); JNIEnv* env = ats.env(); - jmethodID ctor = env->GetMethodID(g_java_capturer_class, "<init>", - "(Ljava/lang/String;)V"); - assert(ctor); + jmethodID factory = + env->GetStaticMethodID(g_java_capturer_class, "create", + "(Ljava/lang/String;)" + "Lorg/webrtc/videoengine/VideoCaptureAndroid;"); + assert(factory); jstring j_deviceName = env->NewStringUTF(_deviceUniqueId); - _jCapturer = env->NewGlobalRef( - env->NewObject(g_java_capturer_class, ctor, j_deviceName)); + _jCapturer = env->NewGlobalRef(env->CallStaticObjectMethod( + g_java_capturer_class, factory, j_deviceName)); assert(_jCapturer); return 0; } diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm index 36d5f56b16..ca7bf4db9b 100644 --- a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm @@ -88,10 +88,11 @@ AVCaptureDeviceFormat* _Nullable FindFormat( - (void)capturer:(RTCVideoCapturer* _Nonnull)capturer didCaptureVideoFrame:(RTCVideoFrame* _Nonnull)frame { - rtc::scoped_refptr<webrtc::videocapturemodule::VideoCaptureAvFoundation> cap; + webrtc::scoped_refptr<webrtc::videocapturemodule::VideoCaptureAvFoundation> + cap; { webrtc::MutexLock lock(&_mutex); - cap = rtc::scoped_refptr(_capturer); + cap = webrtc::scoped_refptr(_capturer); } if (!cap) return; cap->OnFrame(frame); diff --git a/dom/media/test/background_video.js b/dom/media/test/background_video.js index 508f8fd89a..6cce797474 100644 --- a/dom/media/test/background_video.js +++ b/dom/media/test/background_video.js @@ -72,7 +72,7 @@ function appendVideoToDocWithoutLoad(token, width, height) { } function loadAndWaitUntilLoadedmetadata(video, url, preloadType = "metadata") { - return new Promise((resolve, reject) => { + return new Promise(resolve => { video.preload = preloadType; video.addEventListener( "loadedmetadata", diff --git a/dom/media/test/bipbop-clearkey-video-av1.mp4 b/dom/media/test/bipbop-clearkey-video-av1.mp4 Binary files differnew file mode 100644 index 0000000000..78c1e56b2b --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.mp4 diff --git a/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ b/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ new file mode 100644 index 0000000000..4030ea1d3d --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/dom/media/test/bipbop-clearkey-video-av1.webm b/dom/media/test/bipbop-clearkey-video-av1.webm Binary files differnew file mode 100644 index 0000000000..840888a8b1 --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.webm diff --git a/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ b/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ new file mode 100644 index 0000000000..4030ea1d3d --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/dom/media/test/browser/browser.toml b/dom/media/test/browser/browser.toml index cf25369576..3e83448d47 100644 --- a/dom/media/test/browser/browser.toml +++ b/dom/media/test/browser/browser.toml @@ -2,6 +2,7 @@ subsuite = "media-bc" prefs = ["gfx.font_loader.delay=0"] support-files = [ + "head.js", "file_empty_page.html", "file_media.html", "../av1.mp4", @@ -16,11 +17,14 @@ support-files = [ "../small-shot.mp3", "../small-shot.ogg", "../TestPatternHDR.mp4", + "../../mediasource/test/bipbop/bipbop2s.mp4", ] ["browser_encrypted_play_time_telemetry.js"] skip-if = ["apple_silicon"] # Disabled due to bleedover with other tests when run in regular suites; passes in "failures" jobs +["browser_glean_first_frame_loaded_time.js"] + ["browser_tab_visibility_and_play_time.js"] ["browser_telemetry_video_hardware_decoding_support.js"] diff --git a/dom/media/test/browser/browser_encrypted_play_time_telemetry.js b/dom/media/test/browser/browser_encrypted_play_time_telemetry.js index ff4f2753ec..1a64717419 100644 --- a/dom/media/test/browser/browser_encrypted_play_time_telemetry.js +++ b/dom/media/test/browser/browser_encrypted_play_time_telemetry.js @@ -29,76 +29,6 @@ async function clearTelemetry() { }); } -// Opens a tab containing a blank page, returns a promise that will resolve -// to that tab. -async function openTab() { - const emptyPageUri = - "https://example.com/browser/dom/media/test/browser/file_empty_page.html"; - return BrowserTestUtils.openNewForegroundTab(window.gBrowser, emptyPageUri); -} - -// Creates and configures a video element for EME playback in `tab`. Does not -// start playback for the element. Returns a promise that will resolve once -// the element is setup and ready for playback. -async function loadEmeVideo(tab) { - const emeHelperUri = - gTestPath.substr(0, gTestPath.lastIndexOf("/")) + "/eme_standalone.js"; - return SpecialPowers.spawn( - tab.linkedBrowser, - [emeHelperUri], - async _emeHelperUri => { - // Begin helper functions. - async function once(target, name) { - return new Promise(r => - target.addEventListener(name, r, { once: true }) - ); - } - - // Helper to clone data into content so the EME helper can use the data. - function cloneIntoContent(data) { - return Cu.cloneInto(data, content.wrappedJSObject); - } - // End helper functions. - - // Load the EME helper into content. - Services.scriptloader.loadSubScript(_emeHelperUri, content); - // Setup EME with the helper. - let video = content.document.createElement("video"); - video.id = "media"; - content.document.body.appendChild(video); - let emeHelper = new content.wrappedJSObject.EmeHelper(); - emeHelper.SetKeySystem( - content.wrappedJSObject.EmeHelper.GetClearkeyKeySystemString() - ); - emeHelper.SetInitDataTypes(cloneIntoContent(["webm"])); - emeHelper.SetVideoCapabilities( - cloneIntoContent([{ contentType: 'video/webm; codecs="vp9"' }]) - ); - emeHelper.AddKeyIdAndKey( - "2cdb0ed6119853e7850671c3e9906c3c", - "808b9adac384de1e4f56140f4ad76194" - ); - emeHelper.onerror = error => { - is(false, `Got unexpected error from EME helper: ${error}`); - }; - await emeHelper.ConfigureEme(video); - // Done setting up EME. - - // Setup MSE. - const ms = new content.wrappedJSObject.MediaSource(); - video.src = content.wrappedJSObject.URL.createObjectURL(ms); - await once(ms, "sourceopen"); - const sb = ms.addSourceBuffer("video/webm"); - const videoFile = "sintel-short-clearkey-subsample-encrypted-video.webm"; - let fetchResponse = await content.fetch(videoFile); - sb.appendBuffer(await fetchResponse.arrayBuffer()); - await once(sb, "updateend"); - ms.endOfStream(); - await once(ms, "sourceended"); - } - ); -} - // Plays the media in `tab` until the 'ended' event is fire. Returns a promise // that resolves once that state has been reached. async function playMediaThrough(tab) { diff --git a/dom/media/test/browser/browser_glean_first_frame_loaded_time.js b/dom/media/test/browser/browser_glean_first_frame_loaded_time.js new file mode 100644 index 0000000000..1acfa9957e --- /dev/null +++ b/dom/media/test/browser/browser_glean_first_frame_loaded_time.js @@ -0,0 +1,81 @@ +"use strict"; + +// Disabling undef warning because in `run()` we use functions from head.js +/* eslint-disable no-undef */ + +/** + * This test is used to ensure that Glean probe 'first_frame_loaded' can be + * recorded correctly in different situations. + */ + +const testCases = [ + { + expected: { + playback_type: "Non-MSE playback", + video_codec: "video/avc", + resolution: "AV,240<h<=480", + key_system: undefined, + }, + async run(tab) { + await loadVideo(tab); + }, + }, + { + expected: { + playback_type: "MSE playback", + video_codec: "video/avc", + resolution: "AV,240<h<=480", + key_system: undefined, + }, + async run(tab) { + await loadMseVideo(tab); + }, + }, + { + expected: { + playback_type: "EME playback", + video_codec: "video/vp9", + resolution: "V,240<h<=480", + key_system: "org.w3.clearkey", + }, + async run(tab) { + await loadEmeVideo(tab); + }, + }, +]; + +add_task(async function testGleanMediaPlayackFirstFrameLoaded() { + for (let test of testCases) { + Services.fog.testResetFOG(); + + const expected = test.expected; + info(`running test for '${expected.playback_type}'`); + const tab = await openTab(); + await test.run(tab); + + info(`waiting until glean probe is ready on the parent process`); + await Services.fog.testFlushAllChildren(); + + info("checking the collected results"); + const extra = Glean.mediaPlayback.firstFrameLoaded.testGetValue()[0].extra; + Assert.greater( + parseInt(extra.first_frame_loaded_time), + 0, + `${extra.first_frame_loaded_time} is correct` + ); + is( + extra.playback_type, + expected.playback_type, + `${extra.playback_type} is correct` + ); + is( + extra.video_codec, + expected.video_codec, + `${extra.video_codec} is correct` + ); + is(extra.resolution, expected.resolution, `${extra.resolution} is correct`); + is(extra.key_system, expected.key_system, `${extra.key_system} is correct`); + + BrowserTestUtils.removeTab(tab); + } +}); diff --git a/dom/media/test/browser/browser_tab_visibility_and_play_time.js b/dom/media/test/browser/browser_tab_visibility_and_play_time.js index 4d33826091..b152f490c9 100644 --- a/dom/media/test/browser/browser_tab_visibility_and_play_time.js +++ b/dom/media/test/browser/browser_tab_visibility_and_play_time.js @@ -54,7 +54,7 @@ async function openMediaTab(url) { return new Promise(resolve => { element.addEventListener( "timeupdate", - e => { + () => { resolve(); }, { once: true } diff --git a/dom/media/test/browser/head.js b/dom/media/test/browser/head.js new file mode 100644 index 0000000000..7ef578a804 --- /dev/null +++ b/dom/media/test/browser/head.js @@ -0,0 +1,119 @@ +"use strict"; + +/* import-globals-from ../eme_standalone.js */ + +// Opens a tab containing a blank page, returns a promise that will resolve +// to that tab. +function openTab() { + const emptyPageUri = + "https://example.com/browser/dom/media/test/browser/file_empty_page.html"; + return BrowserTestUtils.openNewForegroundTab(window.gBrowser, emptyPageUri); +} + +// Creates and configures a video element for non-MSE playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadVideo(tab) { + return SpecialPowers.spawn(tab.linkedBrowser, [], async _ => { + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + + video.src = "gizmo.mp4"; + video.load(); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await new Promise(r => (video.onloadeddata = r)); + }); +} + +// Creates and configures a video element for MSE playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadMseVideo(tab) { + return SpecialPowers.spawn(tab.linkedBrowser, [], async _ => { + async function once(target, name) { + return new Promise(r => target.addEventListener(name, r, { once: true })); + } + + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + + info(`starting setup MSE`); + const ms = new content.wrappedJSObject.MediaSource(); + video.src = content.wrappedJSObject.URL.createObjectURL(ms); + await once(ms, "sourceopen"); + const sb = ms.addSourceBuffer("video/mp4"); + const videoFile = "bipbop2s.mp4"; + let fetchResponse = await content.fetch(videoFile); + sb.appendBuffer(await fetchResponse.arrayBuffer()); + await once(sb, "updateend"); + ms.endOfStream(); + await once(ms, "sourceended"); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await once(video, "loadeddata"); + }); +} + +// Creates and configures a video element for EME playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadEmeVideo(tab) { + const emeHelperUri = + gTestPath.substr(0, gTestPath.lastIndexOf("/")) + "/eme_standalone.js"; + return SpecialPowers.spawn( + tab.linkedBrowser, + [emeHelperUri], + async _emeHelperUri => { + async function once(target, name) { + return new Promise(r => + target.addEventListener(name, r, { once: true }) + ); + } + + // Helper to clone data into content so the EME helper can use the data. + function cloneIntoContent(data) { + return Cu.cloneInto(data, content.wrappedJSObject); + } + + info(`starting setup EME`); + Services.scriptloader.loadSubScript(_emeHelperUri, content); + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + let emeHelper = new content.wrappedJSObject.EmeHelper(); + emeHelper.SetKeySystem( + content.wrappedJSObject.EmeHelper.GetClearkeyKeySystemString() + ); + emeHelper.SetInitDataTypes(cloneIntoContent(["webm"])); + emeHelper.SetVideoCapabilities( + cloneIntoContent([{ contentType: 'video/webm; codecs="vp9"' }]) + ); + emeHelper.AddKeyIdAndKey( + "2cdb0ed6119853e7850671c3e9906c3c", + "808b9adac384de1e4f56140f4ad76194" + ); + emeHelper.onerror = error => { + is(false, `Got unexpected error from EME helper: ${error}`); + }; + await emeHelper.ConfigureEme(video); + + info(`starting setup MSE`); + const ms = new content.wrappedJSObject.MediaSource(); + video.src = content.wrappedJSObject.URL.createObjectURL(ms); + await once(ms, "sourceopen"); + const sb = ms.addSourceBuffer("video/webm"); + const videoFile = "sintel-short-clearkey-subsample-encrypted-video.webm"; + let fetchResponse = await content.fetch(videoFile); + sb.appendBuffer(await fetchResponse.arrayBuffer()); + await once(sb, "updateend"); + ms.endOfStream(); + await once(ms, "sourceended"); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await once(video, "loadeddata"); + } + ); +} diff --git a/dom/media/test/browser/wmfme/browser.toml b/dom/media/test/browser/wmfme/browser.toml index 422ea29228..7449d232d9 100644 --- a/dom/media/test/browser/wmfme/browser.toml +++ b/dom/media/test/browser/wmfme/browser.toml @@ -3,11 +3,17 @@ subsuite = "media-bc" tags = "media-engine-compatible" run-if = ["wmfme"] support-files = [ - "head.js", "file_video.html", + "head.js", + "../head.js", + "../../eme_standalone.js", "../../gizmo.mp4", + "../../sintel-short-clearkey-subsample-encrypted-video.webm", + "../../../mediasource/test/bipbop/bipbop2s.mp4", ] ["browser_wmfme_crash.js"] +["browser_wmfme_glean_first_frame_loaded_time.js"] + ["browser_wmfme_max_crashes.js"] diff --git a/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js b/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js new file mode 100644 index 0000000000..f9e97aaa49 --- /dev/null +++ b/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js @@ -0,0 +1,98 @@ +"use strict"; + +// Disabling undef warning because in `run()` we use functions from head.js +/* eslint-disable no-undef */ + +/** + * This test is used to ensure that Glean probe 'first_frame_loaded' can be + * recorded correctly in different situations. + */ + +/* import-globals-from ../head.js */ +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/dom/media/test/browser/head.js", + this +); + +add_task(async function setupTestingPref() { + await SpecialPowers.pushPrefEnv({ + set: [ + ["media.wmf.media-engine.enabled", 1], + ["media.wmf.media-engine.channel-decoder.enabled", true], + ["media.eme.wmf.clearkey.enabled", true], + ], + }); +}); + +const testCases = [ + { + expected: { + playback_type: "Non-MSE media-engine playback", + video_codec: "video/avc", + resolution: "AV,240<h<=480", + key_system: undefined, + }, + async run(tab) { + await loadVideo(tab); + }, + }, + { + expected: { + playback_type: "MSE media-engine playback", + video_codec: "video/avc", + resolution: "AV,240<h<=480", + key_system: undefined, + }, + async run(tab) { + await loadMseVideo(tab); + }, + }, + // TODO : we're not able to run MFCDM EME playback yet, see bug 1870722. + // { + // expected: { + // playback_type: "EME media-engine playback", + // video_codec: "video/vp9", + // resolution: "V,240<h<=480", + // key_system: "org.w3.clearkey", + // }, + // async run(tab) { + // await loadEmeVideo(tab); + // }, + // }, +]; + +add_task(async function testGleanMediaPlayackFirstFrameLoaded() { + for (let test of testCases) { + Services.fog.testResetFOG(); + + const expected = test.expected; + info(`running test for '${expected.playback_type}'`); + const tab = await openTab(); + await test.run(tab); + + info(`waiting until glean probe is ready on the parent process`); + await Services.fog.testFlushAllChildren(); + + info("checking the collected results"); + const extra = Glean.mediaPlayback.firstFrameLoaded.testGetValue()[0].extra; + Assert.greater( + parseInt(extra.first_frame_loaded_time), + 0, + `${extra.first_frame_loaded_time} is correct` + ); + is( + extra.playback_type, + expected.playback_type, + `${extra.playback_type} is correct` + ); + is( + extra.video_codec, + expected.video_codec, + `${extra.video_codec} is correct` + ); + is(extra.resolution, expected.resolution, `${extra.resolution} is correct`); + is(extra.key_system, expected.key_system, `${extra.key_system} is correct`); + + BrowserTestUtils.removeTab(tab); + } +}); diff --git a/dom/media/test/can_play_type_ogg.js b/dom/media/test/can_play_type_ogg.js index 79bf8a554c..6572ab7c0f 100644 --- a/dom/media/test/can_play_type_ogg.js +++ b/dom/media/test/can_play_type_ogg.js @@ -4,7 +4,7 @@ function check_ogg(v, enabled, finish) { } function basic_test() { - return new Promise(function (resolve, reject) { + return new Promise(function (resolve) { // Ogg types check("video/ogg", "maybe"); check("audio/ogg", "maybe"); diff --git a/dom/media/test/crashtests/crashtests.list b/dom/media/test/crashtests/crashtests.list index 637b161e7f..2383510ce9 100644 --- a/dom/media/test/crashtests/crashtests.list +++ b/dom/media/test/crashtests/crashtests.list @@ -1,4 +1,4 @@ -skip-if(OSX) load 1185191.html # this needs to run near the beginning of the test suite +skip-if(cocoaWidget) load 1185191.html # this needs to run near the beginning of the test suite load 0-timescale.html # bug 1229166 skip-if(Android) pref(media.autoplay.default,0) load 459439-1.html # bug 888557 load 466607-1.html @@ -14,7 +14,7 @@ load 497734-2.html load 576612-1.html load 752784-1.html skip-if(Android) load 789075-1.html # bug 1374405 for android -skip-if(Android&&AndroidVersion=='22') HTTP load 795892-1.html # bug 1358718 +HTTP load 795892-1.html # bug 1358718 load 844563.html load 846612.html load 852838.html @@ -76,7 +76,7 @@ skip-if(Android) test-pref(media.navigator.permission.disabled,true) load 102845 load 1041466.html load 1045650.html load 1080986.html -skip-if(Android&&AndroidVersion=='21') load 1180881.html # bug 1409365 +load 1180881.html # bug 1409365 load 1197935.html load 1122218.html load 1127188.html @@ -113,7 +113,7 @@ HTTP load media-element-source-seek-1.html load offline-buffer-source-ended-1.html load oscillator-ended-1.html load oscillator-ended-2.html -skip-if(Android&&AndroidVersion=='22') load video-replay-after-audio-end.html # bug 1315125, bug 1358876 +load video-replay-after-audio-end.html # bug 1315125, bug 1358876 # This needs to run at the end to avoid leaking busted state into other tests. skip-if(Android||ThreadSanitizer) load 691096-1.html # Bug 1365451 load 1236639.html @@ -122,7 +122,7 @@ load 1414444.mp4 load 1494073.html skip-if(Android) load 1526044.html # Bug 1528391 load 1530897.webm -skip-if(Android&&AndroidVersion<21) load encrypted-track-with-bad-sample-description-index.mp4 # Bug 1533211, unkip after bug 1550912 +load encrypted-track-with-bad-sample-description-index.mp4 # Bug 1533211, unkip after bug 1550912 load encrypted-track-without-tenc.mp4 # Bug 1533215 asserts-if(Android,0-1) load encrypted-track-with-sample-missing-cenc-aux.mp4 # Bug 1533625, bug 1588967 load 1538727.html diff --git a/dom/media/test/eme.js b/dom/media/test/eme.js index 927c99876a..d8eb1e78eb 100644 --- a/dom/media/test/eme.js +++ b/dom/media/test/eme.js @@ -440,7 +440,7 @@ function SetupEME(v, test, token) { } function fetchWithXHR(uri, onLoadFunction) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { var xhr = new XMLHttpRequest(); xhr.open("GET", uri, true); xhr.responseType = "arraybuffer"; @@ -463,7 +463,7 @@ function fetchWithXHR(uri, onLoadFunction) { } function once(target, name, cb) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { target.addEventListener( name, function (arg) { diff --git a/dom/media/test/manifest.js b/dom/media/test/manifest.js index bb4557d498..c357309021 100644 --- a/dom/media/test/manifest.js +++ b/dom/media/test/manifest.js @@ -817,7 +817,7 @@ var gOggTrackInfoResults = { function makeAbsolutePathConverter() { const url = SimpleTest.getTestFileURL("chromeHelper.js"); const script = SpecialPowers.loadChromeScript(url); - return new Promise((resolve, reject) => { + return new Promise(resolve => { script.addMessageListener("media-test:cwd", cwd => { if (!cwd) { ok(false, "Failed to find path to test files"); @@ -1891,6 +1891,46 @@ var gEMETests = [ sessionCount: 1, duration: 1.98, }, + // ffmpeg -i bipbop.mp4 -t 00:00:02 -c:v libaom-av1 bipbop_av1.mp4 + // packager-linux-x64 in=bipbop_av1.mp4,stream=video,out=bipbop-clearkey-video-av1.mp4 --enable_raw_key_encryption --keys label=:key_id=8b5df745ad84145b5617c33116e35a67:key=bddfd35dd9be033ee73bc18bc1885056 --clear_lead 0 + { + name: "MP4 av1 video clearkey", + tracks: [ + { + name: "video", + type: 'video/mp4; codecs="av1"', + fragments: ["bipbop-clearkey-video-av1.mp4"], + }, + ], + keys: { + // "keyid" : "key" + "8b5df745ad84145b5617c33116e35a67": "bddfd35dd9be033ee73bc18bc1885056", + }, + sessionType: "temporary", + sessionCount: 1, + duration: 2.0, + skipTests: ["waitingforkey"], + }, + // ffmpeg -i bipbop.mp4 -t 00:00:02 -c:v libaom-av1 bipbop_av1.webm + // packager-linux-x64 in=bipbop_av1.webm,stream=video,out=bipbop-clearkey-video-av1.webm --enable_raw_key_encryption --keys label=:key_id=8b5df745ad84145b5617c33116e35a67:key=bddfd35dd9be033ee73bc18bc1885056 --clear_lead 0 + { + name: "WebM av1 video clearkey", + tracks: [ + { + name: "video", + type: 'video/webm; codecs="av1"', + fragments: ["bipbop-clearkey-video-av1.webm"], + }, + ], + keys: { + // "keyid" : "key" + "8b5df745ad84145b5617c33116e35a67": "bddfd35dd9be033ee73bc18bc1885056", + }, + sessionType: "temporary", + sessionCount: 1, + duration: 2.0, + skipTests: ["waitingforkey"], + }, { name: "WebM vorbis audio & vp8 video clearkey", tracks: [ @@ -2241,7 +2281,7 @@ function removeNodeAndSource(n) { } function once(target, name, cb) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { target.addEventListener( name, function () { @@ -2262,7 +2302,7 @@ function once(target, name, cb) { * @returns {Promise} A promise that is resolved when event happens. */ function nextEvent(video, eventName) { - return new Promise(function (resolve, reject) { + return new Promise(function (resolve) { let f = function (event) { video.removeEventListener(eventName, f); resolve(event); diff --git a/dom/media/test/mochitest.toml b/dom/media/test/mochitest.toml index de8c690255..3c8a382766 100644 --- a/dom/media/test/mochitest.toml +++ b/dom/media/test/mochitest.toml @@ -88,6 +88,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", @@ -798,6 +802,7 @@ skip-if = [ "os == 'mac'", # 1517199 (timeout-on-osx) "os == 'linux'", # 1713397, 1719881 (high intermittent failure on linux tsan), 1776937 "apple_silicon", # Disabled due to bleedover with other tests when run in regular suites; passes in "failures" jobs + "mda_gpu", # Disabled because high intermittent failure ] ["test_chaining.html"] diff --git a/dom/media/test/mochitest_background_video.toml b/dom/media/test/mochitest_background_video.toml index 762a9a8648..e1bc542264 100644 --- a/dom/media/test/mochitest_background_video.toml +++ b/dom/media/test/mochitest_background_video.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_bugs.toml b/dom/media/test/mochitest_bugs.toml index 9d4a0dea2f..5c68d0e795 100644 --- a/dom/media/test/mochitest_bugs.toml +++ b/dom/media/test/mochitest_bugs.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", @@ -769,7 +773,10 @@ support-files = [ ["test_bug465498.html"] ["test_bug495145.html"] -skip-if = ["os == 'win'"] #Bug 1404373 +skip-if = [ + "win10_2009", #Bug 1404373 + "win11_2009", #Bug 1404373 +] ["test_bug495300.html"] @@ -786,15 +793,15 @@ skip-if = ["os == 'android'"] # bug 1285441, android(bug 1232305) tags = "capturestream" ["test_bug895305.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug919265.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug1113600.html"] skip-if = [ - "os == 'win' && os_version == '10.0' && debug", # Bug 1713410 - "os == 'mac'", # Bug 1198168 + "win10_2009 && debug", # Bug 1713410 + "win11_2009 && debug", # Bug 1713410 + "apple_catalina", # Bug 1198168 + "apple_silicon", # Bug 1198168 ] ["test_bug1120222.html"] diff --git a/dom/media/test/mochitest_compat.toml b/dom/media/test/mochitest_compat.toml index c477661a19..86f76f1464 100644 --- a/dom/media/test/mochitest_compat.toml +++ b/dom/media/test/mochitest_compat.toml @@ -90,6 +90,10 @@ support-files = [ "bipbop-cenc-videoinit.mp4^headers^", "bipbop-cenc-video-10s.mp4", "bipbop-cenc-video-10s.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop-clearkey-keyrotation-clear-lead-audio.mp4", "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", @@ -807,15 +811,12 @@ skip-if = ["true"] # bug 475110 - disabled since we don't play Wave files standa ["test_bug1431810_opus_downmix_to_mono.html"] ["test_can_play_type.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_mpeg.html"] ["test_can_play_type_no_ogg.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_ogg.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_wave.html"] @@ -1000,10 +1001,12 @@ skip-if = ["true"] # see bug 1319725 ["test_readyState.html"] ["test_referer.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_reset_src.html"] -skip-if = ["verify && debug && os == 'win'"] +skip-if = [ + "win10_2009 && debug && verify", + "win11_2009 && debug && verify", +] ["test_source.html"] diff --git a/dom/media/test/mochitest_eme.toml b/dom/media/test/mochitest_eme.toml index 696e0e947e..d7f39c3eb8 100644 --- a/dom/media/test/mochitest_eme.toml +++ b/dom/media/test/mochitest_eme.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_media_recorder.toml b/dom/media/test/mochitest_media_recorder.toml index 428f99601d..a4893d9cf4 100644 --- a/dom/media/test/mochitest_media_recorder.toml +++ b/dom/media/test/mochitest_media_recorder.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_seek.toml b/dom/media/test/mochitest_seek.toml index 0c90bd1bfb..d71aac775a 100644 --- a/dom/media/test/mochitest_seek.toml +++ b/dom/media/test/mochitest_seek.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_stream.toml b/dom/media/test/mochitest_stream.toml index 043f8471d4..0badfc52ab 100644 --- a/dom/media/test/mochitest_stream.toml +++ b/dom/media/test/mochitest_stream.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/reftest/color_quads/reftest.list b/dom/media/test/reftest/color_quads/reftest.list index 6071733f76..63a538b78a 100644 --- a/dom/media/test/reftest/color_quads/reftest.list +++ b/dom/media/test/reftest/color_quads/reftest.list @@ -14,35 +14,35 @@ defaults pref(media.av1.enabled,true) # - # yuv420p -fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(OSX,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png -fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(OSX,0-35,0-1947) fuzzy-if(OSX&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png +fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(cocoaWidget,0-35,0-1947) fuzzy-if(cocoaWidget&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm fuzzy-if(winWidget,0-1,0-78) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm -skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(OSX,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm -fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(OSX,0-35,0-1947) fuzzy-if(OSX&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(cocoaWidget,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(cocoaWidget,0-35,0-1947) fuzzy-if(cocoaWidget&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm skip-if(Android) fuzzy(16-48,8107-8818) fuzzy-if(winWidget&&swgl,31-38,8240-184080) fuzzy-if(appleSilicon,33-38,8819-11705) fuzzy-if(useDrawSnapshot,20-20,187200-187200) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm # On Windows & sw render, we noticed that the comparison image captured from AV1 is not equal to its displayed video frame, so we would need to compare other codecs directly to PNG file. That should be fixed in bug 1748540. -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,0-16,0-1718) fuzzy-if(OSX&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,2-36,184281-187407) fuzzy-if(winWidget,0-21,0-360000) fuzzy-if(appleSilicon,36-49,187329-187407) fuzzy-if(useDrawSnapshot,0-1,0-10) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,0-16,0-1718) fuzzy-if(OSX&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,0-16,0-1718) fuzzy-if(cocoaWidget&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,2-36,184281-187407) fuzzy-if(winWidget,0-21,0-360000) fuzzy-if(appleSilicon,36-49,187329-187407) fuzzy-if(useDrawSnapshot,0-1,0-10) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,0-16,0-1718) fuzzy-if(cocoaWidget&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm skip-if(Android) skip-if(!(winWidget&&swgl)) fuzzy(0-35,0-8506) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_img.html?src=color_quads/720p.png skip-if(Android) skip-if(!(winWidget&&swgl)) fuzzy(0-35,0-8506) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_img.html?src=color_quads/720p.png # - # yuv420p10 -skip-if(Android) fuzzy(33-49,1870-2579) fuzzy-if(swgl,34-52,180421-270528) fuzzy-if(useDrawSnapshot,16-16,183840-183840) fuzzy-if(OSX,60-74,270329-271024) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png -skip-if(Android) fuzzy-if(OSX,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm +skip-if(Android) fuzzy(33-49,1870-2579) fuzzy-if(swgl,34-52,180421-270528) fuzzy-if(useDrawSnapshot,16-16,183840-183840) fuzzy-if(cocoaWidget,60-74,270329-271024) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm #[2] skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm -skip-if(Android) fuzzy-if(OSX,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm -skip-if(Android) fuzzy(33-49,174620-270059) fuzzy-if(swgl&&!winWidget,36-52,11553-11555) fuzzy-if(swgl&&winWidget,36-52,11554-187200) fuzzy-if(swgl&&OSX,34-50,11465-270059) fuzzy-if(useDrawSnapshot,20-20,186800-186800) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png -skip-if(Android) fuzzy-if(OSX,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm +skip-if(Android) fuzzy(33-49,174620-270059) fuzzy-if(!winWidget&&swgl,36-52,11553-11555) fuzzy-if(winWidget&&swgl,36-52,11554-187200) fuzzy-if(swgl&&cocoaWidget,34-50,11465-270059) fuzzy-if(useDrawSnapshot,20-20,186800-186800) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm #[2] skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm -skip-if(Android) fuzzy-if(OSX,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm # Android is really broken in a variety of ways for p10. #[2]: yuv420p10 broken in h264.mp4: https://bugzilla.mozilla.org/show_bug.cgi?id=1711812 diff --git a/dom/media/test/reftest/reftest.list b/dom/media/test/reftest/reftest.list index 6bbd7b9a53..0f709a35ee 100644 --- a/dom/media/test/reftest/reftest.list +++ b/dom/media/test/reftest/reftest.list @@ -1,12 +1,12 @@ -skip-if(Android) fuzzy-if(OSX,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html -skip-if(Android) fuzzy-if(OSX,0-87,0-76797) fuzzy-if(appleSilicon,0-87,0-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-60,0-6070) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html -skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-57,0-4282) fuzzy-if(OSX,55-80,4173-4417) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html -skip-if(Android) fuzzy-if(OSX,0-25,0-175921) fuzzy-if(appleSilicon,34-34,40100-40100) fuzzy-if(winWidget,0-71,0-179198) HTTP(..) == gizmo.mp4.seek.html gizmo.mp4.55thframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-87,0-76797) fuzzy-if(appleSilicon,0-87,0-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget,0-60,0-6070) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html +skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget,0-57,0-4282) fuzzy-if(cocoaWidget,55-80,4173-4417) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-25,0-175921) fuzzy-if(appleSilicon,34-34,40100-40100) fuzzy-if(winWidget,0-71,0-179198) HTTP(..) == gizmo.mp4.seek.html gizmo.mp4.55thframe-ref.html # Bug 1758718 -skip-if(Android) skip-if(MinGW) skip-if(OSX) fuzzy(0-10,0-778236) == image-10bits-rendering-video.html image-10bits-rendering-ref.html -skip-if(Android) skip-if(MinGW) fuzzy(0-10,0-778536) fuzzy-if(appleSilicon,0-37,0-699614) == image-10bits-rendering-90-video.html image-10bits-rendering-90-ref.html +skip-if(Android) skip-if(cocoaWidget) fuzzy(0-10,0-778236) == image-10bits-rendering-video.html image-10bits-rendering-ref.html +skip-if(Android) fuzzy(0-10,0-778536) fuzzy-if(appleSilicon,0-37,0-699614) == image-10bits-rendering-90-video.html image-10bits-rendering-90-ref.html # Bug 1758718 -skip-if(Android) fuzzy(0-27,0-573106) skip-if(OSX) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html +skip-if(Android) fuzzy(0-27,0-573106) skip-if(cocoaWidget) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html skip-if(Android) fuzzy(0-31,0-573249) fuzzy-if(appleSilicon,0-37,0-543189) == image-10bits-rendering-720-90-video.html image-10bits-rendering-720-90-ref.html skip-if(Android) fuzzy(0-84,0-771156) fails-if(useDrawSnapshot) == uneven_frame_duration_video.html uneven_frame_duration_video-ref.html # Skip on Windows 7 as the resolution of the video is too high for test machines and will fail in the decoder. # Set media.dormant-on-pause-timeout-ms to avoid decoders becoming dormant and busting test, skip on android as test is too noisy and unstable diff --git a/dom/media/test/test_arraybuffer.html b/dom/media/test/test_arraybuffer.html index 9ef84c53dc..146d3ea5ac 100644 --- a/dom/media/test/test_arraybuffer.html +++ b/dom/media/test/test_arraybuffer.html @@ -49,14 +49,14 @@ function startTest(test, token) { events.forEach(function(e) { v.addEventListener(e, logEvent); }); - once(v, "stalled", function(e) { + once(v, "stalled", function() { // Resource fetch algorithm in local mode should never fire stalled event. // https://html.spec.whatwg.org/multipage/media.html#concept-media-load-resource ok(false, test.name + ": got stalled"); removeNodeAndSource(v); manager.finished(token); }); - once(v, "canplaythrough", function(e) { + once(v, "canplaythrough", function() { ok(true, test.name + ": got canplaythrough"); is(v.readyState, v.HAVE_ENOUGH_DATA, test.name + ": readyState is HAVE_ENOUGH_DATA"); removeNodeAndSource(v); diff --git a/dom/media/test/test_aspectratio_mp4.html b/dom/media/test/test_aspectratio_mp4.html index 5e01875439..959096e902 100644 --- a/dom/media/test/test_aspectratio_mp4.html +++ b/dom/media/test/test_aspectratio_mp4.html @@ -28,7 +28,7 @@ v.onloadedmetadata = function() { is(v.videoHeight, resource.height, "Intrinsic height should match video height"); SimpleTest.finish(); } -v.addEventListener("error", function(ev) { +v.addEventListener("error", function() { if (v.readyState < v.HAVE_METADATA) { info("Video element returns with readyState " + v.readyState + " error.code " + v.error.code); todo(false, "This platform doesn't support to retrieve MP4 metadata."); diff --git a/dom/media/test/test_background_video_resume_after_end_show_last_frame.html b/dom/media/test/test_background_video_resume_after_end_show_last_frame.html index 68f7179e12..991de9ac98 100644 --- a/dom/media/test/test_background_video_resume_after_end_show_last_frame.html +++ b/dom/media/test/test_background_video_resume_after_end_show_last_frame.html @@ -79,7 +79,7 @@ function waitUntilSeekToLastFrame(video) { ); } - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { video.seenEnded = false; video.addEventListener("ended", () => { video.seenEnded = true; diff --git a/dom/media/test/test_buffered.html b/dom/media/test/test_buffered.html index 86d8eec28a..85bbef2189 100644 --- a/dom/media/test/test_buffered.html +++ b/dom/media/test/test_buffered.html @@ -74,7 +74,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { ok(true, `${url}: Fetch succeeded, status=${xhr.status}`); // Request fulfilled. Note sometimes we get 206... Presumably because either diff --git a/dom/media/test/test_bug1431810_opus_downmix_to_mono.html b/dom/media/test/test_bug1431810_opus_downmix_to_mono.html index 647ddf0489..2d81fba1c7 100644 --- a/dom/media/test/test_bug1431810_opus_downmix_to_mono.html +++ b/dom/media/test/test_bug1431810_opus_downmix_to_mono.html @@ -116,14 +116,14 @@ function mediaElementWithPhaseInversionDisabled(audioContext, mediaElement, succ let ac = new AudioContext(); -function testPhaseInversion(mediaElement) { - return new Promise((accept, reject) => { +function testPhaseInversion() { + return new Promise((accept) => { mediaElementWithPhaseInversion(ac, a, accept); }); } -function testPhaseInversionDisabled(mediaElement) { - return new Promise((accept, reject) => { +function testPhaseInversionDisabled() { + return new Promise((accept) => { mediaElementWithPhaseInversionDisabled(ac, b, accept); }); } diff --git a/dom/media/test/test_cueless_webm_seek-1.html b/dom/media/test/test_cueless_webm_seek-1.html index db58a89665..dbc2933b92 100644 --- a/dom/media/test/test_cueless_webm_seek-1.html +++ b/dom/media/test/test_cueless_webm_seek-1.html @@ -101,7 +101,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_cueless_webm_seek-2.html b/dom/media/test/test_cueless_webm_seek-2.html index 720cc18399..f9fd29fcfb 100644 --- a/dom/media/test/test_cueless_webm_seek-2.html +++ b/dom/media/test/test_cueless_webm_seek-2.html @@ -91,7 +91,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_cueless_webm_seek-3.html b/dom/media/test/test_cueless_webm_seek-3.html index d6e3e50d7d..235066e6eb 100644 --- a/dom/media/test/test_cueless_webm_seek-3.html +++ b/dom/media/test/test_cueless_webm_seek-3.html @@ -85,7 +85,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_decode_error_crossorigin.html b/dom/media/test/test_decode_error_crossorigin.html index 24c1430a5b..8fa3db6c38 100644 --- a/dom/media/test/test_decode_error_crossorigin.html +++ b/dom/media/test/test_decode_error_crossorigin.html @@ -19,7 +19,7 @@ function startTest(test, token) { }; const v = document.createElement("video"); manager.started(token); - v.addEventListener("error", event => { + v.addEventListener("error", () => { if (v.readyState == v.HAVE_NOTHING) { is(v.error.code, MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED, "Expected code for a load error"); diff --git a/dom/media/test/test_delay_load.html b/dom/media/test/test_delay_load.html index 05877aa911..6c10e5b957 100644 --- a/dom/media/test/test_delay_load.html +++ b/dom/media/test/test_delay_load.html @@ -87,7 +87,7 @@ var w = window.open("", "testWindow", "width=400,height=400"); testWindows.push(w); v = createVideo(test.name, test.type, "4"); -v.onloadstart = function(e) { +v.onloadstart = function() { // Using a new window to do this is a bit annoying, but if we use an iframe here, // delaying of the iframe's load event might interfere with the firing of our load event // in some confusing way. So it's simpler just to use another window. diff --git a/dom/media/test/test_eme_detach_media_keys.html b/dom/media/test/test_eme_detach_media_keys.html index 69b812032a..5faa68d9c5 100644 --- a/dom/media/test/test_eme_detach_media_keys.html +++ b/dom/media/test/test_eme_detach_media_keys.html @@ -15,7 +15,7 @@ SimpleTest.waitForExplicitFinish(); function createAndSet() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var m; navigator.requestMediaKeySystemAccess(CLEARKEY_KEYSYSTEM, gCencMediaKeySystemConfig) .then(function (access) { diff --git a/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html b/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html index e47fae5891..6911ac2da8 100644 --- a/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html +++ b/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html @@ -72,7 +72,7 @@ function startTest(test, token) ok(false, " Failed to set original mediakeys back."); } - function onCanPlayAgain(ev) { + function onCanPlayAgain() { Promise.all([closeSessions()]) .then(() => { ok(true, " (ENCRYPTED) Playback can be resumed."); @@ -91,7 +91,7 @@ function startTest(test, token) v.currentTime = v.duration / 2; } - function onCanPlay(ev) { + function onCanPlay() { function onSetMediaKeysToNullOK() { ok(true, TimeStamp(token) + " Set MediaKeys to null. OK!"); diff --git a/dom/media/test/test_eme_getstatusforpolicy.html b/dom/media/test/test_eme_getstatusforpolicy.html index 2031d04df5..6e9503e33a 100644 --- a/dom/media/test/test_eme_getstatusforpolicy.html +++ b/dom/media/test/test_eme_getstatusforpolicy.html @@ -68,7 +68,7 @@ add_task(async function testGetStatusForPolicy() { "Should have set MediaKeys on media element"); let keyStatus = await video.mediaKeys.getStatusForPolicy({minHdcpVersion : result.minHdcpVersion}) - .catch(e => ok(false, "getStatusForPolicy failed!")); + .catch(() => ok(false, "getStatusForPolicy failed!")); info(`getStatusForPolicy for HDCP ${result.minHdcpVersion} : ${keyStatus}`); is(keyStatus, result.expectedResult, `Expected ${result.expectedResult}, got ${keyStatus}`); diff --git a/dom/media/test/test_eme_initDataTypes.html b/dom/media/test/test_eme_initDataTypes.html index 587e6fc161..a1e0c15182 100644 --- a/dom/media/test/test_eme_initDataTypes.html +++ b/dom/media/test/test_eme_initDataTypes.html @@ -93,7 +93,7 @@ function PrepareInitData(initDataType, initData) } function Test(test) { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var configs = [{ initDataTypes: [test.initDataType], videoCapabilities: [{contentType: 'video/mp4' }], @@ -113,7 +113,7 @@ function Test(test) { var initData = PrepareInitData(test.initDataType, test.initData); return session.generateRequest(test.initDataType, initData); } - ).catch((x) => { + ).catch(() => { ok(!test.expectPass, "'" + test.name + "' expected to fail."); resolve(); }); diff --git a/dom/media/test/test_eme_mfcdm_generate_request.html b/dom/media/test/test_eme_mfcdm_generate_request.html index 57be325e36..b1b8396c0f 100644 --- a/dom/media/test/test_eme_mfcdm_generate_request.html +++ b/dom/media/test/test_eme_mfcdm_generate_request.html @@ -65,11 +65,11 @@ async function testKeySystemRequest(keySystem) { }]; info(`requestMediaKeySystemAccess for ${keySystem}`); let access = await navigator.requestMediaKeySystemAccess(keySystem, configs) - .catch(e => ok(false, `failed to create key system access`)); + .catch(() => ok(false, `failed to create key system access`)); info('creating media key'); let mediaKeys = await access.createMediaKeys() - .catch(e => ok(false, `failed to create media key`));; + .catch(() => ok(false, `failed to create media key`));; info(`creating a temporary media key session`); let session = mediaKeys.createSession(sessionType); @@ -84,7 +84,7 @@ async function testKeySystemRequest(keySystem) { await session.generateRequest( 'keyids', new TextEncoder().encode(fakeKID)) - .catch(e => ok(false, `failed to generate request`)); + .catch(() => ok(false, `failed to generate request`)); await messagePromise; } diff --git a/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html b/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html index 7e22be25d9..83cd740570 100644 --- a/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html +++ b/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html @@ -89,7 +89,7 @@ add_task(async function testGetStatusForPolicy() { "Should have set MediaKeys on media element"); let keyStatus = await video.mediaKeys.getStatusForPolicy({minHdcpVersion : result.minHdcpVersion}) - .catch(e => ok(false, "getStatusForPolicy failed!")); + .catch(() => ok(false, "getStatusForPolicy failed!")); info(`getStatusForPolicy for HDCP ${result.minHdcpVersion} : ${keyStatus}`); is(keyStatus, result.expectedResult, `Expected ${result.expectedResult}, got ${keyStatus}`); diff --git a/dom/media/test/test_eme_missing_pssh.html b/dom/media/test/test_eme_missing_pssh.html index 29f77d021a..cb94ffbf23 100644 --- a/dom/media/test/test_eme_missing_pssh.html +++ b/dom/media/test/test_eme_missing_pssh.html @@ -63,7 +63,7 @@ } function DownloadMedia(url, type, mediaSource) { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var sourceBuffer = mediaSource.addSourceBuffer(type); fetchWithXHR(url, (response) => { once(sourceBuffer, "updateend", resolve); diff --git a/dom/media/test/test_eme_non_mse_fails.html b/dom/media/test/test_eme_non_mse_fails.html index 6ff17d59ff..e2719f244c 100644 --- a/dom/media/test/test_eme_non_mse_fails.html +++ b/dom/media/test/test_eme_non_mse_fails.html @@ -65,7 +65,7 @@ function TestSetSrc(test, token) manager.started(token); var v = document.createElement("video"); - v.addEventListener("error", function(err) { + v.addEventListener("error", function() { ok(true, token + " got error setting src on video element, as expected"); manager.finished(token); }); diff --git a/dom/media/test/test_eme_playback.html b/dom/media/test/test_eme_playback.html index bcfa058e34..478b7b0a25 100644 --- a/dom/media/test/test_eme_playback.html +++ b/dom/media/test/test_eme_playback.html @@ -109,7 +109,7 @@ function startTest(test, token) var gotEncrypted = 0; let finish = new EMEPromise; - v.addEventListener("encrypted", function(ev) { + v.addEventListener("encrypted", function() { gotEncrypted += 1; }); @@ -119,7 +119,7 @@ function startTest(test, token) is(v.isEncrypted, undefined, "isEncrypted should not be accessible from content"); }); - v.addEventListener("ended", function(ev) { + v.addEventListener("ended", function() { ok(true, TimeStamp(token) + " got ended event"); is(gotEncrypted, test.sessionCount, diff --git a/dom/media/test/test_eme_protection_query.html b/dom/media/test/test_eme_protection_query.html index 8bf97d8100..a8a1c7b376 100644 --- a/dom/media/test/test_eme_protection_query.html +++ b/dom/media/test/test_eme_protection_query.html @@ -53,7 +53,7 @@ async function setupEme(video) { let session = video.mediaKeys.createSession(); video.onencrypted = async encryptedEvent => { - session.onmessage = messageEvent => { + session.onmessage = () => { // Handle license messages. Hard code the license because we always test // with the same file and we know what the license should be. const license = { diff --git a/dom/media/test/test_eme_pssh_in_moof.html b/dom/media/test/test_eme_pssh_in_moof.html index d1965be844..406600056e 100644 --- a/dom/media/test/test_eme_pssh_in_moof.html +++ b/dom/media/test/test_eme_pssh_in_moof.html @@ -61,7 +61,7 @@ // Specialized create media keys function, since the one in eme.js relies
// on listening for encrypted events, and we want to manage those
// ourselves within this test.
- async function createAndSetMediaKeys(video, test, token) {
+ async function createAndSetMediaKeys(video, test) {
function streamType(type) {
var x = test.tracks.find(o => o.name == type);
return x ? x.type : undefined;
diff --git a/dom/media/test/test_eme_requestKeySystemAccess.html b/dom/media/test/test_eme_requestKeySystemAccess.html index b044fe8c84..4014ef937a 100644 --- a/dom/media/test/test_eme_requestKeySystemAccess.html +++ b/dom/media/test/test_eme_requestKeySystemAccess.html @@ -20,7 +20,7 @@ function ValidateConfig(name, expected, observed) { is(observed.label, expected.label, name + " label should match"); if (expected.initDataTypes) { - ok(expected.initDataTypes.every((element, index, array) => observed.initDataTypes.includes(element)), name + " initDataTypes should match."); + ok(expected.initDataTypes.every((element) => observed.initDataTypes.includes(element)), name + " initDataTypes should match."); } if (expected.audioCapabilities) { @@ -45,7 +45,7 @@ function ValidateConfig(name, expected, observed) { function Test(test) { var name = "'" + test.name + "'"; - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var p; if (test.options) { var keySystem = (test.keySystem !== undefined) ? test.keySystem : CLEARKEY_KEYSYSTEM; diff --git a/dom/media/test/test_eme_request_notifications.html b/dom/media/test/test_eme_request_notifications.html index 6c44f892a6..62ef28e57d 100644 --- a/dom/media/test/test_eme_request_notifications.html +++ b/dom/media/test/test_eme_request_notifications.html @@ -17,7 +17,7 @@ function SetPrefs(prefs) { } function observe() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var observer = function(subject, topic, data) { SpecialPowers.Services.obs.removeObserver(observer, "mediakeys-request"); resolve(JSON.parse(data).status); @@ -70,7 +70,7 @@ var tests = [ SimpleTest.waitForExplicitFinish(); tests - .reduce(function(p, c, i, array) { + .reduce(function(p, c) { return p.then(function() { return Test(c); }); diff --git a/dom/media/test/test_eme_sample_groups_playback.html b/dom/media/test/test_eme_sample_groups_playback.html index cef1e26b33..e1e09ad73d 100644 --- a/dom/media/test/test_eme_sample_groups_playback.html +++ b/dom/media/test/test_eme_sample_groups_playback.html @@ -81,7 +81,7 @@ }); var request = new TextEncoder().encode(json); session.generateRequest("keyids", request) - .then(e => { + .then(() => { Log(test.name, "Request license success"); }, reason => { Log("Request license failed! " + reason); @@ -90,7 +90,7 @@ } function DownloadMedia(url, type, mediaSource) { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { var sourceBuffer = mediaSource.addSourceBuffer(type); fetchWithXHR(url, (response) => { once(sourceBuffer, "updateend", resolve); diff --git a/dom/media/test/test_eme_stream_capture_blocked_case1.html b/dom/media/test/test_eme_stream_capture_blocked_case1.html index 0fc8d28364..e6743b4f8a 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case1.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case1.html @@ -35,7 +35,7 @@ function startTest(test, token) var context = new AudioContext(); context.createMediaElementSource(v1); - v1.addEventListener("loadeddata", function(ev) { + v1.addEventListener("loadeddata", function() { ok(false, TimeStamp(case1token) + " should never reach loadeddata, as setMediaKeys should fail"); }); diff --git a/dom/media/test/test_eme_stream_capture_blocked_case2.html b/dom/media/test/test_eme_stream_capture_blocked_case2.html index b60538caf0..6ee415c124 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case2.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case2.html @@ -20,7 +20,7 @@ function startTest(test, token) var case2token = token + "_case2"; let v2 = document.createElement("video"); - v2.addEventListener("loadeddata", function(ev) { + v2.addEventListener("loadeddata", function() { ok(true, case2token + " should reach loadeddata"); var threw = false; try { diff --git a/dom/media/test/test_eme_stream_capture_blocked_case3.html b/dom/media/test/test_eme_stream_capture_blocked_case3.html index e25a900956..23a23542e9 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case3.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case3.html @@ -19,7 +19,7 @@ function startTest(test, token) var case3token = token + "_case3"; let v3 = document.createElement("video"); - v3.addEventListener("loadeddata", function(ev) { + v3.addEventListener("loadeddata", function() { ok(true, TimeStamp(case3token) + " should reach loadeddata"); var threw = false; try { diff --git a/dom/media/test/test_eme_unsetMediaKeys_then_capture.html b/dom/media/test/test_eme_unsetMediaKeys_then_capture.html index 3ecdc79dbf..eb20f2bee2 100644 --- a/dom/media/test/test_eme_unsetMediaKeys_then_capture.html +++ b/dom/media/test/test_eme_unsetMediaKeys_then_capture.html @@ -36,11 +36,11 @@ function startTest(test, token) let finish = new EMEPromise; - function onVideoEnded(ev) { + function onVideoEnded() { ok(true, TimeStamp(token) + " (ENCRYPTED) content playback ended."); function playClearVideo() { - var p1 = once(v, 'loadeddata', (e) => { + var p1 = once(v, 'loadeddata', () => { ok(true, TimeStamp(token) + " Receiving event 'loadeddata' for (CLEAR) content."); let canvasElem = document.createElement('canvas'); document.body.appendChild(canvasElem); diff --git a/dom/media/test/test_eme_wideinve_l1_installation.html b/dom/media/test/test_eme_wideinve_l1_installation.html index fa6a0c350b..da1f1ab0f9 100644 --- a/dom/media/test/test_eme_wideinve_l1_installation.html +++ b/dom/media/test/test_eme_wideinve_l1_installation.html @@ -19,7 +19,7 @@ function SetPrefs(prefs) { } function observe() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var observer = function(subject, topic, data) { SpecialPowers.Services.obs.removeObserver(observer, "mediakeys-request"); resolve(JSON.parse(data).status); @@ -74,7 +74,7 @@ const tests = [ }, ]; -tests.reduce(function(p, c, i, array) { +tests.reduce(function(p, c) { return p.then(function() { return Test(c); }); diff --git a/dom/media/test/test_eme_wv_privacy.html b/dom/media/test/test_eme_wv_privacy.html index a050d2a528..e18d31a36f 100644 --- a/dom/media/test/test_eme_wv_privacy.html +++ b/dom/media/test/test_eme_wv_privacy.html @@ -12,7 +12,7 @@ <script class="testbody" type="text/javascript"> function Test() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var configs = [{ initDataTypes: ['keyids'], videoCapabilities: [{contentType: 'video/mp4' }], diff --git a/dom/media/test/test_hls_player_independency.html b/dom/media/test/test_hls_player_independency.html index cea5c140ed..ed2d5cee52 100644 --- a/dom/media/test/test_hls_player_independency.html +++ b/dom/media/test/test_hls_player_independency.html @@ -27,12 +27,12 @@ function startTest() { var v4x3 = document.getElementById('player4x3'); var v16x9 = document.getElementById('player16x9'); - var p1 = once(v4x3, 'ended', function onended(e) { + var p1 = once(v4x3, 'ended', function onended() { is(v4x3.videoWidth, 400, "4x3 content, the width should be 400."); is(v4x3.videoHeight, 300, "4x3 content, the height should be 300."); }); - var p2 = once(v16x9, 'ended', function onended(e) { + var p2 = once(v16x9, 'ended', function onended() { is(v16x9.videoWidth, 416, "16x9 content, the width should be 416."); is(v16x9.videoHeight, 234, "16x9 content, the height should be 234."); }); diff --git a/dom/media/test/test_hw_video_decoding.html b/dom/media/test/test_hw_video_decoding.html index f93ab9b0a8..43ac27bf71 100644 --- a/dom/media/test/test_hw_video_decoding.html +++ b/dom/media/test/test_hw_video_decoding.html @@ -24,7 +24,10 @@ const PLATFORMS = { // file: "gizmo.webm", // decoder: "wmf VP9 codec hardware video decoder", // }, - // TODO : add AV1, see bug 1861501. + { + file: "av1.mp4", + decoder: "wmf AV1 codec hardware video decoder", + }, ], }, Darwin: { diff --git a/dom/media/test/test_invalid_reject.html b/dom/media/test/test_invalid_reject.html index 583847fe12..4023642632 100644 --- a/dom/media/test/test_invalid_reject.html +++ b/dom/media/test/test_invalid_reject.html @@ -18,7 +18,7 @@ function startTest(test, token) { manager.started(token); // Set up event handlers. Seeing any of these is a failure. - function badEvent(type) { return function(e) { + function badEvent(type) { return function() { ok(false, test.name + " should not fire '" + type + "' event"); }}; var events = [ @@ -31,7 +31,7 @@ function startTest(test, token) { }); // Seeing a decoder error is a success. - v.addEventListener("error", function onerror(e) { + v.addEventListener("error", function onerror() { if (v.readyState == v.HAVE_NOTHING) { is(v.error.code, v.error.MEDIA_ERR_SRC_NOT_SUPPORTED, "decoder should reject " + test.name); diff --git a/dom/media/test/test_invalid_reject_play.html b/dom/media/test/test_invalid_reject_play.html index 3e658f94b8..bbfbbc34cc 100644 --- a/dom/media/test/test_invalid_reject_play.html +++ b/dom/media/test/test_invalid_reject_play.html @@ -18,14 +18,14 @@ function startTest(test, token) { manager.started(token); // Seeing a decoder error is a success. - v.addEventListener("error", function onerror(e) { + v.addEventListener("error", function onerror() { is(v.error.code, v.error.MEDIA_ERR_DECODE, "decoder should reject " + test.name); v.removeEventListener("error", onerror); manager.finished(token); }); - v.addEventListener("ended", function onended(e) { + v.addEventListener("ended", function onended() { ok(false, "decoder should have rejected file before playback ended"); v.removeEventListener("ended", onended); manager.finished(token); diff --git a/dom/media/test/test_load.html b/dom/media/test/test_load.html index de8fd63948..a502a6b23a 100644 --- a/dom/media/test/test_load.html +++ b/dom/media/test/test_load.html @@ -18,7 +18,7 @@ https://bugzilla.mozilla.org/show_bug.cgi?id=479859 <pre id="test"> <script type="text/javascript"> -function log(msg) { +function log() { //document.getElementById('log').innerHTML += "<p>" + msg + "</p>"; } @@ -89,7 +89,7 @@ function prependSource(src, type) { var gTests = [ { // Test 0: adding video to doc, then setting src should load implicitly. - create(src, type) { + create(src) { document.body.appendChild(gMedia); gMedia.src = src; }, @@ -126,7 +126,7 @@ var gTests = [ addSource("404a", type); var s2 = addSource("404b", type); s2.addEventListener("error", - function(e) { + function() { // Should awaken waiting load, causing successful load. addSource(src, type); }); @@ -143,7 +143,7 @@ var gTests = [ addSource("404a", type); var s2 = addSource("404b", type); s2.addEventListener("error", - function(e) { + function() { // Should awaken waiting load, causing successful load. if (!prepended) { prependSource(src, type); diff --git a/dom/media/test/test_load_source.html b/dom/media/test/test_load_source.html index 95a925b61f..0792bb924b 100644 --- a/dom/media/test/test_load_source.html +++ b/dom/media/test/test_load_source.html @@ -26,7 +26,7 @@ https://bugzilla.mozilla.org/show_bug.cgi?id=534571 var v = null; var s = null; -function finish(event) { +function finish() { ok(true, "Should have played both videos"); SimpleTest.finish(); } @@ -34,7 +34,7 @@ function finish(event) { var first = null; var second = null; -function ended(event) { +function ended() { s.type = second.type; s.src = second.name; v.removeEventListener("ended", ended); diff --git a/dom/media/test/test_media_selection.html b/dom/media/test/test_media_selection.html index 42f5a9bd43..33ecabfd58 100644 --- a/dom/media/test/test_media_selection.html +++ b/dom/media/test/test_media_selection.html @@ -32,7 +32,7 @@ function maketest(attach_media, name, type, check_metadata) { manager.finished(token); }); } else { - e.addEventListener('error', function onerror(event) { + e.addEventListener('error', function onerror() { is(errorRun, false, "error handler should run once only!"); errorRun = true; is(e.readyState, HTMLMediaElement.HAVE_NOTHING, @@ -46,7 +46,7 @@ function maketest(attach_media, name, type, check_metadata) { } } -function set_src(element, name, type) { +function set_src(element, name) { element.src = name; document.body.appendChild(element); } diff --git a/dom/media/test/test_mediarecorder_record_canvas_captureStream.html b/dom/media/test/test_mediarecorder_record_canvas_captureStream.html index 0b6cd6dbb5..d80a06e8ff 100644 --- a/dom/media/test/test_mediarecorder_record_canvas_captureStream.html +++ b/dom/media/test/test_mediarecorder_record_canvas_captureStream.html @@ -51,7 +51,7 @@ function startTest() { video.id = "recorded-video"; video.src = URL.createObjectURL(blob); video.play(); - video.onerror = err => { + video.onerror = () => { ok(false, "Should be able to play the recording. Got error. code=" + video.error.code); SimpleTest.finish(); }; diff --git a/dom/media/test/test_mediarecorder_record_changing_video_resolution.html b/dom/media/test/test_mediarecorder_record_changing_video_resolution.html index d6354ee5a1..b4e3a6a3fb 100644 --- a/dom/media/test/test_mediarecorder_record_changing_video_resolution.html +++ b/dom/media/test/test_mediarecorder_record_changing_video_resolution.html @@ -70,7 +70,7 @@ function startTest() { video.src = URL.createObjectURL(blob); video.preload = "metadata"; - video.onerror = err => { + video.onerror = () => { ok(false, "Should be able to play the recording. Got error. code=" + video.error.code); SimpleTest.finish(); }; diff --git a/dom/media/test/test_mediarecorder_record_downsize_resolution.html b/dom/media/test/test_mediarecorder_record_downsize_resolution.html index f9422a3897..5e2fbd13c1 100644 --- a/dom/media/test/test_mediarecorder_record_downsize_resolution.html +++ b/dom/media/test/test_mediarecorder_record_downsize_resolution.html @@ -64,7 +64,7 @@ function startTest() { var video = document.createElement("video"); video.id = "recorded-video"; video.src = URL.createObjectURL(blob); - video.onerror = err => { + video.onerror = () => { ok(false, "Should be able to play the recording. Got error. code=" + video.error.code); SimpleTest.finish(); }; diff --git a/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html b/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html index 961a9644b2..7c70ddf49d 100644 --- a/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html +++ b/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html @@ -53,7 +53,7 @@ async function startTest() { // We'll stop recording upon the 1st blob being received if (dataAvailableCount === 1) { - mediaRecorder.onstop = function (event) { + mediaRecorder.onstop = function () { info('onstop fired'); if (!onDataAvailableFirst) { diff --git a/dom/media/test/test_mediarecorder_record_session.html b/dom/media/test/test_mediarecorder_record_session.html index 88795d82d0..a14d3c2819 100644 --- a/dom/media/test/test_mediarecorder_record_session.html +++ b/dom/media/test/test_mediarecorder_record_session.html @@ -44,7 +44,7 @@ function startTest(test, token) { } // data avaliable. - mediaRecorder.ondataavailable = function(evt) {} + mediaRecorder.ondataavailable = function() {} mediaRecorder.onerror = function(err) { ok(false, 'Unexpected error fired with:' + err); diff --git a/dom/media/test/test_mediarecorder_record_startstopstart.html b/dom/media/test/test_mediarecorder_record_startstopstart.html index b4cc62c709..2556072979 100644 --- a/dom/media/test/test_mediarecorder_record_startstopstart.html +++ b/dom/media/test/test_mediarecorder_record_startstopstart.html @@ -18,7 +18,7 @@ function startTest() { var stopCount = 0; var dataavailable = 0; var expectedMimeType = 'audio/ogg; codecs=opus'; - recorder.onstop = function (e) { + recorder.onstop = function () { info('onstop fired'); is(recorder.stream, dest.stream, 'Media recorder stream = element stream post recording'); @@ -45,7 +45,7 @@ function startTest() { } dataavailable++; } - recorder.onerror = function (e) { + recorder.onerror = function () { ok(false, 'it should execute normally without exception'); } recorder.onwarning = function() { diff --git a/dom/media/test/test_mediarecorder_record_timeslice.html b/dom/media/test/test_mediarecorder_record_timeslice.html index 3e547e77b4..b122c45df5 100644 --- a/dom/media/test/test_mediarecorder_record_timeslice.html +++ b/dom/media/test/test_mediarecorder_record_timeslice.html @@ -68,7 +68,7 @@ function startTest(test, token) { // We'll stop recording upon the 1st blob being received if (dataAvailableCount === 1) { - mediaRecorder.onstop = function (event) { + mediaRecorder.onstop = function () { info('onstop fired'); if (!onDataAvailableFirst) { diff --git a/dom/media/test/test_mediarecorder_record_upsize_resolution.html b/dom/media/test/test_mediarecorder_record_upsize_resolution.html index d02fd08e44..81a761479c 100644 --- a/dom/media/test/test_mediarecorder_record_upsize_resolution.html +++ b/dom/media/test/test_mediarecorder_record_upsize_resolution.html @@ -64,7 +64,7 @@ function startTest() { var video = document.createElement("video"); video.id = "recorded-video"; video.src = URL.createObjectURL(blob); - video.onerror = err => { + video.onerror = () => { ok(false, "Should be able to play the recording. Got error. code=" + video.error.code); SimpleTest.finish(); }; diff --git a/dom/media/test/test_mediatrack_consuming_mediaresource.html b/dom/media/test/test_mediatrack_consuming_mediaresource.html index 515df5c053..052b7cb667 100644 --- a/dom/media/test/test_mediatrack_consuming_mediaresource.html +++ b/dom/media/test/test_mediatrack_consuming_mediaresource.html @@ -29,27 +29,27 @@ function startTest(test, token) { isnot(element.videoTracks, undefined, 'HTMLMediaElement::VideoTracks() property should be available.'); - element.audioTracks.onaddtrack = function(e) { + element.audioTracks.onaddtrack = function() { audioOnaddtrack++; } - element.audioTracks.onremovetrack = function(e) { + element.audioTracks.onremovetrack = function() { audioOnremovetrack++; } - element.audioTracks.onchange = function(e) { + element.audioTracks.onchange = function() { audioOnchange++; } - element.videoTracks.onaddtrack = function(e) { + element.videoTracks.onaddtrack = function() { videoOnaddtrack++; } - element.videoTracks.onremovetrack = function(e) { + element.videoTracks.onremovetrack = function() { videoOnremovetrack++; } - element.videoTracks.onchange = function(e) { + element.videoTracks.onchange = function() { videoOnchange++; } diff --git a/dom/media/test/test_mediatrack_consuming_mediastream.html b/dom/media/test/test_mediatrack_consuming_mediastream.html index b930ca4fdc..79464ef28d 100644 --- a/dom/media/test/test_mediatrack_consuming_mediastream.html +++ b/dom/media/test/test_mediatrack_consuming_mediastream.html @@ -28,19 +28,19 @@ async function startTest() { return; } - element.audioTracks.onaddtrack = function(e) { + element.audioTracks.onaddtrack = function() { audioOnaddtrack++; }; - element.audioTracks.onchange = function(e) { + element.audioTracks.onchange = function() { audioOnchange++; }; - element.videoTracks.onaddtrack = function(e) { + element.videoTracks.onaddtrack = function() { videoOnaddtrack++; }; - element.videoTracks.onchange = function(e) { + element.videoTracks.onchange = function() { videoOnchange++; }; diff --git a/dom/media/test/test_mediatrack_replay_from_end.html b/dom/media/test/test_mediatrack_replay_from_end.html index 16b0cbeb97..143ae42668 100644 --- a/dom/media/test/test_mediatrack_replay_from_end.html +++ b/dom/media/test/test_mediatrack_replay_from_end.html @@ -34,19 +34,19 @@ function startTest(test, token) { let isPlaying = false; let steps = 0; - element.audioTracks.onaddtrack = function(e) { + element.audioTracks.onaddtrack = function() { audioOnaddtrack++; } - element.audioTracks.onremovetrack = function(e) { + element.audioTracks.onremovetrack = function() { audioOnremovetrack++; } - element.videoTracks.onaddtrack = function(e) { + element.videoTracks.onaddtrack = function() { videoOnaddtrack++; } - element.videoTracks.onremovetrack = function(e) { + element.videoTracks.onremovetrack = function() { videoOnremovetrack++; } diff --git a/dom/media/test/test_midflight_redirect_blocked.html b/dom/media/test/test_midflight_redirect_blocked.html index ea85673b45..24c5a96bd3 100644 --- a/dom/media/test/test_midflight_redirect_blocked.html +++ b/dom/media/test/test_midflight_redirect_blocked.html @@ -12,7 +12,7 @@ <script class="testbody" type='application/javascript'> function testIfLoadsToMetadata(test, useCors) { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var elemType = getMajorMimeType(test.type); var element = document.createElement(elemType); diff --git a/dom/media/test/test_mixed_principals.html b/dom/media/test/test_mixed_principals.html index c1deaef697..88523dabc7 100644 --- a/dom/media/test/test_mixed_principals.html +++ b/dom/media/test/test_mixed_principals.html @@ -46,7 +46,7 @@ https://bugzilla.mozilla.org/show_bug.cgi?id=489415 } function runTest(origin, shouldReadBackOnLoad) { - return new Promise(function (resolve, reject) { + return new Promise(function (resolve) { // Load will redirect mid-flight, which will be detected and should error, // and we should no longer be able to readback. var video = document.createElement("video"); diff --git a/dom/media/test/test_new_audio.html b/dom/media/test/test_new_audio.html index e1f8964f73..5e94bf9055 100644 --- a/dom/media/test/test_new_audio.html +++ b/dom/media/test/test_new_audio.html @@ -33,7 +33,7 @@ function startTest(test, token) { a.autoplay = true; document.body.appendChild(a); a.addEventListener("ended", - function(e){ + function(){ ok(true, "[" + a.src + "]We should get to the end. Oh look we did."); a.remove(); manager.finished(token); diff --git a/dom/media/test/test_periodic_timeupdate.html b/dom/media/test/test_periodic_timeupdate.html index 5d40c7e38a..9933e3d5dc 100644 --- a/dom/media/test/test_periodic_timeupdate.html +++ b/dom/media/test/test_periodic_timeupdate.html @@ -36,7 +36,7 @@ add_task(async function testPeriodicTimeupdateShouldOnlyBeDispatchedOnceWithin25 info(`seeking ends`); video._ignoreEvents = false; } - function checkTimeupdate(event) { + function checkTimeupdate() { // When reaching to the end, video would perform a seek to the start // position where one mandatory `timeupdate` would be dispatched. if (video._ignoreEvents) { diff --git a/dom/media/test/test_play_twice.html b/dom/media/test/test_play_twice.html index e94f28a031..ebe6e323b6 100644 --- a/dom/media/test/test_play_twice.html +++ b/dom/media/test/test_play_twice.html @@ -25,7 +25,7 @@ function startTest(test, token) { checkMetadata(t.name, v, test); }}(test, video); - var noLoad = function(t, v) { return function() { + var noLoad = function(t) { return function() { ok(false, t.name + " should not fire 'load' event"); }}(test, video); diff --git a/dom/media/test/test_playback.html b/dom/media/test/test_playback.html index 5e28861e93..2f16020059 100644 --- a/dom/media/test/test_playback.html +++ b/dom/media/test/test_playback.html @@ -35,7 +35,7 @@ function startTest(test, token) { checkMetadata(t.name, v, t); }}(test, video); - var noLoad = function(t, v) { return function() { + var noLoad = function(t) { return function() { ok(false, t.name + " should not fire 'load' event"); }}(test, video); diff --git a/dom/media/test/test_playback_errors.html b/dom/media/test/test_playback_errors.html index 7b3f046099..73d250b5aa 100644 --- a/dom/media/test/test_playback_errors.html +++ b/dom/media/test/test_playback_errors.html @@ -24,7 +24,7 @@ function startTest(test, token) { v.remove(); manager.finished(token); } - var checkError = function(t, v) { return function(evt) { + var checkError = function(t, v) { return function() { v._errorCount++; is(v._errorCount, 1, t.name + " only one error fired"); endedTest(v); diff --git a/dom/media/test/test_playback_hls.html b/dom/media/test/test_playback_hls.html index 7d5c777fc1..0d160e9264 100644 --- a/dom/media/test/test_playback_hls.html +++ b/dom/media/test/test_playback_hls.html @@ -34,7 +34,7 @@ function startTest(test, token) { checkMetadata(t.name, v, t); }}(test, video); - var noLoad = function(t, v) { return function() { + var noLoad = function(t) { return function() { ok(false, t.name + " should not fire 'load' event"); }}(test, video); diff --git a/dom/media/test/test_preload_actions.html b/dom/media/test/test_preload_actions.html index d027147c71..bc1e94d2a9 100644 --- a/dom/media/test/test_preload_actions.html +++ b/dom/media/test/test_preload_actions.html @@ -79,8 +79,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); v.src = test.name; document.body.appendChild(v); // Causes implicit load, which will be halted due to preload:none. @@ -105,8 +105,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "metadata"; - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("loadeddata", this.loadeddata); v.src = test.name; document.body.appendChild(v); // Causes implicit load, which will be halted after @@ -128,8 +128,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "auto"; - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("canplaythrough", this.canplaythrough); v.src = test.name; // Causes implicit load. document.body.appendChild(v); @@ -163,8 +163,8 @@ var tests = [ v._gotLoadedMetaData = false; v._gotSuspend = false; v.preload = "none"; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); v.addEventListener("ended", this.ended); v.src = test.name; @@ -190,8 +190,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); document.body.appendChild(v); // Causes implicit load, which will be halted due to no resource. v.src = test.name; // Load should start, and halt at preload:none. @@ -216,8 +216,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); document.body.appendChild(v); // Causes implicit load, which will be halted due to no resource. var s = document.createElement("source"); @@ -256,14 +256,14 @@ var tests = [ v._gotLoadedMetaData = false; v.preload = "none"; v._gotErrorEvent = false; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); v.addEventListener("ended", this.ended); var s1 = document.createElement("source"); s1.src = "not-a-real-file.404" s1.type = test.type; - s1.addEventListener("error", function(e){v._gotErrorEvent = true;}); + s1.addEventListener("error", function(){v._gotErrorEvent = true;}); v.appendChild(s1); var s2 = document.createElement("source"); s2.src = test.name; @@ -288,8 +288,8 @@ var tests = [ setup(v) { v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadstart", function(e){v.preload = "metadata";}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v.preload = "metadata";}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("loadeddata", this.loadeddata); v.src = test.name; // Causes implicit load. document.body.appendChild(v); @@ -333,8 +333,8 @@ var tests = [ setup(v) { v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadstart", function(e){v.preload = "auto";}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v.preload = "auto";}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("canplaythrough", this.canplaythrough); v.src = test.name; // Causes implicit load. document.body.appendChild(v); @@ -356,8 +356,8 @@ var tests = [ setup(v) { v._gotLoadedMetaData = false; v.preload = "none"; - v.addEventListener("loadstart", function(e){v.preload = "metadata";}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v.preload = "metadata";}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("loadeddata", this.loadeddata); v.src = test.name; // Causes implicit load. document.body.appendChild(v); @@ -384,8 +384,8 @@ var tests = [ v.preload = "auto"; v.src = test.name; v.preload = "none"; - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); v.addEventListener("suspend", this.suspend); document.body.appendChild(v); // Causes implicit load, should load according to preload none document.createElement("source"); @@ -415,8 +415,8 @@ var tests = [ v._gotLoadStart = false; v._gotLoadedMetaData = false; v.preload = "metadata"; - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("ended", this.ended); v.addEventListener("loadeddata", this.loadeddata); v.src = test.name; @@ -439,8 +439,8 @@ var tests = [ v._gotLoadedMetaData = false; v.preload = "none"; v.autoplay = true; - v.addEventListener("loadstart", function(e){v._gotLoadStart = true;}); - v.addEventListener("loadedmetadata", function(e){v._gotLoadedMetaData = true;}); + v.addEventListener("loadstart", function(){v._gotLoadStart = true;}); + v.addEventListener("loadedmetadata", function(){v._gotLoadedMetaData = true;}); v.addEventListener("ended", this.ended); v.src = test.name; // Causes implicit load. document.body.appendChild(v); diff --git a/dom/media/test/test_preload_suspend.html b/dom/media/test/test_preload_suspend.html index 7f1146360f..b715a58dc8 100644 --- a/dom/media/test/test_preload_suspend.html +++ b/dom/media/test/test_preload_suspend.html @@ -85,7 +85,7 @@ var tests = [ } ]; -function startTest(test, token) { +function startTest(test) { var v = document.createElement("video"); v.name = test.name; var key = Math.random(); diff --git a/dom/media/test/test_reset_events_async.html b/dom/media/test/test_reset_events_async.html index 482ec55986..98010cd475 100644 --- a/dom/media/test/test_reset_events_async.html +++ b/dom/media/test/test_reset_events_async.html @@ -23,10 +23,10 @@ https://bugzilla.mozilla.org/show_bug.cgi?id=975270 is(a.networkState, HTMLMediaElement.NETWORK_EMPTY, "Shouldn't be loading"); - a.addEventListener("abort", function(e) { a._abort++; }); - a.addEventListener("emptied", function(e) { a._emptied++; }); + a.addEventListener("abort", function() { a._abort++; }); + a.addEventListener("emptied", function() { a._emptied++; }); a.addEventListener("loadedmetadata", - function(e) { + function() { is(a._abort, 0, "Should not have received 'abort' before 'loadedmetadata"); is(a._emptied, 0, "Should not have received 'emptied' before 'loadedmetadata"); diff --git a/dom/media/test/test_seek-10.html b/dom/media/test/test_seek-10.html index 6c02384ed4..63bb47f52a 100644 --- a/dom/media/test/test_seek-10.html +++ b/dom/media/test/test_seek-10.html @@ -31,7 +31,7 @@ function startTest() { v.currentTime = v.duration * 0.9; } -function done(evt) { +function done() { ok(true, "We don't acutally test anything..."); finish(); } diff --git a/dom/media/test/test_seekToNextFrame.html b/dom/media/test/test_seekToNextFrame.html index 755d06e622..fc00d91389 100644 --- a/dom/media/test/test_seekToNextFrame.html +++ b/dom/media/test/test_seekToNextFrame.html @@ -46,7 +46,7 @@ function startTest(test, token) { ); } - var onLoadedmetadata = function(t, v) { return function() { + var onLoadedmetadata = function() { return function() { callSeekToNextFrame(); }}(test, video); diff --git a/dom/media/test/test_temporary_file_blob_video_plays.html b/dom/media/test/test_temporary_file_blob_video_plays.html index 87f6b3c4e6..09ff377c43 100644 --- a/dom/media/test/test_temporary_file_blob_video_plays.html +++ b/dom/media/test/test_temporary_file_blob_video_plays.html @@ -51,7 +51,7 @@ function startTest() { video.id = "recorded-video"; video.src = URL.createObjectURL(blob); video.play(); - video.onerror = err => { + video.onerror = () => { ok(false, "Should be able to play the recording. Got error. code=" + video.error.code); SimpleTest.finish(); }; diff --git a/dom/media/test/test_video_dimensions.html b/dom/media/test/test_video_dimensions.html index 4d9c2a185f..facd504ac5 100644 --- a/dom/media/test/test_video_dimensions.html +++ b/dom/media/test/test_video_dimensions.html @@ -65,7 +65,7 @@ var startTest = function(test, token) { manager.finished(token); } }; - var setupElement = function(v, id) { + var setupElement = function(v) { v.durationchange = false; v.ondurationchange = ondurationchange; v.resize = false; diff --git a/dom/media/test/test_video_low_power_telemetry.html b/dom/media/test/test_video_low_power_telemetry.html index be609f7ceb..25a5ce21a6 100644 --- a/dom/media/test/test_video_low_power_telemetry.html +++ b/dom/media/test/test_video_low_power_telemetry.html @@ -124,7 +124,7 @@ steps we run before each video. We keep track of which test we are running with } } - async function doPostTest(v) { + async function doPostTest() { info(`Test ${testIndex} attempting to retrieve telemetry.`); let snap = await retrieveSnapshotAndClearTelemetry(); ok(snap, `Test ${testIndex} should have telemetry.`); diff --git a/dom/media/utils/TelemetryProbesReporter.cpp b/dom/media/utils/TelemetryProbesReporter.cpp index a2854812ba..8c5614c048 100644 --- a/dom/media/utils/TelemetryProbesReporter.cpp +++ b/dom/media/utils/TelemetryProbesReporter.cpp @@ -292,6 +292,52 @@ void TelemetryProbesReporter::OnDecodeResumed() { mOwner->DispatchAsyncTestingEvent(u"mozvideodecodesuspendedpaused"_ns); } +void TelemetryProbesReporter::OntFirstFrameLoaded( + const TimeDuration& aLoadedFirstFrameTime, bool aIsMSE, + bool aIsExternalEngineStateMachine) { + const MediaInfo& info = mOwner->GetMediaInfo(); + MOZ_ASSERT(info.HasVideo()); + nsCString resolution; + DetermineResolutionForTelemetry(info, resolution); + + glean::media_playback::FirstFrameLoadedExtra extraData; + extraData.firstFrameLoadedTime = Some(aLoadedFirstFrameTime.ToMilliseconds()); + if (!aIsMSE && !aIsExternalEngineStateMachine) { + extraData.playbackType = Some("Non-MSE playback"_ns); + } else if (aIsMSE && !aIsExternalEngineStateMachine) { + extraData.playbackType = !mOwner->IsEncrypted() ? Some("MSE playback"_ns) + : Some("EME playback"_ns); + } else if (!aIsMSE && aIsExternalEngineStateMachine) { + extraData.playbackType = Some("Non-MSE media-engine playback"_ns); + } else if (aIsMSE && aIsExternalEngineStateMachine) { + extraData.playbackType = !mOwner->IsEncrypted() + ? Some("MSE media-engine playback"_ns) + : Some("EME media-engine playback"_ns); + } else { + extraData.playbackType = Some("ERROR TYPE"_ns); + MOZ_ASSERT(false, "Unexpected playback type!"); + } + extraData.videoCodec = Some(info.mVideo.mMimeType); + extraData.resolution = Some(resolution); + if (const auto keySystem = mOwner->GetKeySystem()) { + extraData.keySystem = Some(NS_ConvertUTF16toUTF8(*keySystem)); + } + + if (MOZ_LOG_TEST(gTelemetryProbesReporterLog, LogLevel::Debug)) { + nsPrintfCString logMessage{ + "Media_Playabck First_Frame_Loaded event, time(ms)=%f, " + "playback-type=%s, videoCodec=%s, resolution=%s", + aLoadedFirstFrameTime.ToMilliseconds(), extraData.playbackType->get(), + extraData.videoCodec->get(), extraData.resolution->get()}; + if (const auto keySystem = mOwner->GetKeySystem()) { + logMessage.Append(nsPrintfCString{ + ", keySystem=%s", NS_ConvertUTF16toUTF8(*keySystem).get()}); + } + LOG("%s", logMessage.get()); + } + glean::media_playback::first_frame_loaded.Record(Some(extraData)); +} + void TelemetryProbesReporter::OnShutdown() { AssertOnMainThreadAndNotShutdown(); LOG("Shutdown"); diff --git a/dom/media/utils/TelemetryProbesReporter.h b/dom/media/utils/TelemetryProbesReporter.h index 51ae2f12bb..be81e8022c 100644 --- a/dom/media/utils/TelemetryProbesReporter.h +++ b/dom/media/utils/TelemetryProbesReporter.h @@ -66,6 +66,8 @@ class TelemetryProbesReporter final { void OnMutedChanged(bool aMuted); void OnDecodeSuspended(); void OnDecodeResumed(); + void OntFirstFrameLoaded(const TimeDuration& aLoadedFirstFrameTime, + bool aIsMSE, bool aIsExternalEngineStateMachine); double GetTotalVideoPlayTimeInSeconds() const; double GetTotalVideoHDRPlayTimeInSeconds() const; diff --git a/dom/media/webaudio/FFTBlock.cpp b/dom/media/webaudio/FFTBlock.cpp index 6789ca0264..79fb934a00 100644 --- a/dom/media/webaudio/FFTBlock.cpp +++ b/dom/media/webaudio/FFTBlock.cpp @@ -30,15 +30,14 @@ #include "FFTBlock.h" +#include "FFVPXRuntimeLinker.h" #include <complex> namespace mozilla { -typedef std::complex<double> Complex; +FFmpegFFTFuncs FFTBlock::sFFTFuncs = {}; -#ifdef MOZ_LIBAV_FFT -FFmpegRDFTFuncs FFTBlock::sRDFTFuncs; -#endif +using Complex = std::complex<double>; static double fdlibm_cabs(const Complex& z) { return fdlibm_hypot(real(z), imag(z)); @@ -51,16 +50,15 @@ static double fdlibm_carg(const Complex& z) { FFTBlock* FFTBlock::CreateInterpolatedBlock(const FFTBlock& block0, const FFTBlock& block1, double interp) { - FFTBlock* newBlock = new FFTBlock(block0.FFTSize()); + uint32_t fftSize = block0.FFTSize(); + FFTBlock* newBlock = new FFTBlock(fftSize, 1.0f / AssertedCast<float>(fftSize)); newBlock->InterpolateFrequencyComponents(block0, block1, interp); // In the time-domain, the 2nd half of the response must be zero, to avoid // circular convolution aliasing... - int fftSize = newBlock->FFTSize(); AlignedTArray<float> buffer(fftSize); - newBlock->GetInverseWithoutScaling(buffer.Elements()); - AudioBufferInPlaceScale(buffer.Elements(), 1.0f / fftSize, fftSize / 2); + newBlock->GetInverse(buffer.Elements()); PodZero(buffer.Elements() + fftSize / 2, fftSize / 2); // Put back into frequency domain. diff --git a/dom/media/webaudio/FFTBlock.h b/dom/media/webaudio/FFTBlock.h index 6af882f9af..ecce39d7b1 100644 --- a/dom/media/webaudio/FFTBlock.h +++ b/dom/media/webaudio/FFTBlock.h @@ -7,31 +7,17 @@ #ifndef FFTBlock_h_ #define FFTBlock_h_ -#ifdef BUILD_ARM_NEON -# include <cmath> -# include "mozilla/arm.h" -# include "dl/sp/api/omxSP.h" -#endif - #include "AlignedTArray.h" #include "AudioNodeEngine.h" -#if defined(MOZ_LIBAV_FFT) -# include "FFmpegRDFTTypes.h" -# include "FFVPXRuntimeLinker.h" -#else -# include "kiss_fft/kiss_fftr.h" -#endif +#include "FFVPXRuntimeLinker.h" +#include "ffvpx/tx.h" namespace mozilla { // This class defines an FFT block, loosely modeled after Blink's FFTFrame // class to make sharing code with Blink easy. -// Currently it's implemented on top of KissFFT on all platforms. class FFTBlock final { union ComplexU { -#if !defined(MOZ_LIBAV_FFT) - kiss_fft_cpx c; -#endif float f[2]; struct { float r; @@ -41,28 +27,13 @@ class FFTBlock final { public: static void MainThreadInit() { -#ifdef MOZ_LIBAV_FFT FFVPXRuntimeLinker::Init(); - if (!sRDFTFuncs.init) { - FFVPXRuntimeLinker::GetRDFTFuncs(&sRDFTFuncs); + if (!sFFTFuncs.init) { + FFVPXRuntimeLinker::GetFFTFuncs(&sFFTFuncs); } -#endif } - - explicit FFTBlock(uint32_t aFFTSize) -#if defined(MOZ_LIBAV_FFT) - : mAvRDFT(nullptr), - mAvIRDFT(nullptr) -#else - : mKissFFT(nullptr), - mKissIFFT(nullptr) -# ifdef BUILD_ARM_NEON - , - mOmxFFT(nullptr), - mOmxIFFT(nullptr) -# endif -#endif - { + explicit FFTBlock(uint32_t aFFTSize, float aInverseScaling = 1.0f) + : mInverseScaling(aInverseScaling) { MOZ_COUNT_CTOR(FFTBlock); SetFFTSize(aFFTSize); } @@ -83,63 +54,33 @@ class FFTBlock final { return; } -#if defined(MOZ_LIBAV_FFT) - PodCopy(mOutputBuffer.Elements()->f, aData, mFFTSize); - sRDFTFuncs.calc(mAvRDFT, mOutputBuffer.Elements()->f); - // Recover packed Nyquist. - mOutputBuffer[mFFTSize / 2].r = mOutputBuffer[0].i; - mOutputBuffer[0].i = 0.0f; -#else -# ifdef BUILD_ARM_NEON - if (mozilla::supports_neon()) { - omxSP_FFTFwd_RToCCS_F32_Sfs(aData, mOutputBuffer.Elements()->f, mOmxFFT); - } else -# endif - { - kiss_fftr(mKissFFT, aData, &(mOutputBuffer.Elements()->c)); - } + mFn(mTxCtx, mOutputBuffer.Elements()->f, const_cast<float*>(aData), + 2 * sizeof(float)); +#ifdef DEBUG + mInversePerformed = false; #endif } - // Inverse-transform internal data and store the resulting FFTSize() - // points in aDataOut. - void GetInverse(float* aDataOut) { - GetInverseWithoutScaling(aDataOut); - AudioBufferInPlaceScale(aDataOut, 1.0f / mFFTSize, mFFTSize); - } - // Inverse-transform internal frequency data and store the resulting // FFTSize() points in |aDataOut|. If frequency data has not already been // scaled, then the output will need scaling by 1/FFTSize(). - void GetInverseWithoutScaling(float* aDataOut) { + void GetInverse(float* aDataOut) { if (!EnsureIFFT()) { std::fill_n(aDataOut, mFFTSize, 0.0f); return; }; - -#if defined(MOZ_LIBAV_FFT) - { - // Even though this function doesn't scale, the libav forward transform - // gives a value that needs scaling by 2 in order for things to turn out - // similar to how we expect from kissfft/openmax. - AudioBufferCopyWithScale(mOutputBuffer.Elements()->f, 2.0f, aDataOut, - mFFTSize); - aDataOut[1] = 2.0f * mOutputBuffer[mFFTSize / 2].r; // Packed Nyquist - sRDFTFuncs.calc(mAvIRDFT, aDataOut); - } -#else -# ifdef BUILD_ARM_NEON - if (mozilla::supports_neon()) { - omxSP_FFTInv_CCSToR_F32_Sfs_unscaled(mOutputBuffer.Elements()->f, - aDataOut, mOmxIFFT); - } else -# endif - { - kiss_fftri(mKissIFFT, &(mOutputBuffer.Elements()->c), aDataOut); - } + // When performing an inverse transform, tx overwrites the input. This + // asserts that forward / inverse transforms are interleaved to avoid having + // to keep the input around. + MOZ_ASSERT(!mInversePerformed); + mIFn(mITxCtx, aDataOut, mOutputBuffer.Elements()->f, 2 * sizeof(float)); +#ifdef DEBUG + mInversePerformed = true; #endif } void Multiply(const FFTBlock& aFrame) { + MOZ_ASSERT(!mInversePerformed); + uint32_t halfSize = mFFTSize / 2; // DFTs are not packed. MOZ_ASSERT(mOutputBuffer[0].i == 0); @@ -161,8 +102,8 @@ class FFTBlock final { MOZ_ASSERT(dataSize <= FFTSize()); AlignedTArray<float> paddedData; paddedData.SetLength(FFTSize()); - AudioBufferCopyWithScale(aData, 1.0f / FFTSize(), paddedData.Elements(), - dataSize); + AudioBufferCopyWithScale(aData, 1.0f / AssertedCast<float>(FFTSize()), + paddedData.Elements(), dataSize); PodZero(paddedData.Elements() + dataSize, mFFTSize - dataSize); PerformFFT(paddedData.Elements()); } @@ -180,46 +121,35 @@ class FFTBlock final { double ExtractAverageGroupDelay(); uint32_t FFTSize() const { return mFFTSize; } - float RealData(uint32_t aIndex) const { return mOutputBuffer[aIndex].r; } - float& RealData(uint32_t aIndex) { return mOutputBuffer[aIndex].r; } - float ImagData(uint32_t aIndex) const { return mOutputBuffer[aIndex].i; } - float& ImagData(uint32_t aIndex) { return mOutputBuffer[aIndex].i; } + float RealData(uint32_t aIndex) const { + MOZ_ASSERT(!mInversePerformed); + return mOutputBuffer[aIndex].r; + } + float& RealData(uint32_t aIndex) { + MOZ_ASSERT(!mInversePerformed); + return mOutputBuffer[aIndex].r; + } + float ImagData(uint32_t aIndex) const { + MOZ_ASSERT(!mInversePerformed); + return mOutputBuffer[aIndex].i; + } + float& ImagData(uint32_t aIndex) { + MOZ_ASSERT(!mInversePerformed); + return mOutputBuffer[aIndex].i; + } size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { size_t amount = 0; -#if defined(MOZ_LIBAV_FFT) - auto ComputedSizeOfContextIfSet = [this](void* aContext) -> size_t { - if (!aContext) { - return 0; - } - // RDFTContext is only forward declared in public headers, but this is - // an estimate based on a value of 231 seen requested from - // _aligned_alloc on Win64. Don't use malloc_usable_size() because the - // context pointer is not necessarily from malloc. - size_t amount = 232; - // Add size of allocations performed in ff_fft_init(). - // The maximum FFT size used is 32768 = 2^15 and so revtab32 is not - // allocated. - MOZ_ASSERT(mFFTSize <= 32768); - amount += mFFTSize * (sizeof(uint16_t) + 2 * sizeof(float)); - - return amount; - }; + // malloc_usable_size can't be used here because the pointer isn't + // necessarily from malloc. This value has been manually checked. + if (mTxCtx) { + amount += 711; + } + if (mTxCtx) { + amount += 711; + } - amount += ComputedSizeOfContextIfSet(mAvRDFT); - amount += ComputedSizeOfContextIfSet(mAvIRDFT); -#else -# ifdef BUILD_ARM_NEON - amount += aMallocSizeOf(mOmxFFT); - amount += aMallocSizeOf(mOmxIFFT); -# endif -# ifdef USE_SIMD -# error kiss fft uses malloc only when USE_SIMD is not defined -# endif - amount += aMallocSizeOf(mKissFFT); - amount += aMallocSizeOf(mKissIFFT); -#endif amount += mOutputBuffer.ShallowSizeOfExcludingThis(aMallocSizeOf); return amount; } @@ -228,119 +158,67 @@ class FFTBlock final { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } - private: FFTBlock(const FFTBlock& other) = delete; void operator=(const FFTBlock& other) = delete; + private: bool EnsureFFT() { -#if defined(MOZ_LIBAV_FFT) - if (!mAvRDFT) { - if (!sRDFTFuncs.init) { + if (!mTxCtx) { + if (!sFFTFuncs.init) { return false; } - - mAvRDFT = sRDFTFuncs.init(FloorLog2(mFFTSize), DFT_R2C); - } -#else -# ifdef BUILD_ARM_NEON - if (mozilla::supports_neon()) { - if (!mOmxFFT) { - mOmxFFT = createOmxFFT(mFFTSize); - } - } else -# endif - { - if (!mKissFFT) { - mKissFFT = kiss_fftr_alloc(mFFTSize, 0, nullptr, nullptr); - } + // Forward transform is always unscaled for our purpose. + float scale = 1.0f; + int rv = sFFTFuncs.init(&mTxCtx, &mFn, AV_TX_FLOAT_RDFT, 0 /* forward */, + AssertedCast<int>(mFFTSize), &scale, 0); + MOZ_ASSERT(!rv, "av_tx_init: invalid parameters (forward)"); + return !rv; } -#endif return true; } - bool EnsureIFFT() { -#if defined(MOZ_LIBAV_FFT) - if (!mAvIRDFT) { - if (!sRDFTFuncs.init) { + if (!mITxCtx) { + if (!sFFTFuncs.init) { return false; } - - mAvIRDFT = sRDFTFuncs.init(FloorLog2(mFFTSize), IDFT_C2R); - } -#else -# ifdef BUILD_ARM_NEON - if (mozilla::supports_neon()) { - if (!mOmxIFFT) { - mOmxIFFT = createOmxFFT(mFFTSize); - } - } else -# endif - { - if (!mKissIFFT) { - mKissIFFT = kiss_fftr_alloc(mFFTSize, 1, nullptr, nullptr); - } + int rv = + sFFTFuncs.init(&mITxCtx, &mIFn, AV_TX_FLOAT_RDFT, 1 /* inverse */, + AssertedCast<int>(mFFTSize), &mInverseScaling, 0); + MOZ_ASSERT(!rv, "av_tx_init: invalid parameters (inverse)"); + return !rv; } -#endif return true; } - -#ifdef BUILD_ARM_NEON - static OMXFFTSpec_R_F32* createOmxFFT(uint32_t aFFTSize) { - MOZ_ASSERT((aFFTSize & (aFFTSize - 1)) == 0); - OMX_INT bufSize; - OMX_INT order = FloorLog2(aFFTSize); - MOZ_ASSERT(aFFTSize >> order == 1); - OMXResult status = omxSP_FFTGetBufSize_R_F32(order, &bufSize); - if (status == OMX_Sts_NoErr) { - OMXFFTSpec_R_F32* context = - static_cast<OMXFFTSpec_R_F32*>(malloc(bufSize)); - if (omxSP_FFTInit_R_F32(context, order) != OMX_Sts_NoErr) { - return nullptr; - } - return context; - } - return nullptr; - } -#endif - void Clear() { -#if defined(MOZ_LIBAV_FFT) - if (mAvRDFT) { - sRDFTFuncs.end(mAvRDFT); - mAvRDFT = nullptr; + if (mTxCtx) { + sFFTFuncs.uninit(&mTxCtx); + mTxCtx = nullptr; + mFn = nullptr; } - if (mAvIRDFT) { - sRDFTFuncs.end(mAvIRDFT); - mAvIRDFT = nullptr; + if (mITxCtx) { + sFFTFuncs.uninit(&mITxCtx); + mITxCtx = nullptr; + mIFn = nullptr; } -#else -# ifdef BUILD_ARM_NEON - free(mOmxFFT); - free(mOmxIFFT); - mOmxFFT = mOmxIFFT = nullptr; -# endif - free(mKissFFT); - free(mKissIFFT); - mKissFFT = mKissIFFT = nullptr; -#endif } void AddConstantGroupDelay(double sampleFrameDelay); void InterpolateFrequencyComponents(const FFTBlock& block0, const FFTBlock& block1, double interp); -#if defined(MOZ_LIBAV_FFT) - static FFmpegRDFTFuncs sRDFTFuncs; - RDFTContext* mAvRDFT; - RDFTContext* mAvIRDFT; -#else - kiss_fftr_cfg mKissFFT; - kiss_fftr_cfg mKissIFFT; -# ifdef BUILD_ARM_NEON - OMXFFTSpec_R_F32* mOmxFFT; - OMXFFTSpec_R_F32* mOmxIFFT; -# endif -#endif + static FFmpegFFTFuncs sFFTFuncs; + // Context and function pointer for forward transform + AVTXContext* mTxCtx{}; + av_tx_fn mFn{}; + // Context and function pointer for inverse transform + AVTXContext* mITxCtx{}; + av_tx_fn mIFn{}; AlignedTArray<ComplexU> mOutputBuffer; - uint32_t mFFTSize; + uint32_t mFFTSize{}; + // A scaling that is performed when doing an inverse transform. The forward + // transform is always unscaled. + float mInverseScaling; +#ifdef DEBUG + bool mInversePerformed = false; +#endif }; } // namespace mozilla diff --git a/dom/media/webaudio/OscillatorNode.cpp b/dom/media/webaudio/OscillatorNode.cpp index 70592f6e96..65a928f126 100644 --- a/dom/media/webaudio/OscillatorNode.cpp +++ b/dom/media/webaudio/OscillatorNode.cpp @@ -329,10 +329,8 @@ class OscillatorNodeEngine final : public AudioNodeEngine { case OscillatorType::Custom: ComputeCustom(output, start, end, frequency, detune); break; - case OscillatorType::EndGuard_: - MOZ_ASSERT_UNREACHABLE("end guard"); - // Avoid `default:` so that `-Wswitch` catches missing enumerators - // at compile time. + // Avoid `default:` so that `-Wswitch` catches missing enumerators at + // compile time. }; } diff --git a/dom/media/webaudio/blink/FFTConvolver.cpp b/dom/media/webaudio/blink/FFTConvolver.cpp index 2ade9031ce..f9b456a0d4 100644 --- a/dom/media/webaudio/blink/FFTConvolver.cpp +++ b/dom/media/webaudio/blink/FFTConvolver.cpp @@ -85,7 +85,7 @@ const float* FFTConvolver::process(FFTBlock* fftKernel, const float* sourceP) { // The input buffer is now filled (get frequency-domain version) m_frame.PerformFFT(m_inputBuffer.Elements()); m_frame.Multiply(*fftKernel); - m_frame.GetInverseWithoutScaling(m_outputBuffer.Elements()); + m_frame.GetInverse(m_outputBuffer.Elements()); // Overlap-add 1st half from previous time AudioBufferAddWithScale(m_lastOverlapBuffer.Elements(), 1.0f, diff --git a/dom/media/webaudio/blink/HRTFKernel.cpp b/dom/media/webaudio/blink/HRTFKernel.cpp index ecaa846a66..96a53609f2 100644 --- a/dom/media/webaudio/blink/HRTFKernel.cpp +++ b/dom/media/webaudio/blink/HRTFKernel.cpp @@ -38,7 +38,7 @@ static float extractAverageGroupDelay(float* impulseP, size_t length) { // Check for power-of-2. MOZ_ASSERT(length && (length & (length - 1)) == 0); - FFTBlock estimationFrame(length); + FFTBlock estimationFrame(length, 1.f / length); estimationFrame.PerformFFT(impulseP); float frameDelay = diff --git a/dom/media/webaudio/blink/PeriodicWave.cpp b/dom/media/webaudio/blink/PeriodicWave.cpp index 6b1d173008..4ed8829928 100644 --- a/dom/media/webaudio/blink/PeriodicWave.cpp +++ b/dom/media/webaudio/blink/PeriodicWave.cpp @@ -266,7 +266,7 @@ void PeriodicWave::createBandLimitedTables(float fundamentalFrequency, // Apply an inverse FFT to generate the time-domain table data. float* data = m_bandLimitedTables[rangeIndex]->Elements(); - frame.GetInverseWithoutScaling(data); + frame.GetInverse(data); // For the first range (which has the highest power), calculate // its peak value then compute normalization scale. diff --git a/dom/media/webaudio/moz.build b/dom/media/webaudio/moz.build index 2e82d3daa8..3ee8c0aa76 100644 --- a/dom/media/webaudio/moz.build +++ b/dom/media/webaudio/moz.build @@ -130,8 +130,6 @@ if CONFIG["TARGET_CPU"] == "aarch64" or CONFIG["BUILD_ARM_NEON"]: LOCAL_INCLUDES += ["/third_party/xsimd/include"] SOURCES += ["AudioNodeEngineNEON.cpp"] SOURCES["AudioNodeEngineNEON.cpp"].flags += CONFIG["NEON_FLAGS"] - if CONFIG["BUILD_ARM_NEON"]: - LOCAL_INCLUDES += ["/media/openmax_dl/dl/api/"] # Are we targeting x86 or x64? If so, build SSEX files. if CONFIG["INTEL_ARCHITECTURE"]: diff --git a/dom/media/webaudio/test/audioBufferSourceNodeDetached_worker.js b/dom/media/webaudio/test/audioBufferSourceNodeDetached_worker.js index 2a5a4bff89..439dbec0c4 100644 --- a/dom/media/webaudio/test/audioBufferSourceNodeDetached_worker.js +++ b/dom/media/webaudio/test/audioBufferSourceNodeDetached_worker.js @@ -1,3 +1,3 @@ -onmessage = function (event) { +onmessage = function () { postMessage("Pong"); }; diff --git a/dom/media/webaudio/test/test_OfflineAudioContext.html b/dom/media/webaudio/test/test_OfflineAudioContext.html index d9403566ae..6d8a907542 100644 --- a/dom/media/webaudio/test/test_OfflineAudioContext.html +++ b/dom/media/webaudio/test/test_OfflineAudioContext.html @@ -92,7 +92,7 @@ addLoadEvent(function() { ctx.startRendering().then(function() { ok(false, "Promise should not resolve when startRendering is called a second time on an OfflineAudioContext") finish(); - }).catch(function(err) { + }).catch(function() { ok(true, "Promise should reject when startRendering is called a second time on an OfflineAudioContext") finish(); }); @@ -106,7 +106,7 @@ addLoadEvent(function() { ctx.startRendering().then(function(b) { is(renderedBuffer, null, "The promise callback should be called first."); setOrCompareRenderedBuffer(b); - }).catch(function (error) { + }).catch(function () { ok(false, "The promise from OfflineAudioContext.startRendering should never be rejected"); }); }); diff --git a/dom/media/webaudio/test/test_WebAudioMemoryReporting.html b/dom/media/webaudio/test/test_WebAudioMemoryReporting.html index 693e558304..027e3bcc56 100644 --- a/dom/media/webaudio/test/test_WebAudioMemoryReporting.html +++ b/dom/media/webaudio/test/test_WebAudioMemoryReporting.html @@ -29,7 +29,7 @@ for (var i = 0; i < nodeTypes.length; ++i) { } } -var handleReport = function(aProcess, aPath, aKind, aUnits, aAmount, aDesc) { +var handleReport = function(aProcess, aPath, aKind, aUnits, aAmount) { if (aPath in usages) { usages[aPath] += aAmount; } diff --git a/dom/media/webaudio/test/test_audioBufferSourceNodeOffset.html b/dom/media/webaudio/test/test_audioBufferSourceNodeOffset.html index 0411b74ce5..e8549af37d 100644 --- a/dom/media/webaudio/test/test_audioBufferSourceNodeOffset.html +++ b/dom/media/webaudio/test/test_audioBufferSourceNodeOffset.html @@ -34,7 +34,7 @@ addLoadEvent(function() { var source = context.createBufferSource(); - source.onended = function(e) { + source.onended = function() { // The timing at which the audioprocess and ended listeners are called can // change, hence the fuzzy equal here. var errorRatio = samplesFromSource / (0.5 * context.sampleRate); diff --git a/dom/media/webaudio/test/test_audioContextSuspendResumeClose.html b/dom/media/webaudio/test/test_audioContextSuspendResumeClose.html index 36cf8f720c..62bdd3fd54 100644 --- a/dom/media/webaudio/test/test_audioContextSuspendResumeClose.html +++ b/dom/media/webaudio/test/test_audioContextSuspendResumeClose.html @@ -72,10 +72,10 @@ function tryLegalOpeerationsOnClosedContext(ctx) { }); }); loadFile("ting-44.1k-1ch.ogg", function(buf) { - ctx.decodeAudioData(buf).then(function(decodedBuf) { + ctx.decodeAudioData(buf).then(function() { ok(true, "decodeAudioData on a closed context should work, it did.") finish(); - }).catch(function(e){ + }).catch(function(){ ok(false, "decodeAudioData on a closed context should work, it did not"); finish(); }); @@ -103,7 +103,7 @@ function testMultiContextOutput() { silentBuffersInARow = 0; - sp2.onaudioprocess = function(e) { + sp2.onaudioprocess = function() { ac1.suspend().then(function() { is(ac1.state, "suspended", "ac1 is suspended"); sp2.onaudioprocess = checkSilence; @@ -350,7 +350,7 @@ function testOfflineAudioContext() { o.onstatechange = beforeStartRendering; - o.startRendering().then(function(buffer) { + o.startRendering().then(function() { finishedRendering = true; }); } diff --git a/dom/media/webaudio/test/test_bug1056032.html b/dom/media/webaudio/test/test_bug1056032.html index ba38267e19..77427ee5f0 100644 --- a/dom/media/webaudio/test/test_bug1056032.html +++ b/dom/media/webaudio/test/test_bug1056032.html @@ -19,7 +19,7 @@ addLoadEvent(function() { xhr.responseType = "arraybuffer"; xhr.onload = function() { var context = new AudioContext(); - context.decodeAudioData(xhr.response, function(b) { + context.decodeAudioData(xhr.response, function() { ok(true, "We can decode an mp3 using decodeAudioData"); SimpleTest.finish(); }, function() { diff --git a/dom/media/webaudio/test/test_bug867174.html b/dom/media/webaudio/test/test_bug867174.html index e949bcec41..dd66c77303 100644 --- a/dom/media/webaudio/test/test_bug867174.html +++ b/dom/media/webaudio/test/test_bug867174.html @@ -21,7 +21,7 @@ addLoadEvent(function() { sp.connect(ctx.destination); source.start(0); - sp.onaudioprocess = function(e) { + sp.onaudioprocess = function() { // Now set the buffer source.buffer = buffer; diff --git a/dom/media/webaudio/test/test_convolverNodeNormalization.html b/dom/media/webaudio/test/test_convolverNodeNormalization.html index 24cb7d1670..b55dbba0c2 100644 --- a/dom/media/webaudio/test/test_convolverNodeNormalization.html +++ b/dom/media/webaudio/test/test_convolverNodeNormalization.html @@ -13,7 +13,7 @@ const LENGTH = 12800; // tolerate 16-bit math. const EPSILON = 1.0 / Math.pow(2, 15); -function test_normalization_via_response_concat(delayIndex) +function test_normalization_via_response_concat() { var context = new OfflineAudioContext(1, LENGTH, sampleRate); diff --git a/dom/media/webaudio/test/test_decodeAudioDataOnDetachedBuffer.html b/dom/media/webaudio/test/test_decodeAudioDataOnDetachedBuffer.html index e7c6d2db0c..cd86c0a1d3 100644 --- a/dom/media/webaudio/test/test_decodeAudioDataOnDetachedBuffer.html +++ b/dom/media/webaudio/test/test_decodeAudioDataOnDetachedBuffer.html @@ -19,7 +19,7 @@ var testDecodeAudioDataOnDetachedBuffer = function(buffer) { is(buffer.byteLength, 0, "Buffer should be detached"); // call decodeAudioData on detached buffer - context.decodeAudioData(buffer).then(function(b) { + context.decodeAudioData(buffer).then(function() { ok(false, "We should not be able to decode the detached buffer but we do"); SimpleTest.finish(); }, function(r) { diff --git a/dom/media/webaudio/test/test_decodeAudioDataPromise.html b/dom/media/webaudio/test/test_decodeAudioDataPromise.html index 139a686db1..a686c275cd 100644 --- a/dom/media/webaudio/test/test_decodeAudioDataPromise.html +++ b/dom/media/webaudio/test/test_decodeAudioDataPromise.html @@ -27,7 +27,7 @@ var ac = new AudioContext(); expectNoException(function() { var p = ac.decodeAudioData(" "); ok(p instanceof Promise, "AudioContext.decodeAudioData should return a Promise"); - p.then(function(data) { + p.then(function() { ok(false, "Promise should not resolve with an invalid source buffer."); finish(); }).catch(function(e) { diff --git a/dom/media/webaudio/test/test_decodeAudioError.html b/dom/media/webaudio/test/test_decodeAudioError.html index f18b971ac4..b6dd6ff74b 100644 --- a/dom/media/webaudio/test/test_decodeAudioError.html +++ b/dom/media/webaudio/test/test_decodeAudioError.html @@ -34,11 +34,11 @@ function errorExpectedWithFile(file, errorMsg) { xhr.open("GET", file, true); xhr.responseType = "arraybuffer"; xhr.onload = function() { - ctx.decodeAudioData(xhr.response, buffer => { + ctx.decodeAudioData(xhr.response, () => { ok(false, "You should not be able to decode that"); finish(); }, e => test(e)) - .then(buffer => { + .then(() => { ok(false, "You should not be able to decode that"); finish(); }) diff --git a/dom/media/webaudio/test/test_dynamicsCompressorNode.html b/dom/media/webaudio/test/test_dynamicsCompressorNode.html index 05b6887a53..8f197aa32f 100644 --- a/dom/media/webaudio/test/test_dynamicsCompressorNode.html +++ b/dom/media/webaudio/test/test_dynamicsCompressorNode.html @@ -48,7 +48,7 @@ addLoadEvent(function() { osc.start(); var iteration = 0; - sp.onaudioprocess = function(e) { + sp.onaudioprocess = function() { if (iteration > 10) { ok(compressor.reduction < 0, "Feeding a full-scale sine to a compressor should result in an db" + diff --git a/dom/media/webaudio/test/test_event_listener_leaks.html b/dom/media/webaudio/test/test_event_listener_leaks.html index a3bcc9259e..f76e1d3e55 100644 --- a/dom/media/webaudio/test/test_event_listener_leaks.html +++ b/dom/media/webaudio/test/test_event_listener_leaks.html @@ -18,7 +18,7 @@ // exercise the leak condition. async function useAudioContext(contentWindow) { let ctx = new contentWindow.AudioContext(); - ctx.onstatechange = e => { + ctx.onstatechange = () => { contentWindow.stateChangeCount += 1; }; diff --git a/dom/media/webaudio/test/test_mediaStreamAudioSourceNodeNoGC.html b/dom/media/webaudio/test/test_mediaStreamAudioSourceNodeNoGC.html index 7920af9f7b..84d41df50e 100644 --- a/dom/media/webaudio/test/test_mediaStreamAudioSourceNodeNoGC.html +++ b/dom/media/webaudio/test/test_mediaStreamAudioSourceNodeNoGC.html @@ -50,7 +50,7 @@ function waitForAudio(analysisFunction, cancelPromise) { }); } -async function test(sourceNode) { +async function test() { try { await analyser.connect(context.destination); diff --git a/dom/media/webaudio/test/test_pannerNodeTail.html b/dom/media/webaudio/test/test_pannerNodeTail.html index 7035780bf2..bb60fe05da 100644 --- a/dom/media/webaudio/test/test_pannerNodeTail.html +++ b/dom/media/webaudio/test/test_pannerNodeTail.html @@ -153,7 +153,7 @@ function startTest() { } source.buffer = buffer; source.start(0); - source.onended = function(e) { + source.onended = function() { gotEnded = true; }; diff --git a/dom/media/webaudio/test/test_scriptProcessorNodeNotConnected.html b/dom/media/webaudio/test/test_scriptProcessorNodeNotConnected.html index fb45895380..0de8818d82 100644 --- a/dom/media/webaudio/test/test_scriptProcessorNodeNotConnected.html +++ b/dom/media/webaudio/test/test_scriptProcessorNodeNotConnected.html @@ -15,7 +15,7 @@ addLoadEvent(function() { var context = new AudioContext(); var sp = context.createScriptProcessor(2048, 2, 2); - sp.onaudioprocess = function(e) { + sp.onaudioprocess = function() { ok(false, "Should not call onaudioprocess if the node is not connected."); sp.onaudioprocess = null; SimpleTest.finish(); diff --git a/dom/media/webaudio/test/webaudio.js b/dom/media/webaudio/test/webaudio.js index 049e0e5af3..100c71f320 100644 --- a/dom/media/webaudio/test/webaudio.js +++ b/dom/media/webaudio/test/webaudio.js @@ -42,7 +42,7 @@ function expectRejectedPromise(that, func, exceptionName) { ok(promise instanceof Promise, "Expect a Promise"); promise - .then(function (res) { + .then(function () { ok(false, "Promise resolved when it should have been rejected."); }) .catch(function (err) { diff --git a/dom/media/webcodecs/AudioData.cpp b/dom/media/webcodecs/AudioData.cpp new file mode 100644 index 0000000000..0b21798be8 --- /dev/null +++ b/dom/media/webcodecs/AudioData.cpp @@ -0,0 +1,731 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "nsStringFwd.h" + +#include <utility> + +#include "AudioSampleFormat.h" +#include "WebCodecsUtils.h" +#include "js/StructuredClone.h" +#include "mozilla/Maybe.h" +#include "mozilla/Result.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGD +# undef LOGD +#endif // LOGD +#define LOGD(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +// +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(AudioData) +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(AudioData) + tmp->CloseIfNeeded(); + NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent) + NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER +NS_IMPL_CYCLE_COLLECTION_UNLINK_END +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(AudioData) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(AudioData) +// AudioData should be released as soon as its refcount drops to zero, +// without waiting for async deletion by the cycle collector, since it may hold +// a large-size PCM buffer. +NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(AudioData, CloseIfNeeded()) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioData) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +/* + * W3C Webcodecs AudioData implementation + */ + +AudioData::AudioData(nsIGlobalObject* aParent, + const AudioDataSerializedData& aData) + : mParent(aParent), + mTimestamp(aData.mTimestamp), + mNumberOfChannels(aData.mNumberOfChannels), + mNumberOfFrames(aData.mNumberOfFrames), + mSampleRate(aData.mSampleRate), + mAudioSampleFormat(aData.mAudioSampleFormat), + // The resource is not copied, but referenced + mResource(aData.mResource) { + MOZ_ASSERT(mParent); + MOZ_ASSERT(mResource, + "Resource should always be present then receiving a transfer."); +} + +AudioData::AudioData(const AudioData& aOther) + : mParent(aOther.mParent), + mTimestamp(aOther.mTimestamp), + mNumberOfChannels(aOther.mNumberOfChannels), + mNumberOfFrames(aOther.mNumberOfFrames), + mSampleRate(aOther.mSampleRate), + mAudioSampleFormat(aOther.mAudioSampleFormat), + // The resource is not copied, but referenced + mResource(aOther.mResource) { + MOZ_ASSERT(mParent); +} + +Result<already_AddRefed<AudioDataResource>, nsresult> +AudioDataResource::Construct( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aInit) { + FallibleTArray<uint8_t> copied; + uint8_t* rv = ProcessTypedArraysFixed( + aInit, [&](const Span<uint8_t>& aData) -> uint8_t* { + return copied.AppendElements(aData.Elements(), aData.Length(), + fallible); + }); + if (!rv) { + LOGE("AudioDataResource::Ctor: OOM"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return MakeAndAddRef<AudioDataResource>(std::move(copied)); +} + +AudioData::AudioData( + nsIGlobalObject* aParent, + already_AddRefed<mozilla::dom::AudioDataResource> aResource, + const AudioDataInit& aInit) + : mParent(aParent), + mTimestamp(aInit.mTimestamp), + mNumberOfChannels(aInit.mNumberOfChannels), + mNumberOfFrames(aInit.mNumberOfFrames), + mSampleRate(aInit.mSampleRate), + mAudioSampleFormat(Some(aInit.mFormat)), + mResource(std::move(aResource)) { + MOZ_ASSERT(mParent); +} + +AudioData::AudioData( + nsIGlobalObject* aParent, + already_AddRefed<mozilla::dom::AudioDataResource> aResource, + int64_t aTimestamp, uint32_t aNumberOfChannels, uint32_t aNumberOfFrames, + float aSampleRate, AudioSampleFormat aAudioSampleFormat) + : mParent(aParent), + mTimestamp(aTimestamp), + mNumberOfChannels(aNumberOfChannels), + mNumberOfFrames(aNumberOfFrames), + mSampleRate(aSampleRate), + mAudioSampleFormat(Some(aAudioSampleFormat)), + mResource(aResource) { + MOZ_ASSERT(mParent); +} + +nsIGlobalObject* AudioData::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* AudioData::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return AudioData_Binding::Wrap(aCx, this, aGivenProto); +} + +uint32_t BytesPerSamples(const mozilla::dom::AudioSampleFormat& aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::U8_planar: + return sizeof(uint8_t); + case AudioSampleFormat::S16: + case AudioSampleFormat::S16_planar: + return sizeof(int16_t); + case AudioSampleFormat::S32: + case AudioSampleFormat::F32: + case AudioSampleFormat::S32_planar: + case AudioSampleFormat::F32_planar: + return sizeof(float); + default: + MOZ_ASSERT_UNREACHABLE("wrong enum value"); + } + return 0; +} + +Result<Ok, nsCString> IsValidAudioDataInit(const AudioDataInit& aInit) { + if (aInit.mSampleRate <= 0.0) { + auto msg = nsLiteralCString("sampleRate must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + if (aInit.mNumberOfFrames == 0) { + auto msg = nsLiteralCString("mNumberOfFrames must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + if (aInit.mNumberOfChannels == 0) { + auto msg = nsLiteralCString("mNumberOfChannels must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + + uint64_t totalSamples = aInit.mNumberOfFrames * aInit.mNumberOfChannels; + uint32_t bytesPerSamples = BytesPerSamples(aInit.mFormat); + uint64_t totalSize = totalSamples * bytesPerSamples; + uint64_t arraySizeBytes = ProcessTypedArraysFixed( + aInit.mData, [&](const Span<uint8_t>& aData) -> uint64_t { + return aData.LengthBytes(); + }); + if (arraySizeBytes < totalSize) { + auto msg = + nsPrintfCString("Array of size %" PRIu64 + " not big enough, should be at least %" PRIu64 " bytes", + arraySizeBytes, totalSize); + LOGD("%s", msg.get()); + return Err(msg); + } + return Ok(); +} + +const char* FormatToString(AudioSampleFormat aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + return "u8"; + case AudioSampleFormat::S16: + return "s16"; + case AudioSampleFormat::S32: + return "s32"; + case AudioSampleFormat::F32: + return "f32"; + case AudioSampleFormat::U8_planar: + return "u8-planar"; + case AudioSampleFormat::S16_planar: + return "s16-planar"; + case AudioSampleFormat::S32_planar: + return "s32-planar"; + case AudioSampleFormat::F32_planar: + return "f32-planar"; + default: + MOZ_ASSERT_UNREACHABLE("wrong enum value"); + } + return "unsupported"; +} + +/* static */ +already_AddRefed<AudioData> AudioData::Constructor(const GlobalObject& aGlobal, + const AudioDataInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + LOGD("[%p] AudioData(fmt: %s, rate: %f, ch: %" PRIu32 ", ts: %" PRId64 ")", + global.get(), FormatToString(aInit.mFormat), aInit.mSampleRate, + aInit.mNumberOfChannels, aInit.mTimestamp); + if (!global) { + LOGE("Global unavailable"); + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + nsString errorMessage; + auto rv = IsValidAudioDataInit(aInit); + if (rv.isErr()) { + LOGD("AudioData::Constructor failure (IsValidAudioDataInit)"); + aRv.ThrowTypeError(rv.inspectErr()); + return nullptr; + } + auto resource = AudioDataResource::Construct(aInit.mData); + if (resource.isErr()) { + LOGD("AudioData::Constructor failure (OOM)"); + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + + return MakeAndAddRef<mozilla::dom::AudioData>(global, resource.unwrap(), + aInit); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-format +Nullable<mozilla::dom::AudioSampleFormat> AudioData::GetFormat() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mAudioSampleFormat); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-samplerate +float AudioData::SampleRate() const { + AssertIsOnOwningThread(); + return mSampleRate; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-numberofframes +uint32_t AudioData::NumberOfFrames() const { + AssertIsOnOwningThread(); + return mNumberOfFrames; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-numberofchannels +uint32_t AudioData::NumberOfChannels() const { + AssertIsOnOwningThread(); + return mNumberOfChannels; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-duration +uint64_t AudioData::Duration() const { + AssertIsOnOwningThread(); + // The spec isn't clear in which direction to convert to integer. + // https://github.com/w3c/webcodecs/issues/726 + return static_cast<uint64_t>( + static_cast<float>(USECS_PER_S * mNumberOfFrames) / mSampleRate); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-timestamp +int64_t AudioData::Timestamp() const { + AssertIsOnOwningThread(); + return mTimestamp; +} + +struct CopyToSpec { + CopyToSpec(uint32_t aFrameCount, uint32_t aFrameOffset, uint32_t mPlaneIndex, + AudioSampleFormat aFormat) + : mFrameCount(aFrameCount), + mFrameOffset(aFrameOffset), + mPlaneIndex(mPlaneIndex), + mFormat(aFormat) {} + + const uint32_t mFrameCount; + const uint32_t mFrameOffset; + const uint32_t mPlaneIndex; + const AudioSampleFormat mFormat; +}; + +bool IsInterleaved(const AudioSampleFormat& aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::S16: + case AudioSampleFormat::S32: + case AudioSampleFormat::F32: + return true; + case AudioSampleFormat::U8_planar: + case AudioSampleFormat::S16_planar: + case AudioSampleFormat::S32_planar: + case AudioSampleFormat::F32_planar: + return false; + }; + MOZ_ASSERT_UNREACHABLE("Invalid enum value"); + return false; +} + +size_t AudioData::ComputeCopyElementCount( + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv) { + // https://w3c.github.io/webcodecs/#compute-copy-element-count + // 1, 2 + auto destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + // 3, 4 + MOZ_ASSERT(destFormat.isSome()); + if (IsInterleaved(destFormat.value())) { + if (aOptions.mPlaneIndex > 0) { + auto msg = "Interleaved format, but plane index > 0"_ns; + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + } else { + if (aOptions.mPlaneIndex >= mNumberOfChannels) { + auto msg = nsPrintfCString( + "Plane index %" PRIu32 + " greater or equal than the number of channels %" PRIu32, + aOptions.mPlaneIndex, mNumberOfChannels); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + } + // 5 -- conversion between all formats supported + // 6 -- all planes have the same number of frames, always + uint64_t frameCount = mNumberOfFrames; + // 7 + if (aOptions.mFrameOffset >= frameCount) { + auto msg = nsPrintfCString("Frame offset of %" PRIu32 + " greater or equal than frame count %" PRIu64, + aOptions.mFrameOffset, frameCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + // 8, 9 + uint64_t copyFrameCount = frameCount - aOptions.mFrameOffset; + if (aOptions.mFrameCount.WasPassed()) { + if (aOptions.mFrameCount.Value() > copyFrameCount) { + auto msg = nsPrintfCString( + "Passed copy frame count of %" PRIu32 + " greater than available source frames for copy of %" PRIu64, + aOptions.mFrameCount.Value(), copyFrameCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + copyFrameCount = aOptions.mFrameCount.Value(); + } + // 10, 11 + uint64_t elementCount = copyFrameCount; + if (IsInterleaved(destFormat.value())) { + elementCount *= mNumberOfChannels; + } + return elementCount; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-allocationsize +// This method returns an int, that can be zero in case of success or error. +// Caller should check aRv to determine success or error. +uint32_t AudioData::AllocationSize(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + if (!mResource) { + auto msg = "allocationSize called on detached AudioData"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return 0; + } + size_t copyElementCount = ComputeCopyElementCount(aOptions, aRv); + if (aRv.Failed()) { + LOGD("AudioData::AllocationSize failure"); + // ComputeCopyElementCount has set the exception type. + return 0; + } + Maybe<mozilla::dom::AudioSampleFormat> destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + if (destFormat.isNothing()) { + auto msg = "AudioData has an unknown format"_ns; + LOGD("%s", msg.get()); + // See https://github.com/w3c/webcodecs/issues/727 -- it isn't clear yet + // what to do here + aRv.ThrowRangeError(msg); + return 0; + } + CheckedInt<size_t> bytesPerSample = BytesPerSamples(destFormat.ref()); + + auto res = bytesPerSample * copyElementCount; + if (res.isValid()) { + return res.value(); + } + aRv.ThrowRangeError("Allocation size too large"); + return 0; +} + +template <typename S, typename D> +void CopySamples(Span<S> aSource, Span<D> aDest, uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + if (IsInterleaved(aSourceFormat) && IsInterleaved(aCopyToSpec.mFormat)) { + MOZ_ASSERT(aCopyToSpec.mPlaneIndex == 0); + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount); + MOZ_ASSERT(aSource.Length() - aCopyToSpec.mFrameOffset >= + aCopyToSpec.mFrameCount); + // This turns into a regular memcpy if the types are in fact equal + ConvertAudioSamples(aSource.data() + aCopyToSpec.mFrameOffset, aDest.data(), + aCopyToSpec.mFrameCount * aSourceChannelCount); + return; + } + if (IsInterleaved(aSourceFormat) && !IsInterleaved(aCopyToSpec.mFormat)) { + DebugOnly<size_t> sourceFrameCount = aSource.Length() / aSourceChannelCount; + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount); + MOZ_ASSERT(aSource.Length() - aCopyToSpec.mFrameOffset >= + aCopyToSpec.mFrameCount); + // Interleaved to planar -- only copy samples of the correct channel to the + // destination + size_t readIndex = aCopyToSpec.mFrameOffset * aSourceChannelCount + + aCopyToSpec.mPlaneIndex; + for (size_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[i] = ConvertAudioSample<D>(aSource[readIndex]); + readIndex += aSourceChannelCount; + } + return; + } + + if (!IsInterleaved(aSourceFormat) && IsInterleaved(aCopyToSpec.mFormat)) { + MOZ_CRASH("This should never be hit -- current spec doesn't support it"); + // Planar to interleaved -- copy of all channels of the source into the + // destination buffer. + MOZ_ASSERT(aCopyToSpec.mPlaneIndex == 0); + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount * aSourceChannelCount); + MOZ_ASSERT(aSource.Length() - + aCopyToSpec.mFrameOffset * aSourceChannelCount >= + aCopyToSpec.mFrameCount * aSourceChannelCount); + size_t writeIndex = 0; + // Scan the source linearly and put each sample at the right position in the + // destination interleaved buffer. + size_t readIndex = 0; + for (size_t channel = 0; channel < aSourceChannelCount; channel++) { + writeIndex = channel; + for (size_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[writeIndex] = ConvertAudioSample<D>(aSource[readIndex]); + readIndex++; + writeIndex += aSourceChannelCount; + } + } + return; + } + if (!IsInterleaved(aSourceFormat) && !IsInterleaved(aCopyToSpec.mFormat)) { + // Planar to Planar / convert + copy from the right index in the source. + size_t offset = + aCopyToSpec.mPlaneIndex * aSource.Length() / aSourceChannelCount; + MOZ_ASSERT(aDest.Length() >= aSource.Length() / aSourceChannelCount - + aCopyToSpec.mFrameOffset); + for (uint32_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[i] = + ConvertAudioSample<D>(aSource[offset + aCopyToSpec.mFrameOffset + i]); + } + } +} + +nsCString AudioData::ToString() const { + if (!mResource) { + return nsCString("AudioData[detached]"); + } + return nsPrintfCString("AudioData[%zu bytes %s %fHz %" PRIu32 "x%" PRIu32 + "ch]", + mResource->Data().LengthBytes(), + FormatToString(mAudioSampleFormat.value()), + mSampleRate, mNumberOfFrames, mNumberOfChannels); +} + +nsCString CopyToToString(size_t aDestBufSize, + const AudioDataCopyToOptions& aOptions) { + return nsPrintfCString( + "AudioDataCopyToOptions[data: %zu bytes %s frame count:%" PRIu32 + " frame offset: %" PRIu32 " plane: %" PRIu32 "]", + aDestBufSize, + aOptions.mFormat.WasPassed() ? FormatToString(aOptions.mFormat.Value()) + : "null", + aOptions.mFrameCount.WasPassed() ? aOptions.mFrameCount.Value() : 0, + aOptions.mFrameOffset, aOptions.mPlaneIndex); +} + +using DataSpanType = + Variant<Span<uint8_t>, Span<int16_t>, Span<int32_t>, Span<float>>; + +DataSpanType GetDataSpan(Span<uint8_t> aSpan, const AudioSampleFormat aFormat) { + const size_t Length = aSpan.Length() / BytesPerSamples(aFormat); + // TODO: Check size so Span can be reasonably constructed? + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::U8_planar: + return AsVariant(aSpan); + case AudioSampleFormat::S16: + case AudioSampleFormat::S16_planar: + return AsVariant(Span(reinterpret_cast<int16_t*>(aSpan.data()), Length)); + case AudioSampleFormat::S32: + case AudioSampleFormat::S32_planar: + return AsVariant(Span(reinterpret_cast<int32_t*>(aSpan.data()), Length)); + case AudioSampleFormat::F32: + case AudioSampleFormat::F32_planar: + return AsVariant(Span(reinterpret_cast<float*>(aSpan.data()), Length)); + } + MOZ_ASSERT_UNREACHABLE("Invalid enum value"); + return AsVariant(aSpan); +} + +void CopySamples(DataSpanType& aSource, DataSpanType& aDest, + uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + aSource.match([&](auto& src) { + aDest.match([&](auto& dst) { + CopySamples(src, dst, aSourceChannelCount, aSourceFormat, aCopyToSpec); + }); + }); +} + +void DoCopy(Span<uint8_t> aSource, Span<uint8_t> aDest, + const uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + DataSpanType source = GetDataSpan(aSource, aSourceFormat); + DataSpanType dest = GetDataSpan(aDest, aCopyToSpec.mFormat); + CopySamples(source, dest, aSourceChannelCount, aSourceFormat, aCopyToSpec); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-copyto +void AudioData::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv) { + AssertIsOnOwningThread(); + + size_t destLength = ProcessTypedArraysFixed( + aDestination, [&](const Span<uint8_t>& aData) -> size_t { + return aData.LengthBytes(); + }); + + LOGD("AudioData::CopyTo %s -> %s", ToString().get(), + CopyToToString(destLength, aOptions).get()); + + if (!mResource) { + auto msg = "copyTo called on closed AudioData"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return; + } + + uint64_t copyElementCount = ComputeCopyElementCount(aOptions, aRv); + if (aRv.Failed()) { + LOGD("AudioData::CopyTo failed in ComputeCopyElementCount"); + return; + } + auto destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + + uint32_t bytesPerSample = BytesPerSamples(destFormat.value()); + CheckedInt<uint32_t> copyLength = bytesPerSample; + copyLength *= copyElementCount; + if (copyLength.value() > destLength) { + auto msg = nsPrintfCString( + "destination buffer of length %zu too small for copying %" PRIu64 + " elements", + destLength, bytesPerSample * copyElementCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return; + } + + uint32_t framesToCopy = mNumberOfFrames - aOptions.mFrameOffset; + if (aOptions.mFrameCount.WasPassed()) { + framesToCopy = aOptions.mFrameCount.Value(); + } + + CopyToSpec copyToSpec(framesToCopy, aOptions.mFrameOffset, + aOptions.mPlaneIndex, destFormat.value()); + + // Now a couple layers of macros to type the pointers and perform the actual + // copy. + ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) { + DoCopy(mResource->Data(), aData, mNumberOfChannels, + mAudioSampleFormat.value(), copyToSpec); + }); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-clone +already_AddRefed<AudioData> AudioData::Clone(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + if (!mResource) { + auto msg = "No media resource in the AudioData now"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return nullptr; + } + + return MakeAndAddRef<AudioData>(*this); +} + +// https://w3c.github.io/webcodecs/#close-audiodata +void AudioData::Close() { + AssertIsOnOwningThread(); + + mResource = nullptr; + mSampleRate = 0; + mNumberOfFrames = 0; + mNumberOfChannels = 0; + mAudioSampleFormat = Nothing(); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A1 +/* static */ +JSObject* AudioData::ReadStructuredClone(JSContext* aCx, + nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const AudioDataSerializedData& aData) { + JS::Rooted<JS::Value> value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr<AudioData> is created + // in the scope below. Otherwise, the static analysis infers the RefPtr cannot + // be safely destructed while the unrooted return JSObject* is on the stack. + { + RefPtr<AudioData> frame = MakeAndAddRef<AudioData>(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + LOGE("GetOrCreateDOMReflect failure"); + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-audiodata%E2%91%A2%E2%91%A2 +bool AudioData::WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // AudioData closed + if (!mResource) { + LOGD("AudioData was already close in WriteStructuredClone"); + return false; + } + const uint32_t index = aHolder->AudioData().Length(); + // https://github.com/w3c/webcodecs/issues/717 + // For now, serialization is only allowed in the same address space, it's OK + // to send a refptr here instead of copying the backing buffer. + aHolder->AudioData().AppendElement(AudioDataSerializedData(*this)); + + return !NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_AUDIODATA, index)); +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-steps +UniquePtr<AudioData::TransferredData> AudioData::Transfer() { + AssertIsOnOwningThread(); + + if (!mResource) { + // Closed + LOGD("AudioData was already close in Transfer"); + return nullptr; + } + + // This adds a ref to the resource + auto serialized = MakeUnique<AudioDataSerializedData>(*this); + // This removes the ref to the resource, effectively transfering the backing + // storage. + Close(); + return serialized; +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-receiving-steps +/* static */ +already_AddRefed<AudioData> AudioData::FromTransferred(nsIGlobalObject* aGlobal, + TransferredData* aData) { + MOZ_ASSERT(aData); + + return MakeAndAddRef<AudioData>(aGlobal, *aData); +} + +void AudioData::CloseIfNeeded() { + if (mResource) { + mResource = nullptr; + } +} + +#undef LOGD +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/AudioData.h b/dom/media/webcodecs/AudioData.h new file mode 100644 index 0000000000..4ae69a225a --- /dev/null +++ b/dom/media/webcodecs/AudioData.h @@ -0,0 +1,176 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_AudioData_h +#define mozilla_dom_AudioData_h + +#include "MediaData.h" +#include "WebCodecsUtils.h" +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Span.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; +class nsIURI; + +namespace mozilla::dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class Promise; +struct AudioDataBufferInit; +struct AudioDataCopyToOptions; +struct AudioDataInit; + +} // namespace mozilla::dom + +namespace mozilla::dom { + +class AudioData; +class AudioDataResource; +struct AudioDataSerializedData; + +class AudioData final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(AudioData) + + public: + AudioData(nsIGlobalObject* aParent, const AudioDataSerializedData& aData); + AudioData(nsIGlobalObject* aParent, + already_AddRefed<AudioDataResource> aResource, + const AudioDataInit& aInit); + AudioData(nsIGlobalObject* aParent, + already_AddRefed<mozilla::dom::AudioDataResource> aResource, + int64_t aTimestamp, uint32_t aNumberOfChannels, + uint32_t aNumberOfFrames, float aSampleRate, + AudioSampleFormat aAudioSampleFormat); + AudioData(const AudioData& aOther); + + protected: + ~AudioData() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<AudioData> Constructor(const GlobalObject& aGlobal, + const AudioDataInit& aInit, + ErrorResult& aRv); + + Nullable<mozilla::dom::AudioSampleFormat> GetFormat() const; + + float SampleRate() const; + + uint32_t NumberOfFrames() const; + + uint32_t NumberOfChannels() const; + + uint64_t Duration() const; // microseconds + + int64_t Timestamp() const; // microseconds + + uint32_t AllocationSize(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv); + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv); + + already_AddRefed<AudioData> Clone(ErrorResult& aRv); + + void Close(); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const AudioDataSerializedData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + // [Transferable] implementations: Transfer, FromTransferred + using TransferredData = AudioDataSerializedData; + + UniquePtr<TransferredData> Transfer(); + + static already_AddRefed<AudioData> FromTransferred(nsIGlobalObject* aGlobal, + TransferredData* aData); + + private: + size_t ComputeCopyElementCount(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv); + + nsCString ToString() const; + // AudioData can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(AudioData); } + void CloseIfNeeded(); + + nsCOMPtr<nsIGlobalObject> mParent; + + friend struct AudioDataSerializedData; + + int64_t mTimestamp; + uint32_t mNumberOfChannels; + uint32_t mNumberOfFrames; + float mSampleRate; + Maybe<AudioSampleFormat> mAudioSampleFormat; + RefPtr<mozilla::dom::AudioDataResource> mResource; +}; + +class AudioDataResource final { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataResource); + explicit AudioDataResource(FallibleTArray<uint8_t>&& aData) + : mData(std::move(aData)) {} + + explicit AudioDataResource() : mData() {} + + static AudioDataResource* Create(const Span<uint8_t>& aData) { + AudioDataResource* resource = new AudioDataResource(); + if (!resource->mData.AppendElements(aData, mozilla::fallible_t())) { + return nullptr; + } + return resource; + } + + static Result<already_AddRefed<AudioDataResource>, nsresult> Construct( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aInit); + + Span<uint8_t> Data() { return Span(mData.Elements(), mData.Length()); }; + + private: + ~AudioDataResource() = default; + // It's always possible for the allocation to fail -- the size is + // controled by script. + FallibleTArray<uint8_t> mData; +}; + +struct AudioDataSerializedData { + explicit AudioDataSerializedData(const AudioData& aFrom) + : mTimestamp(aFrom.Timestamp()), + mNumberOfChannels(aFrom.NumberOfChannels()), + mNumberOfFrames(aFrom.NumberOfFrames()), + mSampleRate(aFrom.SampleRate()), + mAudioSampleFormat(NullableToMaybe(aFrom.GetFormat())), + mResource(aFrom.mResource) {} + int64_t mTimestamp{}; + uint32_t mNumberOfChannels{}; + uint32_t mNumberOfFrames{}; + float mSampleRate{}; + Maybe<AudioSampleFormat> mAudioSampleFormat; + RefPtr<AudioDataResource> mResource; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_AudioData_h diff --git a/dom/media/webcodecs/AudioDecoder.cpp b/dom/media/webcodecs/AudioDecoder.cpp new file mode 100644 index 0000000000..6b554dcacf --- /dev/null +++ b/dom/media/webcodecs/AudioDecoder.cpp @@ -0,0 +1,481 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/AudioDecoder.h" +#include "mozilla/dom/AudioDecoderBinding.h" + +#include "DecoderTraits.h" +#include "MediaContainerType.h" +#include "MediaData.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/EncodedAudioChunk.h" +#include "mozilla/dom/EncodedAudioChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioDecoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(AudioDecoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(AudioDecoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioDecoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +/* + * Below are helper classes + */ + +AudioDecoderConfigInternal::AudioDecoderConfigInternal( + const nsAString& aCodec, uint32_t aSampleRate, uint32_t aNumberOfChannels, + Maybe<RefPtr<MediaByteBuffer>>&& aDescription) + : mCodec(aCodec), + mSampleRate(aSampleRate), + mNumberOfChannels(aNumberOfChannels), + mDescription(std::move(aDescription)) {} + +/*static*/ +UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create( + const AudioDecoderConfig& aConfig) { + nsCString errorMessage; + if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { + LOGE("Failed to create AudioDecoderConfigInternal: %s", errorMessage.get()); + return nullptr; + } + + Maybe<RefPtr<MediaByteBuffer>> description; + if (aConfig.mDescription.WasPassed()) { + auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value()); + if (rv.isErr()) { // Invalid description data. + nsCString error; + GetErrorName(rv.unwrapErr(), error); + LOGE( + "Failed to create AudioDecoderConfigInternal due to invalid " + "description data. Error: %s", + error.get()); + return nullptr; + } + description.emplace(rv.unwrap()); + } + + return UniquePtr<AudioDecoderConfigInternal>(new AudioDecoderConfigInternal( + aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels, + std::move(description))); +} + +/* + * The followings are helpers for AudioDecoder methods + */ + +struct AudioMIMECreateParam { + explicit AudioMIMECreateParam(const AudioDecoderConfigInternal& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} + explicit AudioMIMECreateParam(const AudioDecoderConfig& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} + + const nsString mParsedCodec; +}; + +// Map between WebCodecs pcm types as strings and codec numbers +// All other codecs +nsCString ConvertCodecName(const nsCString& aContainer, + const nsCString& aCodec) { + if (!aContainer.EqualsLiteral("x-wav")) { + return aCodec; + } + if (aCodec.EqualsLiteral("ulaw")) { + return nsCString("7"); + } + if (aCodec.EqualsLiteral("alaw")) { + return nsCString("6"); + } + if (aCodec.Find("f32")) { + return nsCString("3"); + } + // Linear PCM + return nsCString("1"); +} + +static nsTArray<nsCString> GuessMIMETypes(const AudioMIMECreateParam& aParam) { + nsCString codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec); + nsTArray<nsCString> types; + for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) { + codec = ConvertCodecName(container, codec); + nsPrintfCString mime("audio/%s; codecs=%s", container.get(), codec.get()); + types.AppendElement(mime); + } + return types; +} + +static bool IsSupportedAudioCodec(const nsAString& aCodec) { + LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") || + IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") || + aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") || + aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") || + aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") || + aCodec.EqualsLiteral("pcm-f32"); +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +template <typename Config> +static bool CanDecodeAudio(const Config& aConfig) { + auto param = AudioMIMECreateParam(aConfig); + if (!IsSupportedAudioCodec(param.mParsedCodec)) { + return false; + } + if (IsOnAndroid() && IsAACCodecString(param.mParsedCodec)) { + return false; + } + // TODO: Instead of calling CanHandleContainerType with the guessed the + // containers, DecoderTraits should provide an API to tell if a codec is + // decodable or not. + for (const nsCString& mime : GuessMIMETypes(param)) { + if (Maybe<MediaContainerType> containerType = + MakeMediaExtendedMIMEType(mime)) { + if (DecoderTraits::CanHandleContainerType( + *containerType, nullptr /* DecoderDoctorDiagnostics */) != + CANPLAY_NO) { + return true; + } + } + } + return false; +} + +static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo( + const AudioDecoderConfigInternal& aConfig) { + // TODO: Instead of calling GetTracksInfo with the guessed containers, + // DecoderTraits should provide an API to create the TrackInfo directly. + for (const nsCString& mime : GuessMIMETypes(AudioMIMECreateParam(aConfig))) { + if (Maybe<MediaContainerType> containerType = + MakeMediaExtendedMIMEType(mime)) { + if (nsTArray<UniquePtr<TrackInfo>> tracks = + DecoderTraits::GetTracksInfo(*containerType); + !tracks.IsEmpty()) { + return tracks; + } + } + } + return {}; +} + +static Result<Ok, nsresult> CloneConfiguration( + RootedDictionary<AudioDecoderConfig>& aDest, JSContext* aCx, + const AudioDecoderConfig& aConfig, ErrorResult& aRv) { + aDest.mCodec = aConfig.mCodec; + if (aConfig.mDescription.WasPassed()) { + aDest.mDescription.Construct(); + MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), + aConfig.mDescription.Value(), aRv)); + } + + aDest.mNumberOfChannels = aConfig.mNumberOfChannels; + aDest.mSampleRate = aConfig.mSampleRate; + + return Ok(); +} + +// https://w3c.github.io/webcodecs/#create-a-audiodata +static RefPtr<AudioData> CreateAudioData(nsIGlobalObject* aGlobalObject, + mozilla::AudioData* aData) { + MOZ_ASSERT(aGlobalObject); + MOZ_ASSERT(aData); + + auto buf = aData->MoveableData(); + // TODO: Ensure buf.Length() is a multiple of aData->mChannels and put it into + // AssertedCast<uint32_t> (sinze return type of buf.Length() is size_t). + uint32_t frames = buf.Length() / aData->mChannels; + RefPtr<AudioDataResource> resource = AudioDataResource::Create(Span{ + reinterpret_cast<uint8_t*>(buf.Data()), buf.Length() * sizeof(float)}); + return MakeRefPtr<AudioData>(aGlobalObject, resource.forget(), + aData->mTime.ToMicroseconds(), aData->mChannels, + frames, AssertedCast<float>(aData->mRate), + mozilla::dom::AudioSampleFormat::F32); +} + +/* static */ +bool AudioDecoderTraits::IsSupported( + const AudioDecoderConfigInternal& aConfig) { + return CanDecodeAudio(aConfig); +} + +/* static */ +Result<UniquePtr<TrackInfo>, nsresult> AudioDecoderTraits::CreateTrackInfo( + const AudioDecoderConfigInternal& aConfig) { + LOG("Create a AudioInfo from %s config", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig); + if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kAudioTrack) { + LOGE("Failed to get TrackInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + UniquePtr<TrackInfo> track(std::move(tracks[0])); + AudioInfo* ai = track->GetAsAudioInfo(); + if (!ai) { + LOGE("Failed to get AudioInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + if (aConfig.mDescription.isSome()) { + RefPtr<MediaByteBuffer> buf; + buf = aConfig.mDescription.value(); + if (buf) { + LOG("The given config has %zu bytes of description data", buf->Length()); + ai->mCodecSpecificConfig = + AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{buf}}; + } + } + + ai->mChannels = aConfig.mNumberOfChannels; + ai->mRate = aConfig.mSampleRate; + + LOG("Created AudioInfo %s (%" PRIu32 "ch %" PRIu32 + "Hz - with extra-data: %s)", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get(), ai->mChannels, ai->mChannels, + aConfig.mDescription.isSome() ? "yes" : "no"); + + return track; +} + +// https://w3c.github.io/webcodecs/#valid-audiodecoderconfig +/* static */ +bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe<nsString> codec = ParseCodecString(aConfig.mCodec); + if (!codec || codec->IsEmpty()) { + LOGE("Validating AudioDecoderConfig: invalid codec string"); + + aErrorMessage.AppendPrintf("Invalid codec string %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + return false; + } + + LOG("Validating AudioDecoderConfig: codec: %s %uch %uHz %s extradata", + NS_ConvertUTF16toUTF8(codec.value()).get(), aConfig.mNumberOfChannels, + aConfig.mSampleRate, aConfig.mDescription.WasPassed() ? "w/" : "no"); + + if (aConfig.mNumberOfChannels == 0) { + aErrorMessage.AppendPrintf("Invalid number of channels of %u", + aConfig.mNumberOfChannels); + return false; + } + + if (aConfig.mSampleRate == 0) { + aErrorMessage.AppendPrintf("Invalid sample-rate of %u", + aConfig.mNumberOfChannels); + return false; + } + + bool detached = + aConfig.mDescription.WasPassed() && + (aConfig.mDescription.Value().IsArrayBuffer() + ? JS::ArrayBuffer::fromObject( + aConfig.mDescription.Value().GetAsArrayBuffer().Obj()) + .isDetached() + : JS::ArrayBufferView::fromObject( + aConfig.mDescription.Value().GetAsArrayBufferView().Obj()) + .isDetached()); + + if (detached) { + LOGE("description is detached."); + return false; + } + + return true; +} + +/* static */ +UniquePtr<AudioDecoderConfigInternal> AudioDecoderTraits::CreateConfigInternal( + const AudioDecoderConfig& aConfig) { + return AudioDecoderConfigInternal::Create(aConfig); +} + +/* static */ +bool AudioDecoderTraits::IsKeyChunk(const EncodedAudioChunk& aInput) { + return aInput.Type() == EncodedAudioChunkType::Key; +} + +/* static */ +UniquePtr<EncodedAudioChunkData> AudioDecoderTraits::CreateInputInternal( + const EncodedAudioChunk& aInput) { + return aInput.Clone(); +} + +/* + * Below are AudioDecoder implementation + */ + +AudioDecoder::AudioDecoder(nsIGlobalObject* aParent, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<AudioDataOutputCallback>&& aOutputCallback) + : DecoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("AudioDecoder %p ctor", this); +} + +AudioDecoder::~AudioDecoder() { + LOG("AudioDecoder %p dtor", this); + Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); +} + +JSObject* AudioDecoder::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return AudioDecoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-audiodecoder-audiodecoder +/* static */ +already_AddRefed<AudioDecoder> AudioDecoder::Constructor( + const GlobalObject& aGlobal, const AudioDecoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef<AudioDecoder>( + global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError), + RefPtr<AudioDataOutputCallback>(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-audiodecoder-isconfigsupported +/* static */ +already_AddRefed<Promise> AudioDecoder::IsConfigSupported( + const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig, + ErrorResult& aRv) { + LOG("AudioDecoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(errorMessage); + return p.forget(); + } + + RootedDictionary<AudioDecoderConfig> config(aGlobal.Context()); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv); + if (r.isErr()) { + // This can only be an OOM: all members to clone are known to be valid + // because this is check by ::Validate above. + MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY && + aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY)); + return p.forget(); + } + + bool canDecode = CanDecodeAudio(config); + RootedDictionary<AudioDecoderSupport> s(aGlobal.Context()); + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canDecode); + + p->MaybeResolve(s); + return p.forget(); +} + +already_AddRefed<MediaRawData> AudioDecoder::InputDataToMediaRawData( + UniquePtr<EncodedAudioChunkData>&& aData, TrackInfo& aInfo, + const AudioDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aInfo.GetAsAudioInfo()); + + if (!aData) { + LOGE("No data for conversion"); + return nullptr; + } + + RefPtr<MediaRawData> sample = aData->TakeData(); + if (!sample) { + LOGE("Take no data for conversion"); + return nullptr; + } + + LOGV( + "EncodedAudioChunkData %p converted to %zu-byte MediaRawData - time: " + "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64 + "us, key-frame: %s", + aData.get(), sample->Size(), sample->mTime.ToMicroseconds(), + sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), + sample->mKeyframe ? "yes" : "no"); + + return sample.forget(); +} + +nsTArray<RefPtr<AudioData>> AudioDecoder::DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData, + AudioDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + + nsTArray<RefPtr<AudioData>> frames; + for (const RefPtr<MediaData>& data : aData) { + MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::AUDIO_DATA); + RefPtr<mozilla::AudioData> d(data->As<mozilla::AudioData>()); + frames.AppendElement(CreateAudioData(aGlobalObject, d.get())); + } + return frames; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/AudioDecoder.h b/dom/media/webcodecs/AudioDecoder.h new file mode 100644 index 0000000000..54fad68f55 --- /dev/null +++ b/dom/media/webcodecs/AudioDecoder.h @@ -0,0 +1,83 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_AudioDecoder_h +#define mozilla_dom_AudioDecoder_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/RefPtr.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/DecoderTemplate.h" +#include "mozilla/dom/DecoderTypes.h" +#include "mozilla/dom/RootedDictionary.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class AudioDataOutputCallback; +class EncodedAudioChunk; +class EncodedAudioChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class WebCodecsErrorCallback; +struct AudioDecoderConfig; +struct AudioDecoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class AudioDecoder final : public DecoderTemplate<AudioDecoderTraits> { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(AudioDecoder, DOMEventTargetHelper) + + public: + AudioDecoder(nsIGlobalObject* aParent, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<AudioDataOutputCallback>&& aOutputCallback); + + protected: + ~AudioDecoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<AudioDecoder> Constructor( + const GlobalObject& aGlobal, const AudioDecoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed<Promise> IsConfigSupported( + const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual already_AddRefed<MediaRawData> InputDataToMediaRawData( + UniquePtr<EncodedAudioChunkData>&& aData, TrackInfo& aInfo, + const AudioDecoderConfigInternal& aConfig) override; + + virtual nsTArray<RefPtr<AudioData>> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData, + AudioDecoderConfigInternal& aConfig) override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_AudioDecoder_h diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp index 5c63e27d48..095852c01d 100644 --- a/dom/media/webcodecs/DecoderAgent.cpp +++ b/dom/media/webcodecs/DecoderAgent.cpp @@ -6,8 +6,6 @@ #include "DecoderAgent.h" -#include <atomic> - #include "ImageContainer.h" #include "MediaDataDecoderProxy.h" #include "PDMFactory.h" diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp index 0fa25a208b..4d1c310737 100644 --- a/dom/media/webcodecs/DecoderTemplate.cpp +++ b/dom/media/webcodecs/DecoderTemplate.cpp @@ -139,8 +139,8 @@ void DecoderTemplate<DecoderType>::Configure(const ConfigType& aConfig, nsCString errorMessage; if (!DecoderType::Validate(aConfig, errorMessage)) { - aRv.ThrowTypeError( - nsPrintfCString("config is invalid: %s", errorMessage.get())); + LOG("Configure: Validate error: %s", errorMessage.get()); + aRv.ThrowTypeError(errorMessage); return; } @@ -322,13 +322,13 @@ void DecoderTemplate<DecoderType>::OutputDecodedData( MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(mActiveConfig); - nsTArray<RefPtr<VideoFrame>> frames = DecodedDataToOutputType( + nsTArray<RefPtr<OutputType>> frames = DecodedDataToOutputType( GetParentObject(), std::move(aData), *mActiveConfig); - RefPtr<VideoFrameOutputCallback> cb(mOutputCallback); - for (RefPtr<VideoFrame>& frame : frames) { + RefPtr<OutputCallbackType> cb(mOutputCallback); + for (RefPtr<OutputType>& frame : frames) { LOG("Outputing decoded data: ts: %" PRId64, frame->Timestamp()); - RefPtr<VideoFrame> f = frame; - cb->Call((VideoFrame&)(*f)); + RefPtr<OutputType> f = frame; + cb->Call((OutputType&)(*f)); } } @@ -881,6 +881,7 @@ void DecoderTemplate<DecoderType>::DestroyDecoderAgentIfAny() { } template class DecoderTemplate<VideoDecoderTraits>; +template class DecoderTemplate<AudioDecoderTraits>; #undef LOG #undef LOGW diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h index 56aa82046f..339a164f70 100644 --- a/dom/media/webcodecs/DecoderTypes.h +++ b/dom/media/webcodecs/DecoderTypes.h @@ -9,6 +9,9 @@ #include "MediaData.h" #include "mozilla/Maybe.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/AudioDecoderBinding.h" +#include "mozilla/dom/EncodedAudioChunk.h" #include "mozilla/dom/EncodedVideoChunk.h" #include "mozilla/dom/VideoColorSpaceBinding.h" #include "mozilla/dom/VideoDecoderBinding.h" @@ -58,17 +61,17 @@ class VideoDecoderConfigInternal { bool Equals(const VideoDecoderConfigInternal& aOther) const { if (mDescription.isSome() != aOther.mDescription.isSome()) { - return false; + return false; } if (mDescription.isSome() && aOther.mDescription.isSome()) { - auto lhs = mDescription.value(); - auto rhs = aOther.mDescription.value(); - if (lhs->Length() != rhs->Length()) { - return false; - } - if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { - return false; - } + auto lhs = mDescription.value(); + auto rhs = aOther.mDescription.value(); + if (lhs->Length() != rhs->Length()) { + return false; + } + if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { + return false; + } } return mCodec.Equals(aOther.mCodec) && mCodedHeight == aOther.mCodedHeight && @@ -111,6 +114,49 @@ class VideoDecoderTraits { const InputType& aInput); }; +class AudioDecoderConfigInternal { + public: + static UniquePtr<AudioDecoderConfigInternal> Create( + const AudioDecoderConfig& aConfig); + ~AudioDecoderConfigInternal() = default; + + nsString mCodec; + uint32_t mSampleRate; + uint32_t mNumberOfChannels; + Maybe<RefPtr<MediaByteBuffer>> mDescription; + // Compilation fix, should be abstracted by DecoderAgent since those are not + // supported + HardwareAcceleration mHardwareAcceleration = + HardwareAcceleration::No_preference; + Maybe<bool> mOptimizeForLatency; + + private: + AudioDecoderConfigInternal(const nsAString& aCodec, uint32_t aSampleRate, + uint32_t aNumberOfChannels, + Maybe<RefPtr<MediaByteBuffer>>&& aDescription); +}; + +class AudioDecoderTraits { + public: + static constexpr nsLiteralCString Name = "AudioDecoder"_ns; + using ConfigType = AudioDecoderConfig; + using ConfigTypeInternal = AudioDecoderConfigInternal; + using InputType = EncodedAudioChunk; + using InputTypeInternal = EncodedAudioChunkData; + using OutputType = AudioData; + using OutputCallbackType = AudioDataOutputCallback; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static UniquePtr<ConfigTypeInternal> CreateConfigInternal( + const ConfigType& aConfig); + static bool IsKeyChunk(const InputType& aInput); + static UniquePtr<InputTypeInternal> CreateInputInternal( + const InputType& aInput); +}; + } // namespace dom } // namespace mozilla diff --git a/dom/media/webcodecs/EncodedAudioChunk.cpp b/dom/media/webcodecs/EncodedAudioChunk.cpp new file mode 100644 index 0000000000..1cbb2ff9bd --- /dev/null +++ b/dom/media/webcodecs/EncodedAudioChunk.cpp @@ -0,0 +1,260 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/EncodedAudioChunk.h" +#include "mozilla/dom/EncodedAudioChunkBinding.h" + +#include <utility> + +#include "MediaData.h" +#include "TimeUnits.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/Logging.h" +#include "mozilla/PodOperations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/WebCodecsUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; +using mozilla::media::TimeUnit; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedAudioChunk, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedAudioChunk) +NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedAudioChunk) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedAudioChunk) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +EncodedAudioChunkData::EncodedAudioChunkData( + already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration) + : mBuffer(aBuffer), + mType(aType), + mTimestamp(aTimestamp), + mDuration(aDuration) { + MOZ_ASSERT(mBuffer); + MOZ_ASSERT(mBuffer->Length() == mBuffer->Size()); + MOZ_ASSERT(mBuffer->Length() <= + static_cast<size_t>(std::numeric_limits<uint32_t>::max())); +} + +UniquePtr<EncodedAudioChunkData> EncodedAudioChunkData::Clone() const { + if (!mBuffer) { + LOGE("No buffer in EncodedAudioChunkData %p to clone!", this); + return nullptr; + } + + // Since EncodedAudioChunkData can be zero-sized, cloning a zero-sized chunk + // is allowed. + if (mBuffer->Size() == 0) { + LOGW("Cloning an empty EncodedAudioChunkData %p", this); + } + + auto buffer = + MakeRefPtr<MediaAlignedByteBuffer>(mBuffer->Data(), mBuffer->Length()); + if (!buffer || buffer->Size() != mBuffer->Size()) { + LOGE("OOM to copy EncodedAudioChunkData %p", this); + return nullptr; + } + + return MakeUnique<EncodedAudioChunkData>(buffer.forget(), mType, mTimestamp, + Maybe<uint64_t>(mDuration)); +} + +already_AddRefed<MediaRawData> EncodedAudioChunkData::TakeData() { + if (!mBuffer || !(*mBuffer)) { + LOGE("EncodedAudioChunkData %p has no data!", this); + return nullptr; + } + + RefPtr<MediaRawData> sample(new MediaRawData(std::move(*mBuffer))); + sample->mKeyframe = mType == EncodedAudioChunkType::Key; + sample->mTime = TimeUnit::FromMicroseconds(mTimestamp); + sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp); + + if (mDuration) { + CheckedInt64 duration(*mDuration); + if (!duration.isValid()) { + LOGE("EncodedAudioChunkData %p 's duration exceeds TimeUnit's limit", + this); + return nullptr; + } + sample->mDuration = TimeUnit::FromMicroseconds(duration.value()); + } + + return sample.forget(); +} + +EncodedAudioChunk::EncodedAudioChunk( + nsIGlobalObject* aParent, already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration) + : EncodedAudioChunkData(std::move(aBuffer), aType, aTimestamp, + std::move(aDuration)), + mParent(aParent) {} + +EncodedAudioChunk::EncodedAudioChunk(nsIGlobalObject* aParent, + const EncodedAudioChunkData& aData) + : EncodedAudioChunkData(aData), mParent(aParent) {} + +nsIGlobalObject* EncodedAudioChunk::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* EncodedAudioChunk::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return EncodedAudioChunk_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#encodedaudiochunk-constructors +/* static */ +already_AddRefed<EncodedAudioChunk> EncodedAudioChunk::Constructor( + const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + auto buffer = ProcessTypedArrays( + aInit.mData, + [&](const Span<uint8_t>& aData, + JS::AutoCheckCannotGC&&) -> RefPtr<MediaAlignedByteBuffer> { + // Make sure it's in uint32_t's range. + CheckedUint32 byteLength(aData.Length()); + if (!byteLength.isValid()) { + aRv.Throw(NS_ERROR_INVALID_ARG); + return nullptr; + } + if (aData.Length() == 0) { + LOGW("Buffer for constructing EncodedAudioChunk is empty!"); + } + RefPtr<MediaAlignedByteBuffer> buf = MakeRefPtr<MediaAlignedByteBuffer>( + aData.Elements(), aData.Length()); + + // Instead of checking *buf, size comparision is used to allow + // constructing a zero-sized EncodedAudioChunk. + if (!buf || buf->Size() != aData.Length()) { + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + return buf; + }); + + RefPtr<EncodedAudioChunk> chunk(new EncodedAudioChunk( + global, buffer.forget(), aInit.mType, aInit.mTimestamp, + OptionalToMaybe(aInit.mDuration))); + return aRv.Failed() ? nullptr : chunk.forget(); +} + +EncodedAudioChunkType EncodedAudioChunk::Type() const { + AssertIsOnOwningThread(); + + return mType; +} + +int64_t EncodedAudioChunk::Timestamp() const { + AssertIsOnOwningThread(); + + return mTimestamp; +} + +Nullable<uint64_t> EncodedAudioChunk::GetDuration() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mDuration); +} + +uint32_t EncodedAudioChunk::ByteLength() const { + AssertIsOnOwningThread(); + MOZ_ASSERT(mBuffer); + + return static_cast<uint32_t>(mBuffer->Length()); +} + +// https://w3c.github.io/webcodecs/#dom-encodedaudiochunk-copyto +void EncodedAudioChunk::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) { + if (mBuffer->Size() > aData.size_bytes()) { + aRv.ThrowTypeError( + "Destination ArrayBuffer smaller than source EncodedAudioChunk"); + return; + } + + PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size()); + }); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps +/* static */ +JSObject* EncodedAudioChunk::ReadStructuredClone( + JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader, + const EncodedAudioChunkData& aData) { + JS::Rooted<JS::Value> value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr<EncodedAudioChunk> is + // created in the scope below. Otherwise, the static analysis infers the + // RefPtr cannot be safely destructed while the unrooted return JSObject* is + // on the stack. + { + auto frame = MakeRefPtr<EncodedAudioChunk>(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-serialization-steps +bool EncodedAudioChunk::WriteStructuredClone( + JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // Indexing the chunk and send the index to the receiver. + const uint32_t index = + static_cast<uint32_t>(aHolder->EncodedAudioChunks().Length()); + // The serialization is limited to the same process scope so it's ok to + // serialize a reference instead of a copy. + aHolder->EncodedAudioChunks().AppendElement(EncodedAudioChunkData(*this)); + return !NS_WARN_IF( + !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDAUDIOCHUNK, index)); +} + +#undef LOGW +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncodedAudioChunk.h b/dom/media/webcodecs/EncodedAudioChunk.h new file mode 100644 index 0000000000..53c059495a --- /dev/null +++ b/dom/media/webcodecs/EncodedAudioChunk.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncodedAudioChunk_h +#define mozilla_dom_EncodedAudioChunk_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class MediaAlignedByteBuffer; +class MediaRawData; + +namespace dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class StructuredCloneHolder; + +enum class EncodedAudioChunkType : uint8_t; +struct EncodedAudioChunkInit; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +class EncodedAudioChunkData { + public: + EncodedAudioChunkData(already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration); + EncodedAudioChunkData(const EncodedAudioChunkData& aData) = default; + ~EncodedAudioChunkData() = default; + + UniquePtr<EncodedAudioChunkData> Clone() const; + already_AddRefed<MediaRawData> TakeData(); + + protected: + // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX. + RefPtr<MediaAlignedByteBuffer> mBuffer; + EncodedAudioChunkType mType; + int64_t mTimestamp; + Maybe<uint64_t> mDuration; +}; + +class EncodedAudioChunk final : public EncodedAudioChunkData, + public nsISupports, + public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedAudioChunk) + + public: + EncodedAudioChunk(nsIGlobalObject* aParent, + already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration); + + EncodedAudioChunk(nsIGlobalObject* aParent, + const EncodedAudioChunkData& aData); + + protected: + ~EncodedAudioChunk() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<EncodedAudioChunk> Constructor( + const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit, + ErrorResult& aRv); + + EncodedAudioChunkType Type() const; + + int64_t Timestamp() const; + + Nullable<uint64_t> GetDuration() const; + + uint32_t ByteLength() const; + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const EncodedAudioChunkData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + private: + // EncodedAudioChunk can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncodedAudioChunk); + } + + nsCOMPtr<nsIGlobalObject> mParent; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncodedAudioChunk_h diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp index 47ca5bb459..18855e5cea 100644 --- a/dom/media/webcodecs/VideoDecoder.cpp +++ b/dom/media/webcodecs/VideoDecoder.cpp @@ -15,11 +15,8 @@ #include "MediaData.h" #include "VideoUtils.h" #include "mozilla/Assertions.h" -#include "mozilla/CheckedInt.h" -#include "mozilla/DebugOnly.h" #include "mozilla/Logging.h" #include "mozilla/Maybe.h" -#include "mozilla/StaticPrefs_dom.h" #include "mozilla/Try.h" #include "mozilla/Unused.h" #include "mozilla/dom/EncodedVideoChunk.h" @@ -31,7 +28,6 @@ #include "mozilla/dom/WebCodecsUtils.h" #include "nsPrintfCString.h" #include "nsReadableUtils.h" -#include "nsThreadUtils.h" #ifdef XP_MACOSX # include "MacIOSurfaceImage.h" @@ -97,9 +93,6 @@ VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const { return init; }; -static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); - VideoDecoderConfigInternal::VideoDecoderConfigInternal( const nsAString& aCodec, Maybe<uint32_t>&& aCodedHeight, Maybe<uint32_t>&& aCodedWidth, Maybe<VideoColorSpaceInternal>&& aColorSpace, @@ -129,7 +122,7 @@ UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create( Maybe<RefPtr<MediaByteBuffer>> description; if (aConfig.mDescription.WasPassed()) { - auto rv = GetExtraData(aConfig.mDescription.Value()); + auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value()); if (rv.isErr()) { // Invalid description data. LOGE( "Failed to create VideoDecoderConfigInternal due to invalid " @@ -168,12 +161,10 @@ nsString VideoDecoderConfigInternal::ToString() const { if (mColorSpace.isSome()) { rv.AppendPrintf("colorspace %s", "todo"); } - if (mDescription.isSome()) { + if (mDescription.isSome() && mDescription.value()) { rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length()); } - rv.AppendPrintf( - "hw accel: %s", - HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + rv.AppendPrintf("hw accel: %s", GetEnumString(mHardwareAcceleration).get()); if (mOptimizeForLatency.isSome()) { rv.AppendPrintf("optimize for latency: %s", mOptimizeForLatency.value() ? "true" : "false"); @@ -217,24 +208,6 @@ static nsTArray<nsCString> GuessMIMETypes(const MIMECreateParam& aParam) { return types; } -static bool IsSupportedCodec(const nsAString& aCodec) { - // H265 is unsupported. - if (!IsAV1CodecString(aCodec) && !IsVP9CodecString(aCodec) && - !IsVP8CodecString(aCodec) && !IsH264CodecString(aCodec)) { - return false; - } - - // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to - // starts with av01 and vp09. - // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry - if (StringBeginsWith(aCodec, u"vp9"_ns) || - StringBeginsWith(aCodec, u"av1"_ns)) { - return false; - } - - return true; -} - // https://w3c.github.io/webcodecs/#check-configuration-support template <typename Config> static bool CanDecode(const Config& aConfig) { @@ -243,12 +216,7 @@ static bool CanDecode(const Config& aConfig) { if (IsOnAndroid()) { return false; } - if (!IsSupportedCodec(param.mParsedCodec)) { - return false; - } - if (IsOnMacOS() && IsH264CodecString(param.mParsedCodec) && - !StaticPrefs::dom_media_webcodecs_force_osx_h264_enabled()) { - // This will be fixed in Bug 1846796. + if (!IsSupportedVideoCodec(param.mParsedCodec)) { return false; } // TODO: Instead of calling CanHandleContainerType with the guessed the @@ -284,18 +252,9 @@ static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo( return {}; } -static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { - RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>(); - if (!AppendTypedArrayDataTo(aBuffer, *data)) { - return Err(NS_ERROR_OUT_OF_MEMORY); - } - return data->Length() > 0 ? data : nullptr; -} - static Result<Ok, nsresult> CloneConfiguration( RootedDictionary<VideoDecoderConfig>& aDest, JSContext* aCx, - const VideoDecoderConfig& aConfig) { + const VideoDecoderConfig& aConfig, ErrorResult& aRv) { DebugOnly<nsCString> str; MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str)); @@ -312,7 +271,7 @@ static Result<Ok, nsresult> CloneConfiguration( if (aConfig.mDescription.WasPassed()) { aDest.mDescription.Construct(); MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), - aConfig.mDescription.Value())); + aConfig.mDescription.Value(), aRv)); } if (aConfig.mDisplayAspectHeight.WasPassed()) { aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value()); @@ -334,8 +293,10 @@ static Maybe<VideoPixelFormat> GuessPixelFormat(layers::Image* aImage) { // DMABUFSurfaceImage? if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) { const ImageUtils imageUtils(aImage); + Maybe<dom::ImageBitmapFormat> format = imageUtils.GetFormat(); Maybe<VideoPixelFormat> f = - ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + format.isSome() ? ImageBitmapFormatToVideoPixelFormat(format.value()) + : Nothing(); // ImageBitmapFormat cannot distinguish YUV420 or YUV420A. bool hasAlpha = aImage->AsPlanarYCbCrImage() && @@ -391,8 +352,6 @@ static VideoColorSpaceInternal GuessColorSpace( // Make an educated guess based on the coefficients. colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) { switch (aMatrix) { - case VideoMatrixCoefficients::EndGuard_: - MOZ_CRASH("This should not happen"); case VideoMatrixCoefficients::Bt2020_ncl: return VideoColorPrimaries::Bt2020; case VideoMatrixCoefficients::Rgb: @@ -529,9 +488,6 @@ static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) { case VideoMatrixCoefficients::Bt2020_ncl: colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020); break; - case VideoMatrixCoefficients::EndGuard_: - MOZ_ASSERT_UNREACHABLE("bad enum value"); - break; }; } } @@ -794,6 +750,21 @@ bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig, return false; } + bool detached = + aConfig.mDescription.WasPassed() && + (aConfig.mDescription.Value().IsArrayBuffer() + ? JS::ArrayBuffer::fromObject( + aConfig.mDescription.Value().GetAsArrayBuffer().Obj()) + .isDetached() + : JS::ArrayBufferView::fromObject( + aConfig.mDescription.Value().GetAsArrayBufferView().Obj()) + .isDetached()); + + if (detached) { + LOGE("description is detached."); + return false; + } + return true; } @@ -828,9 +799,7 @@ VideoDecoder::VideoDecoder(nsIGlobalObject* aParent, LOG("VideoDecoder %p ctor", this); } -VideoDecoder::~VideoDecoder() { - LOG("VideoDecoder %p dtor", this); -} +VideoDecoder::~VideoDecoder() { LOG("VideoDecoder %p dtor", this); } JSObject* VideoDecoder::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) { @@ -877,21 +846,17 @@ already_AddRefed<Promise> VideoDecoder::IsConfigSupported( nsCString errorMessage; if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { p->MaybeRejectWithTypeError(nsPrintfCString( - "VideoDecoderConfig is invalid: %s", errorMessage.get())); + "IsConfigSupported: config is invalid: %s", errorMessage.get())); return p.forget(); } - // TODO: Move the following works to another thread to unblock the current - // thread, as what spec suggests. - RootedDictionary<VideoDecoderConfig> config(aGlobal.Context()); - auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv); if (r.isErr()) { - nsresult e = r.unwrapErr(); - LOGE("Failed to clone VideoDecoderConfig. Error: 0x%08" PRIx32, - static_cast<uint32_t>(e)); - p->MaybeRejectWithTypeError("Failed to clone VideoDecoderConfig"); - aRv.Throw(e); + // This can only be an OOM: all members to clone are known to be valid + // because this is check by ::Validate above. + MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY && + aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY)); return p.forget(); } diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp index 0e71417cb0..f593f70c77 100644 --- a/dom/media/webcodecs/VideoEncoder.cpp +++ b/dom/media/webcodecs/VideoEncoder.cpp @@ -101,8 +101,7 @@ VideoEncoderConfigInternal::VideoEncoderConfigInternal( mBitrateMode(aConfig.mBitrateMode), mLatencyMode(aConfig.mLatencyMode), mContentHint(aConfig.mContentHint), - mAvc(aConfig.mAvc) { -} + mAvc(aConfig.mAvc) {} VideoEncoderConfigInternal::VideoEncoderConfigInternal( const VideoEncoderConfig& aConfig) @@ -119,8 +118,7 @@ VideoEncoderConfigInternal::VideoEncoderConfigInternal( mBitrateMode(aConfig.mBitrateMode), mLatencyMode(aConfig.mLatencyMode), mContentHint(OptionalToMaybe(aConfig.mContentHint)), - mAvc(OptionalToMaybe(aConfig.mAvc)) { -} + mAvc(OptionalToMaybe(aConfig.mAvc)) {} nsString VideoEncoderConfigInternal::ToString() const { nsString rv; @@ -137,26 +135,20 @@ nsString VideoEncoderConfigInternal::ToString() const { if (mFramerate.isSome()) { rv.AppendPrintf(", %lfHz", mFramerate.value()); } - rv.AppendPrintf( - " hw: %s", - HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); - rv.AppendPrintf(", alpha: %s", AlphaOptionValues::GetString(mAlpha).data()); + rv.AppendPrintf(" hw: %s", GetEnumString(mHardwareAcceleration).get()); + rv.AppendPrintf(", alpha: %s", GetEnumString(mAlpha).get()); if (mScalabilityMode.isSome()) { rv.AppendPrintf(", scalability mode: %s", NS_ConvertUTF16toUTF8(mScalabilityMode.value()).get()); } - rv.AppendPrintf( - ", bitrate mode: %s", - VideoEncoderBitrateModeValues::GetString(mBitrateMode).data()); - rv.AppendPrintf(", latency mode: %s", - LatencyModeValues::GetString(mLatencyMode).data()); + rv.AppendPrintf(", bitrate mode: %s", GetEnumString(mBitrateMode).get()); + rv.AppendPrintf(", latency mode: %s", GetEnumString(mLatencyMode).get()); if (mContentHint.isSome()) { rv.AppendPrintf(", content hint: %s", NS_ConvertUTF16toUTF8(mContentHint.value()).get()); } if (mAvc.isSome()) { - rv.AppendPrintf(", avc-specific: %s", - AvcBitstreamFormatValues::GetString(mAvc->mFormat).data()); + rv.AppendPrintf(", avc-specific: %s", GetEnumString(mAvc->mFormat).get()); } return rv; @@ -238,43 +230,57 @@ EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const { if (ExtractH264CodecDetails(mCodec, profile, constraints, level)) { if (profile == H264_PROFILE_BASE || profile == H264_PROFILE_MAIN || profile == H264_PROFILE_EXTENDED || profile == H264_PROFILE_HIGH) { - specific.emplace( - H264Specific(static_cast<H264_PROFILE>(profile), static_cast<H264_LEVEL>(level), format)); + specific.emplace(H264Specific(static_cast<H264_PROFILE>(profile), + static_cast<H264_LEVEL>(level), format)); } } } - // Only for vp9, not vp8 - if (codecType == CodecType::VP9) { - uint8_t profile, level, bitdepth, chromasubsampling; - mozilla::VideoColorSpace colorspace; - DebugOnly<bool> rv = ExtractVPXCodecDetails( - mCodec, profile, level, bitdepth, chromasubsampling, colorspace); -#ifdef DEBUG - if (!rv) { - LOGE("Error extracting VPX codec details, non fatal"); - } -#endif - specific.emplace(VP9Specific()); - } + uint8_t numTemporalLayers = 1; MediaDataEncoder::ScalabilityMode scalabilityMode; if (mScalabilityMode) { if (mScalabilityMode->EqualsLiteral("L1T2")) { scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2; + numTemporalLayers = 2; } else if (mScalabilityMode->EqualsLiteral("L1T3")) { scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3; + numTemporalLayers = 3; } else { scalabilityMode = MediaDataEncoder::ScalabilityMode::None; } } else { scalabilityMode = MediaDataEncoder::ScalabilityMode::None; } - return EncoderConfig( - codecType, {mWidth, mHeight}, usage, ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, - AssertedCast<uint8_t>(mFramerate.refOr(0.f)), 0, mBitrate.refOr(0), - mBitrateMode == VideoEncoderBitrateMode::Constant - ? MediaDataEncoder::BitrateMode::Constant - : MediaDataEncoder::BitrateMode::Variable, - hwPref, scalabilityMode, specific); + // Only for vp9, not vp8 + if (codecType == CodecType::VP9) { + uint8_t profile, level, bitdepth, chromasubsampling; + mozilla::VideoColorSpace colorspace; + DebugOnly<bool> rv = ExtractVPXCodecDetails( + mCodec, profile, level, bitdepth, chromasubsampling, colorspace); +#ifdef DEBUG + if (!rv) { + LOGE("Error extracting VPX codec details, non fatal"); + } +#endif + specific.emplace(VP9Specific( + VPXComplexity::Normal, /* Complexity */ + true, /* Resilience */ + numTemporalLayers, /* Number of temporal layers */ + true, /* Denoising */ + false, /* Auto resize */ + false, /* Frame dropping */ + true, /* Adaptive Qp */ + 1, /* Number of spatial layers */ + false /* Flexible */ + )); + } + return EncoderConfig(codecType, {mWidth, mHeight}, usage, + ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, + AssertedCast<uint8_t>(mFramerate.refOr(0.f)), 0, + mBitrate.refOr(0), + mBitrateMode == VideoEncoderBitrateMode::Constant + ? MediaDataEncoder::BitrateMode::Constant + : MediaDataEncoder::BitrateMode::Variable, + hwPref, scalabilityMode, specific); } already_AddRefed<WebCodecsConfigurationChangeList> VideoEncoderConfigInternal::Diff( @@ -326,27 +332,6 @@ VideoEncoderConfigInternal::Diff( return list.forget(); } -/* - * The followings are helpers for VideoEncoder methods - */ -static bool IsEncodeSupportedCodec(const nsAString& aCodec) { - LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get()); - if (!IsVP9CodecString(aCodec) && !IsVP8CodecString(aCodec) && - !IsH264CodecString(aCodec) && !IsAV1CodecString(aCodec)) { - return false; - } - - // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to - // starts with av01 and vp09. - // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry - if (StringBeginsWith(aCodec, u"vp9"_ns) || - StringBeginsWith(aCodec, u"av1"_ns)) { - return false; - } - - return true; -} - // https://w3c.github.io/webcodecs/#check-configuration-support static bool CanEncode(const RefPtr<VideoEncoderConfigInternal>& aConfig) { auto parsedCodecString = @@ -355,7 +340,7 @@ static bool CanEncode(const RefPtr<VideoEncoderConfigInternal>& aConfig) { if (IsOnAndroid()) { return false; } - if (!IsEncodeSupportedCodec(parsedCodecString)) { + if (!IsSupportedVideoCodec(parsedCodecString)) { return false; } diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp index 602bc95c29..bea5611e39 100644 --- a/dom/media/webcodecs/VideoFrame.cpp +++ b/dom/media/webcodecs/VideoFrame.cpp @@ -598,8 +598,6 @@ static bool IsYUVFormat(const VideoPixelFormat& aFormat) { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return false; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return false; } @@ -641,8 +639,6 @@ static VideoColorSpaceInit PickColorSpace( colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709); colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Iec61966_2_1); break; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return colorSpace; @@ -881,8 +877,6 @@ static Result<RefPtr<layers::Image>, nsCString> CreateImageFromBuffer( case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return CreateRGBAImageFromBuffer(aFormat, aSize, aBuffer); - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return Err(nsCString("Invalid image format")); } @@ -1390,8 +1384,9 @@ already_AddRefed<VideoFrame> VideoFrame::Constructor( } const ImageUtils imageUtils(image); + Maybe<dom::ImageBitmapFormat> f = imageUtils.GetFormat(); Maybe<VideoPixelFormat> format = - ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + f.isSome() ? ImageBitmapFormatToVideoPixelFormat(f.value()) : Nothing(); // TODO: Retrive/infer the duration, and colorspace. auto r = InitializeFrameFromOtherFrame( @@ -1789,15 +1784,13 @@ nsCString VideoFrame::ToString() const { return rv; } - rv.AppendPrintf( - "VideoFrame ts: %" PRId64 - ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s", - mTimestamp, - dom::VideoPixelFormatValues::GetString(mResource->mFormat->PixelFormat()) - .data(), - mCodedSize.width, mCodedSize.height, mVisibleRect.width, - mVisibleRect.height, mDisplaySize.width, mDisplaySize.height, - ColorSpaceInitToString(mColorSpace).get()); + rv.AppendPrintf("VideoFrame ts: %" PRId64 + ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s", + mTimestamp, + dom::GetEnumString(mResource->mFormat->PixelFormat()).get(), + mCodedSize.width, mCodedSize.height, mVisibleRect.width, + mVisibleRect.height, mDisplaySize.width, mDisplaySize.height, + ColorSpaceInitToString(mColorSpace).get()); if (mDuration) { rv.AppendPrintf(" dur: %" PRId64, mDuration.value()); @@ -2032,8 +2025,6 @@ gfx::SurfaceFormat VideoFrame::Format::ToSurfaceFormat() const { case VideoPixelFormat::BGRX: format = gfx::SurfaceFormat::B8G8R8X8; break; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return format; } @@ -2056,8 +2047,6 @@ void VideoFrame::Format::MakeOpaque() { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRX: return; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); } @@ -2077,8 +2066,6 @@ nsTArray<VideoFrame::Format::Plane> VideoFrame::Format::Planes() const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return {Plane::RGBA}; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return {}; @@ -2125,8 +2112,6 @@ uint32_t VideoFrame::Format::SampleBytes(const Plane& aPlane) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return 4; // 8 bits/sample, 32 bits/pixel - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return 0; @@ -2154,7 +2139,6 @@ gfx::IntSize VideoFrame::Format::SampleSize(const Plane& aPlane) const { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: - case VideoPixelFormat::EndGuard_: MOZ_ASSERT_UNREACHABLE("invalid format"); return {0, 0}; } @@ -2177,8 +2161,6 @@ bool VideoFrame::Format::IsValidSize(const gfx::IntSize& aSize) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return true; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return false; @@ -2205,8 +2187,6 @@ size_t VideoFrame::Format::SampleCount(const gfx::IntSize& aSize) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return (count * 4).value(); - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); @@ -2252,8 +2232,6 @@ uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return (width * mFormat->SampleBytes(aPlane)).value(); - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("invalid format"); } return 0; case Format::Plane::U: // and UV @@ -2269,7 +2247,6 @@ uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: - case VideoPixelFormat::EndGuard_: MOZ_ASSERT_UNREACHABLE("invalid format"); } return 0; diff --git a/dom/media/webcodecs/WebCodecsUtils.cpp b/dom/media/webcodecs/WebCodecsUtils.cpp index 1e03f616db..3507aba440 100644 --- a/dom/media/webcodecs/WebCodecsUtils.cpp +++ b/dom/media/webcodecs/WebCodecsUtils.cpp @@ -37,7 +37,7 @@ std::atomic<WebCodecsId> sNextId = 0; namespace mozilla::dom { /* - * The followings are helpers for VideoDecoder methods + * The followings are helpers for AudioDecoder and VideoDecoder methods */ nsTArray<nsCString> GuessContainers(const nsAString& aCodec) { @@ -57,6 +57,29 @@ nsTArray<nsCString> GuessContainers(const nsAString& aCodec) { return {"mp4"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; } + if (IsAACCodecString(aCodec)) { + return {"adts"_ns, "mp4"_ns}; + } + + if (aCodec.EqualsLiteral("vorbis") || aCodec.EqualsLiteral("opus")) { + return {"ogg"_ns}; + } + + if (aCodec.EqualsLiteral("flac")) { + return {"flac"_ns}; + } + + if (aCodec.EqualsLiteral("mp3")) { + return {"mp3"_ns}; + } + + if (aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") || + aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") || + aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") || + aCodec.EqualsLiteral("pcm-f32")) { + return {"x-wav"_ns}; + } + return {}; } @@ -106,7 +129,8 @@ static std::tuple<JS::ArrayBufferOrView, size_t, size_t> GetArrayBufferInfo( Result<Ok, nsresult> CloneBuffer( JSContext* aCx, OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc) { + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc, + ErrorResult& aRv) { std::tuple<JS::ArrayBufferOrView, size_t, size_t> info = GetArrayBufferInfo(aCx, aSrc); JS::Rooted<JS::ArrayBufferOrView> abov(aCx); @@ -121,6 +145,8 @@ Result<Ok, nsresult> CloneBuffer( JS::Rooted<JSObject*> cloned(aCx, JS::ArrayBufferClone(aCx, obj, offset, len)); if (NS_WARN_IF(!cloned)) { + aRv.MightThrowJSException(); + aRv.StealExceptionFromJSContext(aCx); return Err(NS_ERROR_OUT_OF_MEMORY); } @@ -151,8 +177,6 @@ gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix) { return gfx::YUVColorSpace::BT601; case VideoMatrixCoefficients::Bt2020_ncl: return gfx::YUVColorSpace::BT2020; - case VideoMatrixCoefficients::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoMatrixCoefficients"); return gfx::YUVColorSpace::Default; @@ -171,8 +195,7 @@ gfx::TransferFunction ToTransferFunction( case VideoTransferCharacteristics::Hlg: return gfx::TransferFunction::HLG; case VideoTransferCharacteristics::Linear: - case VideoTransferCharacteristics::EndGuard_: - break; + return gfx::TransferFunction::Default; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::TransferFunction::Default; @@ -190,8 +213,6 @@ gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries) { return gfx::ColorSpace2::BT2020; case VideoColorPrimaries::Smpte432: return gfx::ColorSpace2::DISPLAY_P3; - case VideoColorPrimaries::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::ColorSpace2::UNKNOWN; @@ -300,13 +321,6 @@ Maybe<VideoPixelFormat> ImageBitmapFormatToVideoPixelFormat( return Nothing(); } -Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraDataFromArrayBuffer( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { - RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>(); - Unused << AppendTypedArrayDataTo(aBuffer, *data); - return data->Length() > 0 ? data : nullptr; -} - bool IsOnAndroid() { #if defined(ANDROID) return true; @@ -364,15 +378,13 @@ struct ConfigurationChangeToString { } nsCString operator()( const HardwareAccelerationChange& aHardwareAccelerationChange) { - return nsPrintfCString("HW acceleration: %s", - dom::HardwareAccelerationValues::GetString( - aHardwareAccelerationChange.get()) - .data()); + return nsPrintfCString( + "HW acceleration: %s", + dom::GetEnumString(aHardwareAccelerationChange.get()).get()); } nsCString operator()(const AlphaChange& aAlphaChange) { - return nsPrintfCString( - "Alpha: %s", - dom::AlphaOptionValues::GetString(aAlphaChange.get()).data()); + return nsPrintfCString("Alpha: %s", + dom::GetEnumString(aAlphaChange.get()).get()); } nsCString operator()(const ScalabilityModeChange& aScalabilityModeChange) { if (aScalabilityModeChange.get().isNothing()) { @@ -383,15 +395,12 @@ struct ConfigurationChangeToString { NS_ConvertUTF16toUTF8(aScalabilityModeChange.get().value()).get()); } nsCString operator()(const BitrateModeChange& aBitrateModeChange) { - return nsPrintfCString( - "Bitrate mode: %s", - dom::VideoEncoderBitrateModeValues::GetString(aBitrateModeChange.get()) - .data()); + return nsPrintfCString("Bitrate mode: %s", + dom::GetEnumString(aBitrateModeChange.get()).get()); } nsCString operator()(const LatencyModeChange& aLatencyModeChange) { - return nsPrintfCString( - "Latency mode: %s", - dom::LatencyModeValues::GetString(aLatencyModeChange.get()).data()); + return nsPrintfCString("Latency mode: %s", + dom::GetEnumString(aLatencyModeChange.get()).get()); } nsCString operator()(const ContentHintChange& aContentHintChange) { return nsPrintfCString("Content hint: %s", @@ -489,9 +498,6 @@ WebCodecsConfigurationChangeList::ToPEMChangeList() const { return rv.forget(); } -#define ENUM_TO_STRING(enumType, enumValue) \ - enumType##Values::GetString(enumValue).data() - nsCString ColorSpaceInitToString( const dom::VideoColorSpaceInit& aColorSpaceInit) { nsCString rv("VideoColorSpace"); @@ -502,18 +508,15 @@ nsCString ColorSpaceInitToString( } if (!aColorSpaceInit.mMatrix.IsNull()) { rv.AppendPrintf(" matrix: %s", - ENUM_TO_STRING(dom::VideoMatrixCoefficients, - aColorSpaceInit.mMatrix.Value())); + GetEnumString(aColorSpaceInit.mMatrix.Value()).get()); } if (!aColorSpaceInit.mTransfer.IsNull()) { rv.AppendPrintf(" transfer: %s", - ENUM_TO_STRING(dom::VideoTransferCharacteristics, - aColorSpaceInit.mTransfer.Value())); + GetEnumString(aColorSpaceInit.mTransfer.Value()).get()); } if (!aColorSpaceInit.mPrimaries.IsNull()) { rv.AppendPrintf(" primaries: %s", - ENUM_TO_STRING(dom::VideoColorPrimaries, - aColorSpaceInit.mPrimaries.Value())); + GetEnumString(aColorSpaceInit.mPrimaries.Value()).get()); } return rv; @@ -575,4 +578,32 @@ nsString ConfigToString(const VideoDecoderConfig& aConfig) { return internal->ToString(); } -}; // namespace mozilla::dom +Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraDataFromArrayBuffer( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>(); + if (!AppendTypedArrayDataTo(aBuffer, *data)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return data->Length() > 0 ? data : nullptr; +} + +bool IsSupportedVideoCodec(const nsAString& aCodec) { + LOG("IsSupportedVideoCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + // The only codec string accepted for vp8 is "vp8" + if (!IsVP9CodecString(aCodec) && !IsH264CodecString(aCodec) && + !IsAV1CodecString(aCodec) && !aCodec.EqualsLiteral("vp8")) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://w3c.github.io/webcodecs/codec_registry.html. + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/WebCodecsUtils.h b/dom/media/webcodecs/WebCodecsUtils.h index 7c0e6b6bbc..196c57421d 100644 --- a/dom/media/webcodecs/WebCodecsUtils.h +++ b/dom/media/webcodecs/WebCodecsUtils.h @@ -8,6 +8,7 @@ #define MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H #include "ErrorList.h" +#include "MediaData.h" #include "js/TypeDecls.h" #include "mozilla/Maybe.h" #include "mozilla/MozPromise.h" @@ -86,7 +87,11 @@ Nullable<T> MaybeToNullable(const Maybe<T>& aOptional) { Result<Ok, nsresult> CloneBuffer( JSContext* aCx, OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc); + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc, + ErrorResult& aRv); + +Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraDataFromArrayBuffer( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); /* * The following are utilities to convert between VideoColorSpace values to @@ -232,6 +237,8 @@ Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString); nsString ConfigToString(const VideoDecoderConfig& aConfig); +bool IsSupportedVideoCodec(const nsAString& aCodec); + } // namespace dom } // namespace mozilla diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build index 267a822286..ddb5aad5cb 100644 --- a/dom/media/webcodecs/moz.build +++ b/dom/media/webcodecs/moz.build @@ -21,8 +21,11 @@ EXPORTS.mozilla += [ ] EXPORTS.mozilla.dom += [ + "AudioData.h", + "AudioDecoder.h", "DecoderTemplate.h", "DecoderTypes.h", + "EncodedAudioChunk.h", "EncodedVideoChunk.h", "EncoderAgent.h", "EncoderTemplate.h", @@ -35,8 +38,11 @@ EXPORTS.mozilla.dom += [ ] UNIFIED_SOURCES += [ + "AudioData.cpp", + "AudioDecoder.cpp", "DecoderAgent.cpp", "DecoderTemplate.cpp", + "EncodedAudioChunk.cpp", "EncodedVideoChunk.cpp", "EncoderAgent.cpp", "EncoderTemplate.cpp", diff --git a/dom/media/webrtc/MediaEngineFake.cpp b/dom/media/webrtc/MediaEngineFake.cpp index 8c69ec5e47..b14d80ffbb 100644 --- a/dom/media/webrtc/MediaEngineFake.cpp +++ b/dom/media/webrtc/MediaEngineFake.cpp @@ -136,10 +136,8 @@ MediaEngineFakeVideoSource::MediaEngineFakeVideoSource() mSettings->mHeight.Construct( int32_t(MediaEnginePrefs::DEFAULT_43_VIDEO_HEIGHT)); mSettings->mFrameRate.Construct(double(MediaEnginePrefs::DEFAULT_VIDEO_FPS)); - mSettings->mFacingMode.Construct( - NS_ConvertASCIItoUTF16(dom::VideoFacingModeEnumValues::strings - [uint8_t(VideoFacingModeEnum::Environment)] - .value)); + mSettings->mFacingMode.Construct(NS_ConvertASCIItoUTF16( + dom::GetEnumString(VideoFacingModeEnum::Environment))); } nsString MediaEngineFakeVideoSource::GetGroupId() { diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp index a6648b68c7..dbb5e7ff70 100644 --- a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -108,8 +108,7 @@ MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource( Maybe<VideoFacingModeEnum> facingMode = GetFacingMode(mMediaDevice->mRawName); if (facingMode.isSome()) { - NS_ConvertASCIItoUTF16 facingString( - dom::VideoFacingModeEnumValues::GetString(*facingMode)); + NS_ConvertASCIItoUTF16 facingString(dom::GetEnumString(*facingMode)); mSettings->mFacingMode.Construct(facingString); mFacingMode.emplace(facingString); } diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp index ee45bb0317..9d778d411d 100644 --- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -105,9 +105,9 @@ nsresult MediaEngineWebRTCMicrophoneSource::EvaluateSettings( prefs.mChannels = c.mChannelCount.Get(std::min(prefs.mChannels, maxChannels)); prefs.mChannels = std::max(1, std::min(prefs.mChannels, maxChannels)); - LOG("Audio config: agc: %d, noise: %d, channels: %d", - prefs.mAgcOn ? prefs.mAgc : -1, prefs.mNoiseOn ? prefs.mNoise : -1, - prefs.mChannels); + LOG("Mic source %p Audio config: aec: %s, agc: %s, noise: %s, channels: %d", + this, prefs.mAecOn ? "on" : "off", prefs.mAgcOn ? "on" : "off", + prefs.mNoiseOn ? "on" : "off", prefs.mChannels); *aOutPrefs = prefs; diff --git a/dom/media/webrtc/MediaTrackConstraints.h b/dom/media/webrtc/MediaTrackConstraints.h index 61e0ed85ea..a948bf1499 100644 --- a/dom/media/webrtc/MediaTrackConstraints.h +++ b/dom/media/webrtc/MediaTrackConstraints.h @@ -20,17 +20,6 @@ namespace mozilla { class LocalMediaDevice; class MediaDevice; -template <class EnumValuesStrings, class Enum> -static Enum StringToEnum(const EnumValuesStrings& aStrings, - const nsAString& aValue, Enum aDefaultValue) { - for (size_t i = 0; aStrings[i].value; i++) { - if (aValue.EqualsASCII(aStrings[i].value)) { - return Enum(i); - } - } - return aDefaultValue; -} - // Helper classes for orthogonal constraints without interdependencies. // Instead of constraining values, constrain the constraints themselves. class NormalizedConstraintSet { diff --git a/dom/media/webrtc/MediaTransportChild.h b/dom/media/webrtc/MediaTransportChild.h index ef08fc55aa..0f9f5e8714 100644 --- a/dom/media/webrtc/MediaTransportChild.h +++ b/dom/media/webrtc/MediaTransportChild.h @@ -21,8 +21,10 @@ class MediaTransportChild : public dom::PMediaTransportChild { mozilla::ipc::IPCResult RecvOnCandidate(const string& transportId, const CandidateInfo& candidateInfo); mozilla::ipc::IPCResult RecvOnAlpnNegotiated(const string& alpn); - mozilla::ipc::IPCResult RecvOnGatheringStateChange(const int& state); - mozilla::ipc::IPCResult RecvOnConnectionStateChange(const int& state); + mozilla::ipc::IPCResult RecvOnGatheringStateChange(const string& transportId, + const int& state); + mozilla::ipc::IPCResult RecvOnConnectionStateChange(const string& transportId, + const int& state); mozilla::ipc::IPCResult RecvOnPacketReceived(const string& transportId, const MediaPacket& packet); mozilla::ipc::IPCResult RecvOnEncryptedSending(const string& transportId, diff --git a/dom/media/webrtc/PMediaTransport.ipdl b/dom/media/webrtc/PMediaTransport.ipdl index e7501ed814..ae6580f768 100644 --- a/dom/media/webrtc/PMediaTransport.ipdl +++ b/dom/media/webrtc/PMediaTransport.ipdl @@ -88,8 +88,8 @@ parent: child: async OnCandidate(string transportId, CandidateInfo candidateInfo); async OnAlpnNegotiated(string alpn); - async OnGatheringStateChange(int state); - async OnConnectionStateChange(int state); + async OnGatheringStateChange(string transportId, int state); + async OnConnectionStateChange(string transportId, int state); async OnPacketReceived(string transportId, MediaPacket packet); async OnEncryptedSending(string transportId, MediaPacket packet); async OnStateChange(string transportId, int state); diff --git a/dom/media/webrtc/WebrtcGlobal.h b/dom/media/webrtc/WebrtcGlobal.h index d610ee5d10..e9ec2ced51 100644 --- a/dom/media/webrtc/WebrtcGlobal.h +++ b/dom/media/webrtc/WebrtcGlobal.h @@ -8,6 +8,7 @@ #include "WebrtcIPCTraits.h" #include "ipc/EnumSerializer.h" #include "ipc/IPCMessageUtilsSpecializations.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "mozilla/dom/BindingDeclarations.h" #include "mozilla/dom/RTCDataChannelBinding.h" #include "mozilla/dom/RTCStatsReportBinding.h" @@ -62,30 +63,22 @@ namespace IPC { template <> struct ParamTraits<mozilla::dom::RTCStatsType> - : public ContiguousEnumSerializer<mozilla::dom::RTCStatsType, - mozilla::dom::RTCStatsType::Codec, - mozilla::dom::RTCStatsType::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer<mozilla::dom::RTCStatsType> {}; template <> struct ParamTraits<mozilla::dom::RTCStatsIceCandidatePairState> - : public ContiguousEnumSerializer< - mozilla::dom::RTCStatsIceCandidatePairState, - mozilla::dom::RTCStatsIceCandidatePairState::Frozen, - mozilla::dom::RTCStatsIceCandidatePairState::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::RTCStatsIceCandidatePairState> {}; template <> struct ParamTraits<mozilla::dom::RTCIceCandidateType> - : public ContiguousEnumSerializer< - mozilla::dom::RTCIceCandidateType, - mozilla::dom::RTCIceCandidateType::Host, - mozilla::dom::RTCIceCandidateType::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::RTCIceCandidateType> {}; template <> struct ParamTraits<mozilla::dom::RTCBundlePolicy> - : public ContiguousEnumSerializer< - mozilla::dom::RTCBundlePolicy, - mozilla::dom::RTCBundlePolicy::Balanced, - mozilla::dom::RTCBundlePolicy::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer<mozilla::dom::RTCBundlePolicy> { +}; DEFINE_IPC_SERIALIZER_WITH_FIELDS(mozilla::dom::RTCIceServerInternal, mUrls, mCredentialProvided, mUserNameProvided); @@ -218,10 +211,8 @@ DEFINE_IPC_SERIALIZER_WITH_FIELDS(mozilla::dom::RTCDataChannelStats, mId, template <> struct ParamTraits<mozilla::dom::RTCDataChannelState> - : public ContiguousEnumSerializer< - mozilla::dom::RTCDataChannelState, - mozilla::dom::RTCDataChannelState::Connecting, - mozilla::dom::RTCDataChannelState::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::RTCDataChannelState> {}; DEFINE_IPC_SERIALIZER_WITH_FIELDS(mozilla::dom::RTCCodecStats, mTimestamp, mType, mId, mPayloadType, mCodecType, @@ -230,9 +221,7 @@ DEFINE_IPC_SERIALIZER_WITH_FIELDS(mozilla::dom::RTCCodecStats, mTimestamp, template <> struct ParamTraits<mozilla::dom::RTCCodecType> - : public ContiguousEnumSerializer<mozilla::dom::RTCCodecType, - mozilla::dom::RTCCodecType::Encode, - mozilla::dom::RTCCodecType::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer<mozilla::dom::RTCCodecType> {}; } // namespace IPC #endif // _WEBRTC_GLOBAL_H_ diff --git a/dom/media/webrtc/WebrtcIPCTraits.h b/dom/media/webrtc/WebrtcIPCTraits.h index b076745608..5b526da67f 100644 --- a/dom/media/webrtc/WebrtcIPCTraits.h +++ b/dom/media/webrtc/WebrtcIPCTraits.h @@ -9,6 +9,7 @@ #include "ipc/IPCMessageUtils.h" #include "ipc/IPCMessageUtilsSpecializations.h" #include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "mozilla/dom/RTCConfigurationBinding.h" #include "mozilla/media/webrtc/WebrtcGlobal.h" #include "mozilla/dom/CandidateInfo.h" @@ -62,17 +63,15 @@ struct ParamTraits<mozilla::dom::OwningStringOrStringSequence> { } }; -template <typename T> -struct WebidlEnumSerializer - : public ContiguousEnumSerializer<T, T(0), T::EndGuard_> {}; - template <> struct ParamTraits<mozilla::dom::RTCIceCredentialType> - : public WebidlEnumSerializer<mozilla::dom::RTCIceCredentialType> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::RTCIceCredentialType> {}; template <> struct ParamTraits<mozilla::dom::RTCIceTransportPolicy> - : public WebidlEnumSerializer<mozilla::dom::RTCIceTransportPolicy> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::RTCIceTransportPolicy> {}; DEFINE_IPC_SERIALIZER_WITH_FIELDS(mozilla::dom::RTCIceServer, mCredential, mCredentialType, mUrl, mUrls, mUsername) diff --git a/dom/media/webrtc/jsapi/MediaTransportHandler.cpp b/dom/media/webrtc/jsapi/MediaTransportHandler.cpp index 3ee95f36f6..cf95ef41d6 100644 --- a/dom/media/webrtc/jsapi/MediaTransportHandler.cpp +++ b/dom/media/webrtc/jsapi/MediaTransportHandler.cpp @@ -156,9 +156,9 @@ class MediaTransportHandlerSTS : public MediaTransportHandler, using MediaTransportHandler::OnRtcpStateChange; using MediaTransportHandler::OnStateChange; - void OnGatheringStateChange(NrIceCtx* aIceCtx, - NrIceCtx::GatheringState aState); - void OnConnectionStateChange(NrIceCtx* aIceCtx, + void OnGatheringStateChange(const std::string& aTransportId, + NrIceMediaStream::GatheringState aState); + void OnConnectionStateChange(NrIceMediaStream* aIceStream, NrIceCtx::ConnectionState aState); void OnCandidateFound(NrIceMediaStream* aStream, const std::string& aCandidate, @@ -589,8 +589,6 @@ void MediaTransportHandlerSTS::CreateIceCtx(const std::string& aName) { __func__); } - mIceCtx->SignalGatheringStateChange.connect( - this, &MediaTransportHandlerSTS::OnGatheringStateChange); mIceCtx->SignalConnectionStateChange.connect( this, &MediaTransportHandlerSTS::OnConnectionStateChange); @@ -784,6 +782,8 @@ void MediaTransportHandlerSTS::EnsureProvisionalTransport( stream->SignalCandidate.connect( this, &MediaTransportHandlerSTS::OnCandidateFound); + stream->SignalGatheringStateChange.connect( + this, &MediaTransportHandlerSTS::OnGatheringStateChange); } // Begins an ICE restart if this stream has a different ufrag/pwd @@ -1181,31 +1181,31 @@ void MediaTransportHandler::OnAlpnNegotiated(const std::string& aAlpn) { } void MediaTransportHandler::OnGatheringStateChange( - dom::RTCIceGatheringState aState) { + const std::string& aTransportId, dom::RTCIceGathererState aState) { if (mCallbackThread && !mCallbackThread->IsOnCurrentThread()) { mCallbackThread->Dispatch( // This is being called from sigslot, which does not hold a strong ref. WrapRunnable(this, &MediaTransportHandler::OnGatheringStateChange, - aState), + aTransportId, aState), NS_DISPATCH_NORMAL); return; } - SignalGatheringStateChange(aState); + SignalGatheringStateChange(aTransportId, aState); } void MediaTransportHandler::OnConnectionStateChange( - dom::RTCIceConnectionState aState) { + const std::string& aTransportId, dom::RTCIceTransportState aState) { if (mCallbackThread && !mCallbackThread->IsOnCurrentThread()) { mCallbackThread->Dispatch( // This is being called from sigslot, which does not hold a strong ref. WrapRunnable(this, &MediaTransportHandler::OnConnectionStateChange, - aState), + aTransportId, aState), NS_DISPATCH_NORMAL); return; } - SignalConnectionStateChange(aState); + SignalConnectionStateChange(aTransportId, aState); } void MediaTransportHandler::OnPacketReceived(const std::string& aTransportId, @@ -1583,48 +1583,48 @@ RefPtr<TransportFlow> MediaTransportHandlerSTS::CreateTransportFlow( return flow; } -static mozilla::dom::RTCIceGatheringState toDomIceGatheringState( - NrIceCtx::GatheringState aState) { +static mozilla::dom::RTCIceGathererState toDomIceGathererState( + NrIceMediaStream::GatheringState aState) { switch (aState) { - case NrIceCtx::ICE_CTX_GATHER_INIT: - return dom::RTCIceGatheringState::New; - case NrIceCtx::ICE_CTX_GATHER_STARTED: - return dom::RTCIceGatheringState::Gathering; - case NrIceCtx::ICE_CTX_GATHER_COMPLETE: - return dom::RTCIceGatheringState::Complete; + case NrIceMediaStream::ICE_STREAM_GATHER_INIT: + return dom::RTCIceGathererState::New; + case NrIceMediaStream::ICE_STREAM_GATHER_STARTED: + return dom::RTCIceGathererState::Gathering; + case NrIceMediaStream::ICE_STREAM_GATHER_COMPLETE: + return dom::RTCIceGathererState::Complete; } MOZ_CRASH(); } void MediaTransportHandlerSTS::OnGatheringStateChange( - NrIceCtx* aIceCtx, NrIceCtx::GatheringState aState) { - OnGatheringStateChange(toDomIceGatheringState(aState)); + const std::string& aTransportId, NrIceMediaStream::GatheringState aState) { + OnGatheringStateChange(aTransportId, toDomIceGathererState(aState)); } -static mozilla::dom::RTCIceConnectionState toDomIceConnectionState( +static mozilla::dom::RTCIceTransportState toDomIceTransportState( NrIceCtx::ConnectionState aState) { switch (aState) { case NrIceCtx::ICE_CTX_INIT: - return dom::RTCIceConnectionState::New; + return dom::RTCIceTransportState::New; case NrIceCtx::ICE_CTX_CHECKING: - return dom::RTCIceConnectionState::Checking; + return dom::RTCIceTransportState::Checking; case NrIceCtx::ICE_CTX_CONNECTED: - return dom::RTCIceConnectionState::Connected; + return dom::RTCIceTransportState::Connected; case NrIceCtx::ICE_CTX_COMPLETED: - return dom::RTCIceConnectionState::Completed; + return dom::RTCIceTransportState::Completed; case NrIceCtx::ICE_CTX_FAILED: - return dom::RTCIceConnectionState::Failed; + return dom::RTCIceTransportState::Failed; case NrIceCtx::ICE_CTX_DISCONNECTED: - return dom::RTCIceConnectionState::Disconnected; + return dom::RTCIceTransportState::Disconnected; case NrIceCtx::ICE_CTX_CLOSED: - return dom::RTCIceConnectionState::Closed; + return dom::RTCIceTransportState::Closed; } MOZ_CRASH(); } void MediaTransportHandlerSTS::OnConnectionStateChange( - NrIceCtx* aIceCtx, NrIceCtx::ConnectionState aState) { - OnConnectionStateChange(toDomIceConnectionState(aState)); + NrIceMediaStream* aIceStream, NrIceCtx::ConnectionState aState) { + OnConnectionStateChange(aIceStream->GetId(), toDomIceTransportState(aState)); } // The stuff below here will eventually go into the MediaTransportChild class diff --git a/dom/media/webrtc/jsapi/MediaTransportHandler.h b/dom/media/webrtc/jsapi/MediaTransportHandler.h index a776cb6fd7..100eff019e 100644 --- a/dom/media/webrtc/jsapi/MediaTransportHandler.h +++ b/dom/media/webrtc/jsapi/MediaTransportHandler.h @@ -12,7 +12,8 @@ #include "transport/dtlsidentity.h" // For DtlsDigest #include "mozilla/dom/RTCPeerConnectionBinding.h" #include "mozilla/dom/RTCConfigurationBinding.h" -#include "transport/nricectx.h" // Need some enums +#include "mozilla/dom/RTCIceTransportBinding.h" // RTCIceTransportState +#include "transport/nricectx.h" // Need some enums #include "common/CandidateInfo.h" #include "transport/nr_socket_proxy_config.h" #include "RTCStatsReport.h" @@ -122,8 +123,10 @@ class MediaTransportHandler { sigslot::signal2<const std::string&, const CandidateInfo&> SignalCandidate; sigslot::signal2<const std::string&, bool> SignalAlpnNegotiated; - sigslot::signal1<dom::RTCIceGatheringState> SignalGatheringStateChange; - sigslot::signal1<dom::RTCIceConnectionState> SignalConnectionStateChange; + sigslot::signal2<const std::string&, dom::RTCIceGathererState> + SignalGatheringStateChange; + sigslot::signal2<const std::string&, dom::RTCIceTransportState> + SignalConnectionStateChange; sigslot::signal2<const std::string&, const MediaPacket&> SignalPacketReceived; sigslot::signal2<const std::string&, const MediaPacket&> @@ -142,8 +145,10 @@ class MediaTransportHandler { void OnCandidate(const std::string& aTransportId, const CandidateInfo& aCandidateInfo); void OnAlpnNegotiated(const std::string& aAlpn); - void OnGatheringStateChange(dom::RTCIceGatheringState aState); - void OnConnectionStateChange(dom::RTCIceConnectionState aState); + void OnGatheringStateChange(const std::string& aTransportId, + dom::RTCIceGathererState aState); + void OnConnectionStateChange(const std::string& aTransportId, + dom::RTCIceTransportState aState); void OnPacketReceived(const std::string& aTransportId, const MediaPacket& aPacket); void OnEncryptedSending(const std::string& aTransportId, diff --git a/dom/media/webrtc/jsapi/MediaTransportHandlerIPC.cpp b/dom/media/webrtc/jsapi/MediaTransportHandlerIPC.cpp index 847e9fd3de..4d13ae0ac0 100644 --- a/dom/media/webrtc/jsapi/MediaTransportHandlerIPC.cpp +++ b/dom/media/webrtc/jsapi/MediaTransportHandlerIPC.cpp @@ -406,21 +406,21 @@ mozilla::ipc::IPCResult MediaTransportChild::RecvOnAlpnNegotiated( } mozilla::ipc::IPCResult MediaTransportChild::RecvOnGatheringStateChange( - const int& state) { + const string& transportId, const int& state) { MutexAutoLock lock(mMutex); if (mUser) { - mUser->OnGatheringStateChange( - static_cast<dom::RTCIceGatheringState>(state)); + mUser->OnGatheringStateChange(transportId, + static_cast<dom::RTCIceGathererState>(state)); } return ipc::IPCResult::Ok(); } mozilla::ipc::IPCResult MediaTransportChild::RecvOnConnectionStateChange( - const int& state) { + const string& transportId, const int& state) { MutexAutoLock lock(mMutex); if (mUser) { mUser->OnConnectionStateChange( - static_cast<dom::RTCIceConnectionState>(state)); + transportId, static_cast<dom::RTCIceTransportState>(state)); } return ipc::IPCResult::Ok(); } diff --git a/dom/media/webrtc/jsapi/MediaTransportParent.cpp b/dom/media/webrtc/jsapi/MediaTransportParent.cpp index 31d0b9f30f..4bbd75b41c 100644 --- a/dom/media/webrtc/jsapi/MediaTransportParent.cpp +++ b/dom/media/webrtc/jsapi/MediaTransportParent.cpp @@ -49,14 +49,16 @@ class MediaTransportParent::Impl : public sigslot::has_slots<> { NS_ENSURE_TRUE_VOID(mParent->SendOnAlpnNegotiated(aAlpn)); } - void OnGatheringStateChange(dom::RTCIceGatheringState aState) { - NS_ENSURE_TRUE_VOID( - mParent->SendOnGatheringStateChange(static_cast<int>(aState))); + void OnGatheringStateChange(const std::string& aTransportId, + dom::RTCIceGathererState aState) { + NS_ENSURE_TRUE_VOID(mParent->SendOnGatheringStateChange( + aTransportId, static_cast<int>(aState))); } - void OnConnectionStateChange(dom::RTCIceConnectionState aState) { - NS_ENSURE_TRUE_VOID( - mParent->SendOnConnectionStateChange(static_cast<int>(aState))); + void OnConnectionStateChange(const std::string& aTransportId, + dom::RTCIceTransportState aState) { + NS_ENSURE_TRUE_VOID(mParent->SendOnConnectionStateChange( + aTransportId, static_cast<int>(aState))); } void OnPacketReceived(const std::string& aTransportId, diff --git a/dom/media/webrtc/jsapi/PeerConnectionImpl.cpp b/dom/media/webrtc/jsapi/PeerConnectionImpl.cpp index 985100153a..9afa7e5dd2 100644 --- a/dom/media/webrtc/jsapi/PeerConnectionImpl.cpp +++ b/dom/media/webrtc/jsapi/PeerConnectionImpl.cpp @@ -85,6 +85,7 @@ #include "mozilla/dom/RTCCertificate.h" #include "mozilla/dom/RTCSctpTransportBinding.h" // RTCSctpTransportState #include "mozilla/dom/RTCDtlsTransportBinding.h" // RTCDtlsTransportState +#include "mozilla/dom/RTCIceTransportBinding.h" // RTCIceTransportState #include "mozilla/dom/RTCRtpReceiverBinding.h" #include "mozilla/dom/RTCRtpSenderBinding.h" #include "mozilla/dom/RTCStatsReportBinding.h" @@ -249,19 +250,41 @@ void PeerConnectionAutoTimer::UnregisterConnection(bool aContainedAV) { bool PeerConnectionAutoTimer::IsStopped() { return mRefCnt == 0; } +// There is not presently an implementation of these for nsTHashMap :( +inline void ImplCycleCollectionUnlink( + PeerConnectionImpl::RTCDtlsTransportMap& aMap) { + for (auto& tableEntry : aMap) { + ImplCycleCollectionUnlink(*tableEntry.GetModifiableData()); + } + aMap.Clear(); +} + +inline void ImplCycleCollectionTraverse( + nsCycleCollectionTraversalCallback& aCallback, + PeerConnectionImpl::RTCDtlsTransportMap& aMap, const char* aName, + uint32_t aFlags = 0) { + for (auto& tableEntry : aMap) { + ImplCycleCollectionTraverse(aCallback, *tableEntry.GetModifiableData(), + aName, aFlags); + } +} + NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(PeerConnectionImpl) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(PeerConnectionImpl) tmp->Close(); tmp->BreakCycles(); - NS_IMPL_CYCLE_COLLECTION_UNLINK(mPCObserver, mWindow, mCertificate, - mSTSThread, mReceiveStreams, mOperations, - mSctpTransport, mKungFuDeathGrip) + NS_IMPL_CYCLE_COLLECTION_UNLINK( + mPCObserver, mWindow, mCertificate, mSTSThread, mReceiveStreams, + mOperations, mTransportIdToRTCDtlsTransport, mSctpTransport, + mLastStableSctpTransport, mLastStableSctpDtlsTransport, mKungFuDeathGrip) NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(PeerConnectionImpl) NS_IMPL_CYCLE_COLLECTION_TRAVERSE( mPCObserver, mWindow, mCertificate, mSTSThread, mReceiveStreams, - mOperations, mTransceivers, mSctpTransport, mKungFuDeathGrip) + mOperations, mTransceivers, mTransportIdToRTCDtlsTransport, + mSctpTransport, mLastStableSctpTransport, mLastStableSctpDtlsTransport, + mKungFuDeathGrip) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTING_ADDREF(PeerConnectionImpl) @@ -849,6 +872,7 @@ nsresult PeerConnectionImpl::GetDatachannelParameters( if (!datachannelTransceiver || !datachannelTransceiver->mTransport.mComponents || + !datachannelTransceiver->mTransport.mDtls || !datachannelTransceiver->mSendTrack.GetNegotiatedDetails()) { return NS_ERROR_FAILURE; } @@ -1061,7 +1085,7 @@ bool PeerConnectionImpl::CreatedSender(const dom::RTCRtpSender& aSender) const { return aSender.IsMyPc(this); } -nsresult PeerConnectionImpl::InitializeDataChannel() { +nsresult PeerConnectionImpl::MaybeInitializeDataChannel() { PC_AUTO_ENTER_API_CALL(false); CSFLogDebug(LOGTAG, "%s", __FUNCTION__); @@ -1440,6 +1464,16 @@ void PeerConnectionImpl::UpdateNegotiationNeeded() { })); } +RefPtr<dom::RTCRtpTransceiver> PeerConnectionImpl::GetTransceiver( + const std::string& aTransceiverId) { + for (const auto& transceiver : mTransceivers) { + if (transceiver->GetJsepTransceiverId() == aTransceiverId) { + return transceiver; + } + } + return nullptr; +} + void PeerConnectionImpl::NotifyDataChannel( already_AddRefed<DataChannel> aChannel) { PC_AUTO_ENTER_API_CALL_NO_CHECK(); @@ -1637,7 +1671,6 @@ JsepSdpType ToJsepSdpType(dom::RTCSdpType aType) { return kJsepSdpAnswer; case dom::RTCSdpType::Rollback: return kJsepSdpRollback; - case dom::RTCSdpType::EndGuard_:; } MOZ_CRASH("Nonexistent dom::RTCSdpType"); @@ -1989,10 +2022,14 @@ nsresult PeerConnectionImpl::OnAlpnNegotiated(bool aPrivacyRequested) { void PeerConnectionImpl::OnDtlsStateChange(const std::string& aTransportId, TransportLayer::State aState) { - auto it = mTransportIdToRTCDtlsTransport.find(aTransportId); - if (it != mTransportIdToRTCDtlsTransport.end()) { - it->second->UpdateState(aState); + nsCString key(aTransportId.data(), aTransportId.size()); + RefPtr<RTCDtlsTransport> dtlsTransport = + mTransportIdToRTCDtlsTransport.Get(key); + if (!dtlsTransport) { + return; } + + dtlsTransport->UpdateState(aState); // Whenever the state of an RTCDtlsTransport changes or when the [[IsClosed]] // slot turns true, the user agent MUST update the connection state by // queueing a task that runs the following steps: @@ -2024,9 +2061,9 @@ RTCPeerConnectionState PeerConnectionImpl::GetNewConnectionState() const { // Would use a bitset, but that requires lots of static_cast<size_t> // Oh well. std::set<RTCDtlsTransportState> statesFound; - for (const auto& [id, dtlsTransport] : mTransportIdToRTCDtlsTransport) { - Unused << id; - statesFound.insert(dtlsTransport->State()); + std::set<RefPtr<RTCDtlsTransport>> transports(GetActiveTransports()); + for (const auto& transport : transports) { + statesFound.insert(transport->State()); } // failed The previous state doesn't apply, and either @@ -2075,9 +2112,9 @@ RTCPeerConnectionState PeerConnectionImpl::GetNewConnectionState() const { bool PeerConnectionImpl::UpdateConnectionState() { auto newState = GetNewConnectionState(); if (newState != mConnectionState) { - CSFLogDebug(LOGTAG, "%s: %d -> %d (%p)", __FUNCTION__, - static_cast<int>(mConnectionState), static_cast<int>(newState), - this); + CSFLogInfo(LOGTAG, "%s: %d -> %d (%p)", __FUNCTION__, + static_cast<int>(mConnectionState), static_cast<int>(newState), + this); mConnectionState = newState; if (mConnectionState != RTCPeerConnectionState::Closed) { return true; @@ -2569,7 +2606,7 @@ PeerConnectionImpl::Close() { transceiver->Close(); } - mTransportIdToRTCDtlsTransport.clear(); + mTransportIdToRTCDtlsTransport.Clear(); mQueuedIceCtxOperations.clear(); @@ -2964,18 +3001,25 @@ void PeerConnectionImpl::DoSetDescriptionSuccessPostProcessing( InvalidateLastReturnedParameters(); } + if (aSdpType == dom::RTCSdpType::Offer && + mSignalingState == RTCSignalingState::Stable) { + // If description is of type "offer" and + // connection.[[SignalingState]] is "stable" then for each + // transceiver in connection's set of transceivers, run the following + // steps: + SaveStateForRollback(); + } + // Section 4.4.1.5 Set the RTCSessionDescription: if (aSdpType == dom::RTCSdpType::Rollback) { // - step 4.5.10, type is rollback - RollbackRTCDtlsTransports(); + RestoreStateForRollback(); } else if (!(aRemote && aSdpType == dom::RTCSdpType::Offer)) { // - step 4.5.9 type is not rollback // - step 4.5.9.1 when remote is false // - step 4.5.9.2.13 when remote is true, type answer or pranswer // More simply: not rollback, and not for remote offers. - bool markAsStable = aSdpType == dom::RTCSdpType::Offer && - mSignalingState == RTCSignalingState::Stable; - UpdateRTCDtlsTransports(markAsStable); + UpdateRTCDtlsTransports(); } // Did we just apply a local description? @@ -2992,11 +3036,6 @@ void PeerConnectionImpl::DoSetDescriptionSuccessPostProcessing( } if (mJsepSession->GetState() == kJsepStateStable) { - if (aSdpType != dom::RTCSdpType::Rollback) { - // We need this initted for UpdateTransports - InitializeDataChannel(); - } - // If we're rolling back a local offer, we might need to remove some // transports, and stomp some MediaPipeline setup, but nothing further // needs to be done. @@ -3071,6 +3110,10 @@ void PeerConnectionImpl::DoSetDescriptionSuccessPostProcessing( // Spec does not actually tell us to do this, but that is probably a // spec bug. // https://github.com/w3c/webrtc-pc/issues/2817 + bool gatheringStateChanged = UpdateIceGatheringState(); + + bool iceConnectionStateChanged = UpdateIceConnectionState(); + bool connectionStateChanged = UpdateConnectionState(); // This only gets populated for remote descriptions @@ -3104,6 +3147,16 @@ void PeerConnectionImpl::DoSetDescriptionSuccessPostProcessing( pcObserver->OnStateChange(PCObserverStateType::SignalingState, jrv); } + if (gatheringStateChanged) { + pcObserver->OnStateChange(PCObserverStateType::IceGatheringState, + jrv); + } + + if (iceConnectionStateChanged) { + pcObserver->OnStateChange(PCObserverStateType::IceConnectionState, + jrv); + } + if (connectionStateChanged) { pcObserver->OnStateChange(PCObserverStateType::ConnectionState, jrv); } @@ -3290,61 +3343,162 @@ void PeerConnectionImpl::SendLocalIceCandidateToContent( } void PeerConnectionImpl::IceConnectionStateChange( - dom::RTCIceConnectionState domState) { + const std::string& aTransportId, dom::RTCIceTransportState domState) { + // If connection.[[IsClosed]] is true, abort these steps. PC_AUTO_ENTER_API_CALL_VOID_RETURN(false); - CSFLogDebug(LOGTAG, "%s: %d -> %d", __FUNCTION__, - static_cast<int>(mIceConnectionState), - static_cast<int>(domState)); + CSFLogDebug(LOGTAG, "IceConnectionStateChange: %s %d (%p)", + aTransportId.c_str(), static_cast<int>(domState), this); - if (domState == mIceConnectionState) { - // no work to be done since the states are the same. - // this can happen during ICE rollback situations. + // Let transport be the RTCIceTransport whose state is changing. + nsCString key(aTransportId.data(), aTransportId.size()); + RefPtr<RTCDtlsTransport> dtlsTransport = + mTransportIdToRTCDtlsTransport.Get(key); + if (!dtlsTransport) { return; } + RefPtr<RTCIceTransport> transport = dtlsTransport->IceTransport(); - mIceConnectionState = domState; + if (domState == RTCIceTransportState::Closed) { + mTransportIdToRTCDtlsTransport.Remove(key); + } - // Would be nice if we had a means of converting one of these dom enums - // to a string that wasn't almost as much text as this switch statement... - switch (mIceConnectionState) { - case RTCIceConnectionState::New: - STAMP_TIMECARD(mTimeCard, "Ice state: new"); - break; - case RTCIceConnectionState::Checking: - // For telemetry - mIceStartTime = TimeStamp::Now(); - STAMP_TIMECARD(mTimeCard, "Ice state: checking"); - break; - case RTCIceConnectionState::Connected: - STAMP_TIMECARD(mTimeCard, "Ice state: connected"); - StartCallTelem(); - break; - case RTCIceConnectionState::Completed: - STAMP_TIMECARD(mTimeCard, "Ice state: completed"); - break; - case RTCIceConnectionState::Failed: - STAMP_TIMECARD(mTimeCard, "Ice state: failed"); - break; - case RTCIceConnectionState::Disconnected: - STAMP_TIMECARD(mTimeCard, "Ice state: disconnected"); - break; - case RTCIceConnectionState::Closed: - STAMP_TIMECARD(mTimeCard, "Ice state: closed"); - break; - default: - MOZ_ASSERT_UNREACHABLE("Unexpected mIceConnectionState!"); + // Let selectedCandidatePairChanged be false. + // TODO(bug 1307994) + + // Let transportIceConnectionStateChanged be false. + bool transportIceConnectionStateChanged = false; + + // Let connectionIceConnectionStateChanged be false. + bool connectionIceConnectionStateChanged = false; + + // Let connectionStateChanged be false. + bool connectionStateChanged = false; + + if (transport->State() == domState) { + return; + } + + // If transport's RTCIceTransportState was changed, run the following steps: + + // Set transport.[[IceTransportState]] to the new indicated + // RTCIceTransportState. + transport->SetState(domState); + + // Set transportIceConnectionStateChanged to true. + transportIceConnectionStateChanged = true; + + // Set connection.[[IceConnectionState]] to the value of deriving a new state + // value as described by the RTCIceConnectionState enum. + if (UpdateIceConnectionState()) { + // If connection.[[IceConnectionState]] changed in the previous step, set + // connectionIceConnectionStateChanged to true. + connectionIceConnectionStateChanged = true; + } + + // Set connection.[[ConnectionState]] to the value of deriving a new state + // value as described by the RTCPeerConnectionState enum. + if (UpdateConnectionState()) { + // If connection.[[ConnectionState]] changed in the previous step, set + // connectionStateChanged to true. + connectionStateChanged = true; + } + + // If selectedCandidatePairChanged is true, fire an event named + // selectedcandidatepairchange at transport. + // TODO(bug 1307994) + + // If transportIceConnectionStateChanged is true, fire an event named + // statechange at transport. + if (transportIceConnectionStateChanged) { + transport->FireStateChangeEvent(); } - bool connectionStateChanged = UpdateConnectionState(); WrappableJSErrorResult rv; RefPtr<PeerConnectionObserver> pcObserver(mPCObserver); - pcObserver->OnStateChange(PCObserverStateType::IceConnectionState, rv); + + // If connectionIceConnectionStateChanged is true, fire an event named + // iceconnectionstatechange at connection. + if (connectionIceConnectionStateChanged) { + pcObserver->OnStateChange(PCObserverStateType::IceConnectionState, rv); + } + + // If connectionStateChanged is true, fire an event named + // connectionstatechange at connection. if (connectionStateChanged) { pcObserver->OnStateChange(PCObserverStateType::ConnectionState, rv); } } +RTCIceConnectionState PeerConnectionImpl::GetNewIceConnectionState() const { + // closed The RTCPeerConnection object's [[IsClosed]] slot is true. + if (IsClosed()) { + return RTCIceConnectionState::Closed; + } + + // Would use a bitset, but that requires lots of static_cast<size_t> + // Oh well. + std::set<RTCIceTransportState> statesFound; + std::set<RefPtr<RTCDtlsTransport>> transports(GetActiveTransports()); + for (const auto& transport : transports) { + RefPtr<dom::RTCIceTransport> iceTransport = transport->IceTransport(); + CSFLogWarn(LOGTAG, "GetNewIceConnectionState: %p %d", iceTransport.get(), + static_cast<int>(iceTransport->State())); + statesFound.insert(iceTransport->State()); + } + + // failed None of the previous states apply and any RTCIceTransports are + // in the "failed" state. + if (statesFound.count(RTCIceTransportState::Failed)) { + return RTCIceConnectionState::Failed; + } + + // disconnected None of the previous states apply and any + // RTCIceTransports are in the "disconnected" state. + if (statesFound.count(RTCIceTransportState::Disconnected)) { + return RTCIceConnectionState::Disconnected; + } + + // new None of the previous states apply and all RTCIceTransports are + // in the "new" or "closed" state, or there are no transports. + if (!statesFound.count(RTCIceTransportState::Checking) && + !statesFound.count(RTCIceTransportState::Completed) && + !statesFound.count(RTCIceTransportState::Connected)) { + return RTCIceConnectionState::New; + } + + // checking None of the previous states apply and any RTCIceTransports are + // in the "new" or "checking" state. + if (statesFound.count(RTCIceTransportState::New) || + statesFound.count(RTCIceTransportState::Checking)) { + return RTCIceConnectionState::Checking; + } + + // completed None of the previous states apply and all RTCIceTransports are + // in the "completed" or "closed" state. + if (!statesFound.count(RTCIceTransportState::Connected)) { + return RTCIceConnectionState::Completed; + } + + // connected None of the previous states apply. + return RTCIceConnectionState::Connected; +} + +bool PeerConnectionImpl::UpdateIceConnectionState() { + auto newState = GetNewIceConnectionState(); + if (newState != mIceConnectionState) { + CSFLogInfo(LOGTAG, "%s: %d -> %d (%p)", __FUNCTION__, + static_cast<int>(mIceConnectionState), + static_cast<int>(newState), this); + mIceConnectionState = newState; + if (mIceConnectionState != RTCIceConnectionState::Closed) { + return true; + } + } + + return false; +} + void PeerConnectionImpl::OnCandidateFound(const std::string& aTransportId, const CandidateInfo& aCandidateInfo) { if (mStunAddrsRequest && !aCandidateInfo.mMDNSAddress.empty()) { @@ -3378,18 +3532,82 @@ void PeerConnectionImpl::OnCandidateFound(const std::string& aTransportId, } void PeerConnectionImpl::IceGatheringStateChange( - dom::RTCIceGatheringState state) { + const std::string& aTransportId, dom::RTCIceGathererState state) { + // If connection.[[IsClosed]] is true, abort these steps. PC_AUTO_ENTER_API_CALL_VOID_RETURN(false); - CSFLogDebug(LOGTAG, "%s %d", __FUNCTION__, static_cast<int>(state)); - if (mIceGatheringState == state) { + CSFLogWarn(LOGTAG, "IceGatheringStateChange: %s %d (%p)", + aTransportId.c_str(), static_cast<int>(state), this); + + // Let transport be the RTCIceTransport for which candidate gathering + // began/finished. + nsCString key(aTransportId.data(), aTransportId.size()); + RefPtr<RTCDtlsTransport> dtlsTransport = + mTransportIdToRTCDtlsTransport.Get(key); + if (!dtlsTransport) { + return; + } + RefPtr<RTCIceTransport> transport = dtlsTransport->IceTransport(); + + if (transport->GatheringState() == state) { return; } - mIceGatheringState = state; + // Set transport.[[IceGathererState]] to gathering. + // or + // Set transport.[[IceGathererState]] to complete. + transport->SetGatheringState(state); + + // Set connection.[[IceGatheringState]] to the value of deriving a new state + // value as described by the RTCIceGatheringState enum. + // + // Let connectionIceGatheringStateChanged be true if + // connection.[[IceGatheringState]] changed in the previous step, otherwise + // false. + bool gatheringStateChanged = UpdateIceGatheringState(); + + // Do not read or modify state beyond this point. + + // Fire an event named gatheringstatechange at transport. + transport->FireGatheringStateChangeEvent(); + + // If connectionIceGatheringStateChanged is true, fire an event named + // icegatheringstatechange at connection. + if (gatheringStateChanged) { + // NOTE: If we're in the "complete" case, our JS code will fire a null + // icecandidate event after firing the icegatheringstatechange event. + // Fire an event named icecandidate using the RTCPeerConnectionIceEvent + // interface with the candidate attribute set to null at connection. + JSErrorResult rv; + mPCObserver->OnStateChange(PCObserverStateType::IceGatheringState, rv); + } +} + +bool PeerConnectionImpl::UpdateIceGatheringState() { + // If connection.[[IsClosed]] is true, abort these steps. + if (IsClosed()) { + return false; + } + + // Let newState be the value of deriving a new state value as + // described by the RTCIceGatheringState enum. + auto newState = GetNewIceGatheringState(); + + // If connection.[[IceGatheringState]] is equal to newState, abort + // these steps. + if (newState == mIceGatheringState) { + return false; + } + + CSFLogInfo(LOGTAG, "UpdateIceGatheringState: %d -> %d (%p)", + static_cast<int>(mIceGatheringState), static_cast<int>(newState), + this); + // Set connection.[[IceGatheringState]] to newState. + mIceGatheringState = newState; - // Would be nice if we had a means of converting one of these dom enums - // to a string that wasn't almost as much text as this switch statement... + // Would be nice if we had a means of converting one of these dom + // enums to a string that wasn't almost as much text as this switch + // statement... switch (mIceGatheringState) { case RTCIceGatheringState::New: STAMP_TIMECARD(mTimeCard, "Ice gathering state: new"); @@ -3404,8 +3622,42 @@ void PeerConnectionImpl::IceGatheringStateChange( MOZ_ASSERT_UNREACHABLE("Unexpected mIceGatheringState!"); } - JSErrorResult rv; - mPCObserver->OnStateChange(PCObserverStateType::IceGatheringState, rv); + return true; +} + +RTCIceGatheringState PeerConnectionImpl::GetNewIceGatheringState() const { + // new Any of the RTCIceTransports are in the "new" gathering state + // and none of the transports are in the "gathering" state, or there are no + // transports. + + // NOTE! This derives the RTCIce**Gathering**State from the individual + // RTCIce**Gatherer**State of the transports. These are different enums. + // But they have exactly the same values, in the same order. + // ¯\_(ツ)_/¯ + bool foundComplete = false; + std::set<RefPtr<RTCDtlsTransport>> transports(GetActiveTransports()); + for (const auto& transport : transports) { + RefPtr<dom::RTCIceTransport> iceTransport = transport->IceTransport(); + switch (iceTransport->GatheringState()) { + case RTCIceGathererState::New: + break; + case RTCIceGathererState::Gathering: + // gathering Any of the RTCIceTransports are in the "gathering" + // state. + return RTCIceGatheringState::Gathering; + case RTCIceGathererState::Complete: + foundComplete = true; + break; + } + } + + if (!foundComplete) { + return RTCIceGatheringState::New; + } + + // This could change depending on the outcome in + // https://github.com/w3c/webrtc-pc/issues/2914 + return RTCIceGatheringState::Complete; } void PeerConnectionImpl::UpdateDefaultCandidate( @@ -3878,11 +4130,8 @@ void PeerConnectionImpl::StunAddrsHandler::OnStunAddrsAvailable( pcw.impl()->mStunAddrs = addrs.Clone(); pcw.impl()->mLocalAddrsRequestState = STUN_ADDR_REQUEST_COMPLETE; pcw.impl()->FlushIceCtxOperationQueueIfReady(); - // If parent process returns 0 STUN addresses, change ICE connection - // state to failed. - if (!pcw.impl()->mStunAddrs.Length()) { - pcw.impl()->IceConnectionStateChange(dom::RTCIceConnectionState::Failed); - } + // If this fails, ICE cannot succeed, but we need to still go through the + // motions. } void PeerConnectionImpl::InitLocalAddrs() { @@ -3952,109 +4201,120 @@ void PeerConnectionImpl::EnsureTransports(const JsepSession& aSession) { GatherIfReady(); } -void PeerConnectionImpl::UpdateRTCDtlsTransports(bool aMarkAsStable) { +void PeerConnectionImpl::UpdateRTCDtlsTransports() { + // We use mDataConnection below, make sure it is initted if necessary + MaybeInitializeDataChannel(); + + // Make sure that the SCTP transport is unset if we do not see a DataChannel. + // We'll restore this if we do see a DataChannel. + RefPtr<dom::RTCSctpTransport> oldSctp = mSctpTransport.forget(); + mJsepSession->ForEachTransceiver( - [this, self = RefPtr<PeerConnectionImpl>(this)]( - const JsepTransceiver& jsepTransceiver) { + [this, self = RefPtr<PeerConnectionImpl>(this), + oldSctp](const JsepTransceiver& jsepTransceiver) { std::string transportId = jsepTransceiver.mTransport.mTransportId; - if (transportId.empty()) { - return; + RefPtr<dom::RTCDtlsTransport> dtlsTransport; + if (!transportId.empty()) { + nsCString key(transportId.data(), transportId.size()); + dtlsTransport = mTransportIdToRTCDtlsTransport.GetOrInsertNew( + key, GetParentObject()); } - if (!mTransportIdToRTCDtlsTransport.count(transportId)) { - mTransportIdToRTCDtlsTransport.emplace( - transportId, new RTCDtlsTransport(GetParentObject())); + + if (jsepTransceiver.GetMediaType() == SdpMediaSection::kApplication) { + // Spec says we only update the RTCSctpTransport when negotiation + // completes. This is probably a spec bug. + // https://github.com/w3c/webrtc-pc/issues/2898 + if (!dtlsTransport || !mDataConnection) { + return; + } + + // Why on earth does the spec use a floating point for this? + double maxMessageSize = + static_cast<double>(mDataConnection->GetMaxMessageSize()); + Nullable<uint16_t> maxChannels; + + if (!oldSctp) { + mSctpTransport = new RTCSctpTransport( + GetParentObject(), *dtlsTransport, maxMessageSize, maxChannels); + } else { + // Restore the SCTP transport we had before this function was called + oldSctp->SetTransport(*dtlsTransport); + oldSctp->SetMaxMessageSize(maxMessageSize); + oldSctp->SetMaxChannels(maxChannels); + mSctpTransport = oldSctp; + } + } else { + RefPtr<dom::RTCRtpTransceiver> domTransceiver = + GetTransceiver(jsepTransceiver.GetUuid()); + if (domTransceiver) { + domTransceiver->SetDtlsTransport(dtlsTransport); + } } }); +} - for (auto& transceiver : mTransceivers) { - std::string transportId = transceiver->GetTransportId(); - if (transportId.empty()) { - continue; - } - if (mTransportIdToRTCDtlsTransport.count(transportId)) { - transceiver->SetDtlsTransport(mTransportIdToRTCDtlsTransport[transportId], - aMarkAsStable); - } +void PeerConnectionImpl::SaveStateForRollback() { + // This could change depending on the outcome in + // https://github.com/w3c/webrtc-pc/issues/2899 + if (mSctpTransport) { + // We have to save both of these things, because the DTLS transport could + // change without the SCTP transport changing. + mLastStableSctpTransport = mSctpTransport; + mLastStableSctpDtlsTransport = mSctpTransport->Transport(); + } else { + mLastStableSctpTransport = nullptr; + mLastStableSctpDtlsTransport = nullptr; } - // Spec says we only update the RTCSctpTransport when negotiation completes + for (auto& transceiver : mTransceivers) { + transceiver->SaveStateForRollback(); + } } -void PeerConnectionImpl::RollbackRTCDtlsTransports() { +void PeerConnectionImpl::RestoreStateForRollback() { for (auto& transceiver : mTransceivers) { transceiver->RollbackToStableDtlsTransport(); } + + mSctpTransport = mLastStableSctpTransport; + if (mSctpTransport) { + mSctpTransport->SetTransport(*mLastStableSctpDtlsTransport); + } } -void PeerConnectionImpl::RemoveRTCDtlsTransportsExcept( - const std::set<std::string>& aTransportIds) { - for (auto iter = mTransportIdToRTCDtlsTransport.begin(); - iter != mTransportIdToRTCDtlsTransport.end();) { - if (!aTransportIds.count(iter->first)) { - iter = mTransportIdToRTCDtlsTransport.erase(iter); - } else { - ++iter; +std::set<RefPtr<dom::RTCDtlsTransport>> +PeerConnectionImpl::GetActiveTransports() const { + std::set<RefPtr<dom::RTCDtlsTransport>> result; + for (const auto& transceiver : mTransceivers) { + if (transceiver->GetDtlsTransport()) { + result.insert(transceiver->GetDtlsTransport()); } } + + if (mSctpTransport && mSctpTransport->Transport()) { + result.insert(mSctpTransport->Transport()); + } + return result; } nsresult PeerConnectionImpl::UpdateTransports(const JsepSession& aSession, const bool forceIceTcp) { std::set<std::string> finalTransports; - Maybe<std::string> sctpTransport; mJsepSession->ForEachTransceiver( [&, this, self = RefPtr<PeerConnectionImpl>(this)]( const JsepTransceiver& transceiver) { - if (transceiver.GetMediaType() == SdpMediaSection::kApplication && - transceiver.HasTransport()) { - sctpTransport = Some(transceiver.mTransport.mTransportId); - } - if (transceiver.HasOwnTransport()) { finalTransports.insert(transceiver.mTransport.mTransportId); UpdateTransport(transceiver, forceIceTcp); } }); - // clean up the unused RTCDtlsTransports - RemoveRTCDtlsTransportsExcept(finalTransports); - mTransportHandler->RemoveTransportsExcept(finalTransports); for (const auto& transceiverImpl : mTransceivers) { transceiverImpl->UpdateTransport(); } - if (sctpTransport.isSome()) { - auto it = mTransportIdToRTCDtlsTransport.find(*sctpTransport); - if (it == mTransportIdToRTCDtlsTransport.end()) { - // What? - MOZ_ASSERT(false); - return NS_ERROR_FAILURE; - } - if (!mDataConnection) { - // What? - MOZ_ASSERT(false); - return NS_ERROR_FAILURE; - } - RefPtr<RTCDtlsTransport> dtlsTransport = it->second; - // Why on earth does the spec use a floating point for this? - double maxMessageSize = - static_cast<double>(mDataConnection->GetMaxMessageSize()); - Nullable<uint16_t> maxChannels; - - if (!mSctpTransport) { - mSctpTransport = new RTCSctpTransport(GetParentObject(), *dtlsTransport, - maxMessageSize, maxChannels); - } else { - mSctpTransport->SetTransport(*dtlsTransport); - mSctpTransport->SetMaxMessageSize(maxMessageSize); - mSctpTransport->SetMaxChannels(maxChannels); - } - } else { - mSctpTransport = nullptr; - } - return NS_OK; } @@ -4141,6 +4401,9 @@ nsresult PeerConnectionImpl::UpdateMediaPipelines() { void PeerConnectionImpl::StartIceChecks(const JsepSession& aSession) { MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mJsepSession->GetState() == kJsepStateStable); + + auto transports = GetActiveTransports(); if (!mCanRegisterMDNSHostnamesDirectly) { for (auto& pair : mMDNSHostnamesToRegister) { @@ -4472,32 +4735,31 @@ std::string PeerConnectionImpl::GetTransportIdMatchingSendTrack( } void PeerConnectionImpl::SignalHandler::IceGatheringStateChange_s( - dom::RTCIceGatheringState aState) { + const std::string& aTransportId, dom::RTCIceGathererState aState) { ASSERT_ON_THREAD(mSTSThread); - GetMainThreadSerialEventTarget()->Dispatch( NS_NewRunnableFunction(__func__, - [handle = mHandle, aState] { + [handle = mHandle, aTransportId, aState] { PeerConnectionWrapper wrapper(handle); if (wrapper.impl()) { wrapper.impl()->IceGatheringStateChange( - aState); + aTransportId, aState); } }), NS_DISPATCH_NORMAL); } void PeerConnectionImpl::SignalHandler::IceConnectionStateChange_s( - dom::RTCIceConnectionState aState) { + const std::string& aTransportId, dom::RTCIceTransportState aState) { ASSERT_ON_THREAD(mSTSThread); GetMainThreadSerialEventTarget()->Dispatch( NS_NewRunnableFunction(__func__, - [handle = mHandle, aState] { + [handle = mHandle, aTransportId, aState] { PeerConnectionWrapper wrapper(handle); if (wrapper.impl()) { wrapper.impl()->IceConnectionStateChange( - aState); + aTransportId, aState); } }), NS_DISPATCH_NORMAL); diff --git a/dom/media/webrtc/jsapi/PeerConnectionImpl.h b/dom/media/webrtc/jsapi/PeerConnectionImpl.h index 085658b206..d7b54ad721 100644 --- a/dom/media/webrtc/jsapi/PeerConnectionImpl.h +++ b/dom/media/webrtc/jsapi/PeerConnectionImpl.h @@ -16,6 +16,7 @@ #include "nsPIDOMWindow.h" #include "nsIUUIDGenerator.h" #include "nsIThread.h" +#include "nsTHashSet.h" #include "mozilla/Mutex.h" #include "mozilla/Attributes.h" @@ -217,8 +218,10 @@ class PeerConnectionImpl final virtual const std::string& GetName(); // ICE events - void IceConnectionStateChange(dom::RTCIceConnectionState state); - void IceGatheringStateChange(dom::RTCIceGatheringState state); + void IceConnectionStateChange(const std::string& aTransportId, + dom::RTCIceTransportState state); + void IceGatheringStateChange(const std::string& aTransportId, + dom::RTCIceGathererState state); void OnCandidateFound(const std::string& aTransportId, const CandidateInfo& aCandidateInfo); void UpdateDefaultCandidate(const std::string& defaultAddr, @@ -411,7 +414,7 @@ class PeerConnectionImpl final void RecordEndOfCallTelemetry(); - nsresult InitializeDataChannel(); + nsresult MaybeInitializeDataChannel(); NS_IMETHODIMP_TO_ERRORRESULT_RETREF(nsDOMDataChannel, CreateDataChannel, ErrorResult& rv, const nsAString& aLabel, @@ -481,6 +484,9 @@ class PeerConnectionImpl final aTransceiversOut = mTransceivers.Clone(); } + RefPtr<dom::RTCRtpTransceiver> GetTransceiver( + const std::string& aTransceiverId); + // Gets the RTC Signaling State of the JSEP session dom::RTCSignalingState GetSignalingState() const; @@ -499,6 +505,12 @@ class PeerConnectionImpl final dom::RTCPeerConnectionState GetNewConnectionState() const; // Returns whether we need to fire a state change event bool UpdateConnectionState(); + dom::RTCIceConnectionState GetNewIceConnectionState() const; + // Returns whether we need to fire a state change event + bool UpdateIceConnectionState(); + dom::RTCIceGatheringState GetNewIceGatheringState() const; + // Returns whether we need to fire a state change event + bool UpdateIceGatheringState(); // initialize telemetry for when calls start void StartCallTelem(); @@ -585,6 +597,9 @@ class PeerConnectionImpl final void BreakCycles(); + using RTCDtlsTransportMap = + nsTHashMap<nsCStringHashKey, RefPtr<dom::RTCDtlsTransport>>; + private: virtual ~PeerConnectionImpl(); PeerConnectionImpl(const PeerConnectionImpl& rhs); @@ -805,10 +820,10 @@ class PeerConnectionImpl final // Ensure ICE transports exist that we might need when offer/answer concludes void EnsureTransports(const JsepSession& aSession); - void UpdateRTCDtlsTransports(bool aMarkAsStable); - void RollbackRTCDtlsTransports(); - void RemoveRTCDtlsTransportsExcept( - const std::set<std::string>& aTransportIds); + void UpdateRTCDtlsTransports(); + void SaveStateForRollback(); + void RestoreStateForRollback(); + std::set<RefPtr<dom::RTCDtlsTransport>> GetActiveTransports() const; // Activate ICE transports at the conclusion of offer/answer, // or when rollback occurs. @@ -861,9 +876,12 @@ class PeerConnectionImpl final std::set<std::pair<std::string, std::string>> mLocalIceCredentialsToReplace; nsTArray<RefPtr<dom::RTCRtpTransceiver>> mTransceivers; - std::map<std::string, RefPtr<dom::RTCDtlsTransport>> - mTransportIdToRTCDtlsTransport; + RTCDtlsTransportMap mTransportIdToRTCDtlsTransport; RefPtr<dom::RTCSctpTransport> mSctpTransport; + // This is similar to [[LastStableStateSender/ReceiverTransport]], but for + // DataChannel. + RefPtr<dom::RTCSctpTransport> mLastStableSctpTransport; + RefPtr<dom::RTCDtlsTransport> mLastStableSctpDtlsTransport; // Used whenever we need to dispatch a runnable to STS to tweak something // on our ICE ctx, but are not ready to do so at the moment (eg; we are @@ -924,8 +942,10 @@ class PeerConnectionImpl final void ConnectSignals(); // ICE events - void IceGatheringStateChange_s(dom::RTCIceGatheringState aState); - void IceConnectionStateChange_s(dom::RTCIceConnectionState aState); + void IceGatheringStateChange_s(const std::string& aTransportId, + dom::RTCIceGathererState aState); + void IceConnectionStateChange_s(const std::string& aTransportId, + dom::RTCIceTransportState aState); void OnCandidateFound_s(const std::string& aTransportId, const CandidateInfo& aCandidateInfo); void AlpnNegotiated_s(const std::string& aAlpn, bool aPrivacyRequested); diff --git a/dom/media/webrtc/jsapi/RTCDtlsTransport.cpp b/dom/media/webrtc/jsapi/RTCDtlsTransport.cpp index 243f06d2f1..83e0aeee82 100644 --- a/dom/media/webrtc/jsapi/RTCDtlsTransport.cpp +++ b/dom/media/webrtc/jsapi/RTCDtlsTransport.cpp @@ -9,7 +9,8 @@ namespace mozilla::dom { -NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCDtlsTransport, DOMEventTargetHelper) +NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCDtlsTransport, DOMEventTargetHelper, + mIceTransport) NS_IMPL_ADDREF_INHERITED(RTCDtlsTransport, DOMEventTargetHelper) NS_IMPL_RELEASE_INHERITED(RTCDtlsTransport, DOMEventTargetHelper) @@ -19,7 +20,9 @@ NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCDtlsTransport) NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) RTCDtlsTransport::RTCDtlsTransport(nsPIDOMWindowInner* aWindow) - : DOMEventTargetHelper(aWindow), mState(RTCDtlsTransportState::New) {} + : DOMEventTargetHelper(aWindow), + mState(RTCDtlsTransportState::New), + mIceTransport(new RTCIceTransport(aWindow)) {} JSObject* RTCDtlsTransport::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) { diff --git a/dom/media/webrtc/jsapi/RTCDtlsTransport.h b/dom/media/webrtc/jsapi/RTCDtlsTransport.h index 3800502154..3e744b5112 100644 --- a/dom/media/webrtc/jsapi/RTCDtlsTransport.h +++ b/dom/media/webrtc/jsapi/RTCDtlsTransport.h @@ -6,6 +6,7 @@ #define _RTCDtlsTransport_h_ #include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/RTCIceTransport.h" #include "mozilla/RefPtr.h" #include "js/RootingAPI.h" #include "transport/transportlayer.h" @@ -30,6 +31,7 @@ class RTCDtlsTransport : public DOMEventTargetHelper { JS::Handle<JSObject*> aGivenProto) override; IMPL_EVENT_HANDLER(statechange) RTCDtlsTransportState State() const { return mState; } + RefPtr<RTCIceTransport> IceTransport() { return mIceTransport; } void UpdateStateNoEvent(TransportLayer::State aState); void UpdateState(TransportLayer::State aState); @@ -38,6 +40,7 @@ class RTCDtlsTransport : public DOMEventTargetHelper { virtual ~RTCDtlsTransport() = default; RTCDtlsTransportState mState; + RefPtr<RTCIceTransport> mIceTransport; }; } // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCIceTransport.cpp b/dom/media/webrtc/jsapi/RTCIceTransport.cpp new file mode 100644 index 0000000000..4c5e6eef4f --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCIceTransport.cpp @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RTCIceTransport.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/EventBinding.h" +#include "mozilla/dom/RTCIceTransportBinding.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCIceTransport, DOMEventTargetHelper) + +NS_IMPL_ADDREF_INHERITED(RTCIceTransport, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(RTCIceTransport, DOMEventTargetHelper) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCIceTransport) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +RTCIceTransport::RTCIceTransport(nsPIDOMWindowInner* aWindow) + : DOMEventTargetHelper(aWindow), + mState(RTCIceTransportState::New), + mGatheringState(RTCIceGathererState::New) {} + +JSObject* RTCIceTransport::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return RTCIceTransport_Binding::Wrap(aCx, this, aGivenProto); +} + +void RTCIceTransport::SetState(RTCIceTransportState aState) { mState = aState; } + +void RTCIceTransport::SetGatheringState(RTCIceGathererState aState) { + mGatheringState = aState; +} + +void RTCIceTransport::FireStateChangeEvent() { + EventInit init; + init.mBubbles = false; + init.mCancelable = false; + + RefPtr<Event> event = Event::Constructor(this, u"statechange"_ns, init); + + DispatchTrustedEvent(event); +} + +void RTCIceTransport::FireGatheringStateChangeEvent() { + EventInit init; + init.mBubbles = false; + init.mCancelable = false; + + RefPtr<Event> event = + Event::Constructor(this, u"gatheringstatechange"_ns, init); + + DispatchTrustedEvent(event); +} + +} // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCIceTransport.h b/dom/media/webrtc/jsapi/RTCIceTransport.h new file mode 100644 index 0000000000..931aa66bac --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCIceTransport.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCICETRANSPORT_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCICETRANSPORT_H_ + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/RefPtr.h" +#include "js/RootingAPI.h" +#include "transport/transportlayer.h" + +class nsPIDOMWindowInner; + +namespace mozilla::dom { + +enum class RTCIceTransportState : uint8_t; +enum class RTCIceGathererState : uint8_t; + +class RTCIceTransport : public DOMEventTargetHelper { + public: + explicit RTCIceTransport(nsPIDOMWindowInner* aWindow); + + // nsISupports + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCIceTransport, + DOMEventTargetHelper) + + // webidl + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + IMPL_EVENT_HANDLER(statechange) + IMPL_EVENT_HANDLER(gatheringstatechange) + RTCIceTransportState State() const { return mState; } + RTCIceGathererState GatheringState() const { return mGatheringState; } + + void SetState(RTCIceTransportState aState); + void SetGatheringState(RTCIceGathererState aState); + + void FireStateChangeEvent(); + void FireGatheringStateChangeEvent(); + + private: + virtual ~RTCIceTransport() = default; + + RTCIceTransportState mState; + RTCIceGathererState mGatheringState; +}; + +} // namespace mozilla::dom +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCICETRANSPORT_H_ diff --git a/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp b/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp index efe83fb782..96db9fc312 100644 --- a/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp @@ -709,9 +709,9 @@ void RTCRtpReceiver::UpdateTransport() { // Add unique payload types as a last-ditch fallback auto uniquePts = GetJsepTransceiver() .mRecvTrack.GetNegotiatedDetails() - ->GetUniquePayloadTypes(); + ->GetUniqueReceivePayloadTypes(); for (unsigned char& uniquePt : uniquePts) { - filter->AddUniquePT(uniquePt); + filter->AddUniqueReceivePT(uniquePt); } } diff --git a/dom/media/webrtc/jsapi/RTCRtpSender.cpp b/dom/media/webrtc/jsapi/RTCRtpSender.cpp index 4ba0491521..da198c62e4 100644 --- a/dom/media/webrtc/jsapi/RTCRtpSender.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpSender.cpp @@ -1502,7 +1502,6 @@ Maybe<RTCRtpSender::VideoConfig> RTCRtpSender::GetNewVideoConfig() { case dom::MediaSourceEnum::Microphone: case dom::MediaSourceEnum::AudioCapture: - case dom::MediaSourceEnum::EndGuard_: MOZ_ASSERT(false); break; } diff --git a/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp b/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp index b219619f87..6b7b456e46 100644 --- a/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp @@ -239,7 +239,6 @@ SdpDirectionAttribute::Direction ToSdpDirection( case dom::RTCRtpTransceiverDirection::Inactive: case dom::RTCRtpTransceiverDirection::Stopped: return SdpDirectionAttribute::Direction::kInactive; - case dom::RTCRtpTransceiverDirection::EndGuard_:; } MOZ_CRASH("Invalid transceiver direction!"); } @@ -279,12 +278,13 @@ void RTCRtpTransceiver::Init(const RTCRtpTransceiverInit& aInit, mDirection = aInit.mDirection; } -void RTCRtpTransceiver::SetDtlsTransport(dom::RTCDtlsTransport* aDtlsTransport, - bool aStable) { +void RTCRtpTransceiver::SetDtlsTransport( + dom::RTCDtlsTransport* aDtlsTransport) { mDtlsTransport = aDtlsTransport; - if (aStable) { - mLastStableDtlsTransport = mDtlsTransport; - } +} + +void RTCRtpTransceiver::SaveStateForRollback() { + mLastStableDtlsTransport = mDtlsTransport; } void RTCRtpTransceiver::RollbackToStableDtlsTransport() { @@ -366,12 +366,17 @@ void RTCRtpTransceiver::InitConduitControl() { } void RTCRtpTransceiver::Close() { - // Called via PCImpl::Close -> PCImpl::CloseInt -> PCImpl::ShutdownMedia -> - // PCMedia::SelfDestruct. Satisfies step 7 of + // Called via PCImpl::Close + // Satisfies steps 7 and 9 of // https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-close + // No events are fired for this. mShutdown = true; if (mDtlsTransport) { mDtlsTransport->UpdateStateNoEvent(TransportLayer::TS_CLOSED); + // Might not be set if we're cycle-collecting + if (mDtlsTransport->IceTransport()) { + mDtlsTransport->IceTransport()->SetState(RTCIceTransportState::Closed); + } } StopImpl(); } @@ -401,9 +406,9 @@ void RTCRtpTransceiver::Unlink() { // TODO: Only called from one place in PeerConnectionImpl, synchronously, when // the JSEP engine has successfully completed an offer/answer exchange. This is // a bit squirrely, since identity validation happens asynchronously in -// PeerConnection.jsm. This probably needs to happen once all the "in parallel" -// steps have succeeded, but before we queue the task for JS observable state -// updates. +// PeerConnection.sys.mjs. This probably needs to happen once all the "in +// parallel" steps have succeeded, but before we queue the task for JS +// observable state updates. nsresult RTCRtpTransceiver::UpdateTransport() { if (!mHasTransport) { return NS_OK; @@ -439,9 +444,9 @@ void RTCRtpTransceiver::ResetSync() { mSyncGroup = std::string(); } // TODO: Only called from one place in PeerConnectionImpl, synchronously, when // the JSEP engine has successfully completed an offer/answer exchange. This is // a bit squirrely, since identity validation happens asynchronously in -// PeerConnection.jsm. This probably needs to happen once all the "in parallel" -// steps have succeeded, but before we queue the task for JS observable state -// updates. +// PeerConnection.sys.mjs. This probably needs to happen once all the "in +// parallel" steps have succeeded, but before we queue the task for JS +// observable state updates. nsresult RTCRtpTransceiver::SyncWithMatchingVideoConduits( nsTArray<RefPtr<RTCRtpTransceiver>>& transceivers) { if (mStopped) { diff --git a/dom/media/webrtc/jsapi/RTCRtpTransceiver.h b/dom/media/webrtc/jsapi/RTCRtpTransceiver.h index afa6142d35..4fa7ef157c 100644 --- a/dom/media/webrtc/jsapi/RTCRtpTransceiver.h +++ b/dom/media/webrtc/jsapi/RTCRtpTransceiver.h @@ -119,7 +119,8 @@ class RTCRtpTransceiver : public nsISupports, public nsWrapperCache { void SyncFromJsep(const JsepSession& aSession); std::string GetMidAscii() const; - void SetDtlsTransport(RTCDtlsTransport* aDtlsTransport, bool aStable); + void SetDtlsTransport(RTCDtlsTransport* aDtlsTransport); + void SaveStateForRollback(); void RollbackToStableDtlsTransport(); std::string GetTransportId() const { diff --git a/dom/media/webrtc/jsapi/moz.build b/dom/media/webrtc/jsapi/moz.build index 78a6241cd6..05920fc151 100644 --- a/dom/media/webrtc/jsapi/moz.build +++ b/dom/media/webrtc/jsapi/moz.build @@ -31,6 +31,7 @@ UNIFIED_SOURCES += [ "RTCEncodedAudioFrame.cpp", "RTCEncodedFrameBase.cpp", "RTCEncodedVideoFrame.cpp", + "RTCIceTransport.cpp", "RTCRtpReceiver.cpp", "RTCRtpScriptTransform.cpp", "RTCRtpScriptTransformer.cpp", @@ -50,6 +51,7 @@ EXPORTS.mozilla.dom += [ "RTCEncodedAudioFrame.h", "RTCEncodedFrameBase.h", "RTCEncodedVideoFrame.h", + "RTCIceTransport.h", "RTCRtpReceiver.h", "RTCRtpScriptTransform.h", "RTCRtpScriptTransformer.h", diff --git a/dom/media/webrtc/jsep/JsepSessionImpl.cpp b/dom/media/webrtc/jsep/JsepSessionImpl.cpp index 50def6eb6c..c091b6d618 100644 --- a/dom/media/webrtc/jsep/JsepSessionImpl.cpp +++ b/dom/media/webrtc/jsep/JsepSessionImpl.cpp @@ -1121,11 +1121,17 @@ nsresult JsepSessionImpl::HandleNegotiatedSession( CopyBundleTransports(); - std::vector<JsepTrack*> remoteTracks; + std::vector<JsepTrack*> receiveTracks; for (auto& transceiver : mTransceivers) { - remoteTracks.push_back(&transceiver.mRecvTrack); + // Do not count payload types for non-active recv tracks as duplicates. If + // we receive an RTP packet with a payload type that is used by both a + // sendrecv and a sendonly m-section, there is no ambiguity; it is for the + // sendrecv m-section. + if (transceiver.mRecvTrack.GetActive()) { + receiveTracks.push_back(&transceiver.mRecvTrack); + } } - JsepTrack::SetUniquePayloadTypes(remoteTracks); + JsepTrack::SetUniqueReceivePayloadTypes(receiveTracks); mNegotiations++; diff --git a/dom/media/webrtc/jsep/JsepTrack.cpp b/dom/media/webrtc/jsep/JsepTrack.cpp index 59c038cfc0..2498200ab2 100644 --- a/dom/media/webrtc/jsep/JsepTrack.cpp +++ b/dom/media/webrtc/jsep/JsepTrack.cpp @@ -662,7 +662,7 @@ nsresult JsepTrack::Negotiate(const SdpMediaSection& answer, // works, however, if that payload type appeared in only one m-section. // We figure that out here. /* static */ -void JsepTrack::SetUniquePayloadTypes(std::vector<JsepTrack*>& tracks) { +void JsepTrack::SetUniqueReceivePayloadTypes(std::vector<JsepTrack*>& tracks) { // Maps to track details if no other track contains the payload type, // otherwise maps to nullptr. std::map<uint16_t, JsepTrackNegotiatedDetails*> payloadTypeToDetailsMap; @@ -697,7 +697,7 @@ void JsepTrack::SetUniquePayloadTypes(std::vector<JsepTrack*>& tracks) { auto trackDetails = ptAndDetails.second; if (trackDetails) { - trackDetails->mUniquePayloadTypes.push_back( + trackDetails->mUniqueReceivePayloadTypes.push_back( static_cast<uint8_t>(uniquePt)); } } diff --git a/dom/media/webrtc/jsep/JsepTrack.h b/dom/media/webrtc/jsep/JsepTrack.h index d11735f43a..74a0d2396c 100644 --- a/dom/media/webrtc/jsep/JsepTrack.h +++ b/dom/media/webrtc/jsep/JsepTrack.h @@ -31,7 +31,7 @@ class JsepTrackNegotiatedDetails { JsepTrackNegotiatedDetails(const JsepTrackNegotiatedDetails& orig) : mExtmap(orig.mExtmap), - mUniquePayloadTypes(orig.mUniquePayloadTypes), + mUniqueReceivePayloadTypes(orig.mUniqueReceivePayloadTypes), mTias(orig.mTias), mRtpRtcpConf(orig.mRtpRtcpConf) { for (const auto& encoding : orig.mEncodings) { @@ -71,8 +71,8 @@ class JsepTrackNegotiatedDetails { } } - std::vector<uint8_t> GetUniquePayloadTypes() const { - return mUniquePayloadTypes; + std::vector<uint8_t> GetUniqueReceivePayloadTypes() const { + return mUniqueReceivePayloadTypes; } uint32_t GetTias() const { return mTias; } @@ -83,7 +83,7 @@ class JsepTrackNegotiatedDetails { friend class JsepTrack; std::map<std::string, SdpExtmapAttributeList::Extmap> mExtmap; - std::vector<uint8_t> mUniquePayloadTypes; + std::vector<uint8_t> mUniqueReceivePayloadTypes; std::vector<UniquePtr<JsepTrackEncoding>> mEncodings; uint32_t mTias; // bits per second RtpRtcpConfig mRtpRtcpConf; @@ -214,7 +214,7 @@ class JsepTrack { virtual nsresult Negotiate(const SdpMediaSection& answer, const SdpMediaSection& remote, const SdpMediaSection& local); - static void SetUniquePayloadTypes(std::vector<JsepTrack*>& tracks); + static void SetUniqueReceivePayloadTypes(std::vector<JsepTrack*>& tracks); virtual void GetNegotiatedPayloadTypes( std::vector<uint16_t>* payloadTypes) const; diff --git a/dom/media/webrtc/metrics.yaml b/dom/media/webrtc/metrics.yaml index da3d077f2d..aea5cf17fb 100644 --- a/dom/media/webrtc/metrics.yaml +++ b/dom/media/webrtc/metrics.yaml @@ -18,14 +18,16 @@ rtcrtpsender: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 count_setparameters_compat: type: counter @@ -35,14 +37,16 @@ rtcrtpsender: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 used_sendencodings: type: rate @@ -53,14 +57,16 @@ rtcrtpsender: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 rtcrtpsender.setparameters: warn_no_getparameters: @@ -74,14 +80,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 blame_no_getparameters: type: labeled_counter @@ -92,13 +100,15 @@ rtcrtpsender.setparameters: enough call to `getParameters`) Collected only on EARLY_BETA_OR_EARLIER. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - web_activity notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 warn_length_changed: type: rate @@ -110,14 +120,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 blame_length_changed: type: labeled_counter @@ -128,13 +140,15 @@ rtcrtpsender.setparameters: EARLY_BETA_OR_EARLIER. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - web_activity notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 warn_no_transactionid: type: rate @@ -146,14 +160,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 blame_no_transactionid: type: labeled_counter @@ -163,13 +179,15 @@ rtcrtpsender.setparameters: by the eTLD+1 of the site. Collected only on EARLY_BETA_OR_EARLIER. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1831343 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - web_activity notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_length_changed: type: rate @@ -181,14 +199,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_rid_changed: type: rate @@ -201,14 +221,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_no_getparameters: type: rate @@ -220,14 +242,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_no_transactionid: type: rate @@ -238,14 +262,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_stale_transactionid: type: rate @@ -256,14 +282,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_no_encodings: type: rate @@ -277,14 +305,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 fail_other: type: rate @@ -295,14 +325,16 @@ rtcrtpsender.setparameters: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1401592 - https://bugzilla.mozilla.org/show_bug.cgi?id=1832459 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881403 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 codec.stats: ulpfec_negotiated: @@ -312,13 +344,15 @@ codec.stats: on the first negotiation for each video transceiver. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 labels: - negotiated - not_negotiated @@ -329,13 +363,15 @@ codec.stats: Count how many other fec options are being offered. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 video_preferred_codec: type: labeled_counter @@ -343,13 +379,15 @@ codec.stats: Counts the preferred video codec being signaled to us to identify preferred video codec. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 audio_preferred_codec: type: labeled_counter @@ -357,10 +395,12 @@ codec.stats: Counts the preferred audio codec being signaled to us to identify preferred audio codec. bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1858213 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1881396 data_sensitivity: - technical notification_emails: - webrtc-telemetry-alerts@mozilla.com - expires: 126 + expires: 132 diff --git a/dom/media/webrtc/tests/mochitests/head.js b/dom/media/webrtc/tests/mochitests/head.js index 9a4e5682a3..1fd559217a 100644 --- a/dom/media/webrtc/tests/mochitests/head.js +++ b/dom/media/webrtc/tests/mochitests/head.js @@ -424,6 +424,7 @@ function setupEnvironment() { ["media.peerconnection.nat_simulator.block_udp", false], ["media.peerconnection.nat_simulator.redirect_address", ""], ["media.peerconnection.nat_simulator.redirect_targets", ""], + ["media.peerconnection.treat_warnings_as_errors", true], ], }; diff --git a/dom/media/webrtc/tests/mochitests/iceTestUtils.js b/dom/media/webrtc/tests/mochitests/iceTestUtils.js index d4d1f5c4b4..9e76e3f7df 100644 --- a/dom/media/webrtc/tests/mochitests/iceTestUtils.js +++ b/dom/media/webrtc/tests/mochitests/iceTestUtils.js @@ -300,3 +300,129 @@ async function checkNoRelay(iceServers) { ); pc.close(); } + +function gatheringStateReached(object, state) { + if (object instanceof RTCIceTransport) { + return new Promise(r => + object.addEventListener("gatheringstatechange", function listener() { + if (object.gatheringState == state) { + object.removeEventListener("gatheringstatechange", listener); + r(state); + } + }) + ); + } else if (object instanceof RTCPeerConnection) { + return new Promise(r => + object.addEventListener("icegatheringstatechange", function listener() { + if (object.iceGatheringState == state) { + object.removeEventListener("icegatheringstatechange", listener); + r(state); + } + }) + ); + } else { + throw "First parameter is neither an RTCIceTransport nor an RTCPeerConnection"; + } +} + +function nextGatheringState(object) { + if (object instanceof RTCIceTransport) { + return new Promise(resolve => + object.addEventListener( + "gatheringstatechange", + () => resolve(object.gatheringState), + { once: true } + ) + ); + } else if (object instanceof RTCPeerConnection) { + return new Promise(resolve => + object.addEventListener( + "icegatheringstatechange", + () => resolve(object.iceGatheringState), + { once: true } + ) + ); + } else { + throw "First parameter is neither an RTCIceTransport nor an RTCPeerConnection"; + } +} + +function emptyCandidate(pc) { + return new Promise(r => + pc.addEventListener("icecandidate", function listener(e) { + if (e.candidate && e.candidate.candidate == "") { + pc.removeEventListener("icecandidate", listener); + r(e); + } + }) + ); +} + +function nullCandidate(pc) { + return new Promise(r => + pc.addEventListener("icecandidate", function listener(e) { + if (!e.candidate) { + pc.removeEventListener("icecandidate", listener); + r(e); + } + }) + ); +} + +function connectionStateReached(object, state) { + if (object instanceof RTCIceTransport || object instanceof RTCDtlsTransport) { + return new Promise(resolve => + object.addEventListener("statechange", function listener() { + if (object.state == state) { + object.removeEventListener("statechange", listener); + resolve(state); + } + }) + ); + } else if (object instanceof RTCPeerConnection) { + return new Promise(resolve => + object.addEventListener("connectionstatechange", function listener() { + if (object.connectionState == state) { + object.removeEventListener("connectionstatechange", listener); + resolve(state); + } + }) + ); + } else { + throw "First parameter is neither an RTCIceTransport, an RTCDtlsTransport, nor an RTCPeerConnection"; + } +} + +function nextConnectionState(object) { + if (object instanceof RTCIceTransport || object instanceof RTCDtlsTransport) { + return new Promise(resolve => + object.addEventListener("statechange", () => resolve(object.state), { + once: true, + }) + ); + } else if (object instanceof RTCPeerConnection) { + return new Promise(resolve => + object.addEventListener( + "connectionstatechange", + () => resolve(object.connectionState), + { once: true } + ) + ); + } else { + throw "First parameter is neither an RTCIceTransport, an RTCDtlsTransport, nor an RTCPeerConnection"; + } +} + +function nextIceConnectionState(pc) { + if (pc instanceof RTCPeerConnection) { + return new Promise(resolve => + pc.addEventListener( + "iceconnectionstatechange", + () => resolve(pc.iceConnectionState), + { once: true } + ) + ); + } else { + throw "First parameter is not an RTCPeerConnection"; + } +} diff --git a/dom/media/webrtc/tests/mochitests/pc.js b/dom/media/webrtc/tests/mochitests/pc.js index 73e1b2c2f0..5778c61392 100644 --- a/dom/media/webrtc/tests/mochitests/pc.js +++ b/dom/media/webrtc/tests/mochitests/pc.js @@ -12,8 +12,8 @@ const iceStateTransitions = { checking: ["new", "connected", "failed", "closed"], //Note: do we need to // allow 'completed' in // here as well? - connected: ["new", "completed", "disconnected", "closed"], - completed: ["new", "disconnected", "closed"], + connected: ["new", "checking", "completed", "disconnected", "closed"], + completed: ["new", "checking", "disconnected", "closed"], disconnected: ["new", "connected", "completed", "failed", "closed"], failed: ["new", "disconnected", "closed"], closed: [], @@ -367,9 +367,7 @@ PeerConnectionTest.prototype.createDataChannel = function (options) { PeerConnectionTest.prototype.createAnswer = function (peer) { return peer.createAnswer().then(answer => { // make a copy so this does not get updated with ICE candidates - this.originalAnswer = new RTCSessionDescription( - JSON.parse(JSON.stringify(answer)) - ); + this.originalAnswer = JSON.parse(JSON.stringify(answer)); return answer; }); }; @@ -384,9 +382,7 @@ PeerConnectionTest.prototype.createAnswer = function (peer) { PeerConnectionTest.prototype.createOffer = function (peer) { return peer.createOffer().then(offer => { // make a copy so this does not get updated with ICE candidates - this.originalOffer = new RTCSessionDescription( - JSON.parse(JSON.stringify(offer)) - ); + this.originalOffer = JSON.parse(JSON.stringify(offer)); return offer; }); }; @@ -1399,10 +1395,6 @@ PeerConnectionWrapper.prototype = { }); }, - isTrackOnPC(track) { - return !!this.getStreamForRecvTrack(track); - }, - allExpectedTracksAreObserved(expected, observed) { return Object.keys(expected).every(trackId => observed[trackId]); }, @@ -1459,7 +1451,10 @@ PeerConnectionWrapper.prototype = { setupTrackEventHandler() { this._pc.addEventListener("track", ({ track, streams }) => { info(`${this}: 'ontrack' event fired for ${track.id}`); - ok(this.isTrackOnPC(track), `Found track ${track.id}`); + ok( + this._pc.getReceivers().some(r => r.track == track), + `Found track ${track.id}` + ); let gratuitousEvent = true; let streamsContainingTrack = this.remoteStreamsByTrackId.get(track.id); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_RTCIceTransport.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_RTCIceTransport.html new file mode 100644 index 0000000000..893d1aad5b --- /dev/null +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_RTCIceTransport.html @@ -0,0 +1,197 @@ +<!DOCTYPE HTML> +<html> +<head> + <script type="application/javascript" src="pc.js"></script> + <script type="application/javascript" src="iceTestUtils.js"></script> + <script type="application/javascript" src="helpers_from_wpt/sdp.js"></script></head> +<body> +<pre id="test"> +<script type="application/javascript"> + createHTML({ + bug: "1811912", + title: "Tests for RTCIceTransport" + }); + + const tests = [ + async function iceRestartNewGathersFirst() { + await pushPrefs( + ['media.peerconnection.nat_simulator.block_udp', true]); + // We block UDP, and configure a fake UDP STUN server; this should result + // in gathering taking a while, because we'll just be sending UDP packets + // off to be eaten by the NAT simulator. + const pc1 = new RTCPeerConnection({ iceServers: [{ + urls: ['stun:192.168.1.1'] + }] }); + const pc2 = new RTCPeerConnection(); + + const transceiver = pc1.addTransceiver('audio'); + await pc1.setLocalDescription(); + const {iceTransport} = transceiver.sender.transport; + is(await nextGatheringState(iceTransport), 'gathering'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + pc1.setConfiguration({iceServers: []}); + await pc1.setLocalDescription(await pc1.createOffer({iceRestart: true})); + is(iceTransport.gatheringState, 'gathering'); + + const gatheringDonePromise = nextGatheringState(iceTransport); + // The empty candidate should fire for the new underlying transport + // (created by the ICE restart), but there should be no gathering state + // change yet. (Spec says that the RTCIceTransport object is the same + // here, even thougb there's actually a new transport) + await emptyCandidate(pc1); + + // New transport is done gathering, old should not be. + let result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 100))]); + is(result, undefined, 'Gathering should not complete yet'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + // We might or might not have an empty candidate for the generation we + // just abandoned, depending on how the spec shakes out. + + // Completing negotiation should result in a transition to complete very + // quickly + result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 100))]); + is(result, 'complete', 'Gathering should complete soon after the ICE restart is complete'); + pc1.close(); + pc2.close(); + await SpecialPowers.popPrefEnv(); + }, + + async function iceRestartNewGathersFirstThenRollback() { + await pushPrefs( + ['media.peerconnection.nat_simulator.block_udp', true]); + // This should result in gathering taking a while + const pc1 = new RTCPeerConnection({ iceServers: [{ + urls: ['stun:192.168.1.1'] + }] }); + const pc2 = new RTCPeerConnection(); + + const transceiver = pc1.addTransceiver('audio'); + await pc1.setLocalDescription(); + const {iceTransport} = transceiver.sender.transport; + is(await nextGatheringState(iceTransport), 'gathering'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + pc1.setConfiguration({iceServers: []}); + await pc1.setLocalDescription(await pc1.createOffer({iceRestart: true})); + is(iceTransport.gatheringState, 'gathering'); + + const gatheringDonePromise = nextGatheringState(iceTransport); + // This should fire for the new transport, but there should be no + // gathering state change yet. + await emptyCandidate(pc1); + + // Rollback should abandon the new transport, and we should remain in + // 'gathering' + await pc1.setLocalDescription({type: 'rollback', sdp: ''}); + + let result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 1000))]); + is(result, undefined, 'Gathering should not complete'); + pc1.close(); + pc2.close(); + await SpecialPowers.popPrefEnv(); + }, + + async function iceRestartOldGathersFirst() { + await pushPrefs( + ['media.peerconnection.nat_simulator.block_udp', true]); + const pc1 = new RTCPeerConnection(); + const pc2 = new RTCPeerConnection(); + + const transceiver = pc1.addTransceiver('audio'); + await pc1.setLocalDescription(); + const {iceTransport} = transceiver.sender.transport; + is(await nextGatheringState(iceTransport), 'gathering'); + is(await nextGatheringState(iceTransport), 'complete'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + // This should result in gathering taking a while + pc1.setConfiguration({ iceServers: [{ + urls: ['stun:192.168.1.1'] + }] }); + await pc1.setLocalDescription(await pc1.createOffer({iceRestart: true})); + + is(await nextGatheringState(iceTransport), 'gathering'); + + const gatheringDonePromise = nextGatheringState(iceTransport); + let result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 100))]); + is(result, undefined, 'Gathering should not complete'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 100))]); + is(result, undefined, 'Gathering should not complete'); + pc1.close(); + pc2.close(); + await SpecialPowers.popPrefEnv(); + }, + + async function iceRestartOldGathersFirstThenRollback() { + await pushPrefs( + ['media.peerconnection.nat_simulator.block_udp', true]); + const pc1 = new RTCPeerConnection(); + const pc2 = new RTCPeerConnection(); + + const transceiver = pc1.addTransceiver('audio'); + await pc1.setLocalDescription(); + const {iceTransport} = transceiver.sender.transport; + is(await nextGatheringState(iceTransport), 'gathering'); + is(await nextGatheringState(iceTransport), 'complete'); + + await pc2.setRemoteDescription(pc1.localDescription); + await pc2.setLocalDescription(); + await pc1.setRemoteDescription(pc2.localDescription); + + // This should result in gathering taking a while + pc1.setConfiguration({ iceServers: [{ + urls: ['stun:192.168.1.1'] + }] }); + await pc1.setLocalDescription(await pc1.createOffer({iceRestart: true})); + + is(await nextGatheringState(iceTransport), 'gathering', 'ICE restart should put us back in gathering'); + + const gatheringDonePromise = nextGatheringState(iceTransport); + is(iceTransport.gatheringState, 'gathering'); + await pc1.setLocalDescription({type: 'rollback', sdp: ''}); + const result = await Promise.race([gatheringDonePromise, new Promise(r => setTimeout(r, 200))]); + is(iceTransport.gatheringState, 'complete', 'Rollback of ICE restart to transport that was already done gathering should result in a transition back to "complete"'); + is(result, 'complete', 'Rollback that brings the gathering state back to complete should result in a gatheringstatechange event'); + pc1.close(); + pc2.close(); + await SpecialPowers.popPrefEnv(); + }, + ]; + + runNetworkTest(async () => { + for (const test of tests) { + info(`Running test: ${test.name}`); + try { + await test(); + } catch (e) { + ok(false, `Caught ${e.name}: ${e.message} ${e.stack}`); + } + info(`Done running test: ${test.name}`); + // Make sure we don't build up a pile of GC work, and also get PCImpl to + // print their timecards. + await new Promise(r => SpecialPowers.exactGC(r)); + } + }, { useIceServer: true }); +</script> +</pre> +</body> +</html> diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html index 180abc075a..7f3f9f57ad 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html @@ -24,6 +24,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.filtering_type', 'PORT_DEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'PORT_DEPENDENT'], ['media.peerconnection.nat_simulator.block_tcp', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "srflx"; options.expectedRemoteCandidateType = "relay"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html index 7bb51764bd..387341151f 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html @@ -26,6 +26,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_tcp', false], ['media.peerconnection.nat_simulator.block_tls', true], ['media.peerconnection.ice.loopback', true], + // The above sets up 5+ ICE servers which triggers a warning + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tcp"; options.expectedRemoteCandidateType = "relay-tcp"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html index 7446401f87..2ac886c156 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html @@ -25,6 +25,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_udp', true], ['media.peerconnection.nat_simulator.block_tcp', true], ['media.peerconnection.ice.loopback', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tls"; options.expectedRemoteCandidateType = "relay-tls"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html index 78fa8bcb2c..69fc2af80a 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html @@ -24,6 +24,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.filtering_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.block_tcp', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "srflx"; options.expectedRemoteCandidateType = "srflx"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html index 297121cd94..4175419d44 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html @@ -25,6 +25,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_udp', true], ['media.peerconnection.nat_simulator.error_code_for_drop', 3 /*R_INTERNAL*/], ['media.peerconnection.nat_simulator.block_tls', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tcp"; options.expectedRemoteCandidateType = "relay-tcp"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html index ced57ff8a3..313d76441b 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html @@ -14,7 +14,10 @@ createHTML({ runNetworkTest(async () => { await pushPrefs( // Enable mDNS, since there are some checks we want to run with that - ['media.peerconnection.ice.obfuscate_host_addresses', true]); + ['media.peerconnection.ice.obfuscate_host_addresses', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], + ); const offerer = new RTCPeerConnection({iceServers: iceServersArray, iceTransportPolicy: 'relay'}); const answerer = new RTCPeerConnection({iceServers: iceServersArray}); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html index 5cd168af8a..5efb4a17d2 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html @@ -25,7 +25,7 @@ var makePC = (config, expected_error) => { exception = e; } is((exception? exception.name : "success"), expected_error || "success", - "RTCPeerConnection(" + JSON.stringify(config) + ")"); + "RTCPeerConnection(" + JSON.stringify(config) + ") " + exception?.message); }; // The order of properties in objects is not guaranteed in JavaScript, so this @@ -39,8 +39,8 @@ var toComparable = o => }, {}); // This is a test of the iceServers parsing code + readable errors -runNetworkTest(() => { - var exception = null; +runNetworkTest(async () => { + let exception = null; try { new RTCPeerConnection().close(); @@ -56,26 +56,37 @@ runNetworkTest(() => { { urls:"stun:127.0.0.1" }, { urls:"stun:localhost", foo:"" }, { urls: ["stun:127.0.0.1", "stun:localhost"] }, - { urls:"stuns:localhost", foo:"" }, + ]}); + makePC({ iceServers: [ { urls:"turn:[::1]:3478", username:"p", credential:"p" }, { urls:"turn:[::1]:3478", username:"", credential:"" }, { urls:"turns:[::1]:3478", username:"", credential:"" }, + ]}); + makePC({ iceServers: [ { urls:"turn:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls: ["turn:[::1]:3478", "turn:localhost"], username:"p", credential:"p" }, { urls:"turns:localhost:3478?transport=udp", username:"p", credential:"p" }, - { url:"stun:localhost", foo:"" }, - { url:"turn:localhost", username:"p", credential:"p" } ]}); - makePC({ iceServers: [{ urls:"http:0.0.0.0" }] }, "SyntaxError"); try { - new RTCPeerConnection({ iceServers: [{ url:"http:0.0.0.0" }] }).close(); + new RTCPeerConnection({ iceServers: [{ urls:"http:0.0.0.0" }] }).close(); } catch (e) { ok(e.message.indexOf("http") > 0, "RTCPeerConnection() constructor has readable exceptions"); } + const push = prefs => SpecialPowers.pushPrefEnv(prefs); + + // Remaining tests trigger warnings + await push({ set: [['media.peerconnection.treat_warnings_as_errors', false]] }); + + makePC({ iceServers: [ + { urls:"stuns:localhost", foo:"" }, + { url:"stun:localhost", foo:"" }, + { url:"turn:localhost", username:"p", credential:"p" } + ]}); + // Test getConfiguration const config = { bundlePolicy: "max-bundle", @@ -98,41 +109,35 @@ runNetworkTest(() => { JSON.stringify(toComparable(config)), "getConfiguration"); pc.close(); - var push = prefs => SpecialPowers.pushPrefEnv(prefs); - - return Promise.resolve() // This set of tests are setting the about:config User preferences for default // ice servers and checking the outputs when RTCPeerConnection() is // invoked. See Bug 1167922 for more information. - .then(() => push({ set: [['media.peerconnection.default_iceservers', ""]] }) - .then(() => makePC()) - .then(() => push({ set: [['media.peerconnection.default_iceservers', "k"]] })) - .then(() => makePC()) - .then(() => push({ set: [['media.peerconnection.default_iceservers', "[{\"urls\": [\"stun:stun.services.mozilla.com\"]}]"]] })) - .then(() => makePC())) + await push({ set: [['media.peerconnection.default_iceservers', ""]] }); + makePC(); + await push({ set: [['media.peerconnection.default_iceservers', "k"]] }); + makePC(); + await push({ set: [['media.peerconnection.default_iceservers', + "[{\"urls\": [\"stun:stun.services.mozilla.com\"]}]"]]}); + makePC(); // This set of tests check that warnings work. See Bug 1254839 for more. - .then(() => { - let promise = new Promise(resolve => { - SpecialPowers.registerConsoleListener(msg => { - if (msg.message.includes("onaddstream")) { - SpecialPowers.postConsoleSentinel(); - resolve(msg.message); - } - }); + const warning = await new Promise(resolve => { + SpecialPowers.registerConsoleListener(msg => { + if (msg.message.includes("onaddstream")) { + SpecialPowers.postConsoleSentinel(); + resolve(msg.message); + } }); lineNumberAndFunction.func(); - return promise; - }).then(warning => { - is(warning.split('"')[1], - "WebRTC: onaddstream is deprecated! Use peerConnection.ontrack instead.", - "warning logged"); - var remainder = warning.split('"').slice(2).join('"'); - info(remainder); - ok(remainder.includes('file: "' + window.location + '"'), - "warning has this file"); - ok(remainder.includes('line: ' + lineNumberAndFunction.line), - "warning has correct line number"); - }); + }); + is(warning.split('"')[1], + "WebRTC: onaddstream is deprecated! Use peerConnection.ontrack instead.", + "warning logged"); + const remainder = warning.split('"').slice(2).join('"'); + info(remainder); + ok(remainder.includes('file: "' + window.location + '"'), + "warning has this file"); + ok(remainder.includes('line: ' + lineNumberAndFunction.line), + "warning has correct line number"); }); </script> </pre> diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html index 4c890e4400..ffd7a1b0c1 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html @@ -1,4 +1,4 @@ -<!DOCTYPE HTML> +<!DOCTYPE HTML> <html> <head> <script type="application/javascript" src="pc.js"></script> @@ -50,22 +50,27 @@ pc2.onicecandidate = e => { .catch(generateErrorCallback()); }; -var v1, v2; -var delivered = new Promise(resolve => { - pc2.onaddstream = e => { - v2.srcObject = e.stream; - resolve(e.stream); - }; -}); +runNetworkTest(async function() { + // Tests trigger warnings + await SpecialPowers.pushPrefEnv({ + set: [['media.peerconnection.treat_warnings_as_errors', false]] + }); + + const v1 = createMediaElement('video', 'v1'); + const v2 = createMediaElement('video', 'v2'); + + const delivered = new Promise(resolve => { + pc2.onaddstream = e => { + v2.srcObject = e.stream; + resolve(e.stream); + }; + }); -runNetworkTest(function() { - v1 = createMediaElement('video', 'v1'); - v2 = createMediaElement('video', 'v2'); var canPlayThrough = new Promise(resolve => v2.canplaythrough = resolve); is(v2.currentTime, 0, "v2.currentTime is zero at outset"); // not testing legacy gUM here - return navigator.mediaDevices.getUserMedia({ video: true, audio: true }) + await navigator.mediaDevices.getUserMedia({ video: true, audio: true }) .then(stream => pc1.addStream(v1.srcObject = stream)) .then(() => pcall(pc1, pc1.createOffer)) .then(offer => pcall(pc1, pc1.setLocalDescription, offer)) diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html index 5bdc8cc029..8c9470e7f2 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html @@ -32,7 +32,7 @@ // Rolling back should shut down gathering function PC_REMOTE_WAIT_FOR_END_OF_TRICKLE(test) { - return test.pcRemote.endOfTrickleIce; + is(test.pcRemote._pc.iceGatheringState, "new"); }, function PC_REMOTE_SETUP_ICE_HANDLER(test) { diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html index 7cd695ff54..589d8ed6ad 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html @@ -46,10 +46,14 @@ runNetworkTest(() => { { urls:"turn:[::1]:5349", username:"p", credential:"p" }, { urls:"turn:[::1]:3478", username:"p", credential:"p" }, { urls:"turn:[::1]", username:"p", credential:"p" }, + ]}); + makePC({ iceServers: [ { urls:"turn:localhost:53?transport=udp", username:"p", credential:"p" }, { urls:"turn:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls:"turn:localhost:53?transport=tcp", username:"p", credential:"p" }, { urls:"turn:localhost:3478?transport=tcp", username:"p", credential:"p" }, + ]}); + makePC({ iceServers: [ { urls:"turns:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls:"stun:localhost", foo:"" } ]}); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html index d94bb084b7..add3882a4d 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html @@ -21,12 +21,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html index 6bbf9440fc..3307258ef9 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html @@ -31,10 +31,6 @@ test.chain.replaceAfter('PC_REMOTE_CREATE_ANSWER', [ - function PC_LOCAL_EXPECT_ICE_CONNECTED(test) { - test.pcLocal.iceCheckingIceRollbackExpected = true; - }, - function PC_REMOTE_ROLLBACK(test) { return test.setRemoteDescription(test.pcRemote, { type: "rollback" }, STABLE); @@ -58,12 +54,6 @@ return test.pcLocal.endOfTrickleIce; }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); - }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ], 1 // Replaces after second PC_REMOTE_CREATE_ANSWER ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html index f5f9a1f220..a7adb7c3e9 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html @@ -33,9 +33,6 @@ HAVE_LOCAL_OFFER); }); }, - function PC_LOCAL_EXPECT_ICE_CONNECTED(test) { - test.pcLocal.iceCheckingIceRollbackExpected = true; - }, function PC_LOCAL_WAIT_FOR_GATHERING(test) { return new Promise(r => { test.pcLocal._pc.addEventListener("icegatheringstatechange", () => { @@ -54,12 +51,6 @@ function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { return test.pcLocal.endOfTrickleIce; }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); - }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html index 134fa97cc0..ef40d4039c 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html @@ -23,12 +23,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html index 06a3a3c980..85a3fcc45d 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html @@ -24,12 +24,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html index 5d4780211a..80fd1090bc 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html @@ -23,12 +23,15 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + // Expect gathering to start again + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html index c8354356b0..aa0644ebbb 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html @@ -15,12 +15,12 @@ visible: true }); - function makeCodecTopPriority(sdp, codec) { + function makeCodecTopPriority({type, sdp}, codec) { const ptToMove = sdputils.findCodecId(sdp, codec); - return sdp.replace( + return {type, sdp: sdp.replace( // m=video port type pts ptToMove more-pts? new RegExp(`(m=video [^ ]+ [^ ]+)(.*)( ${ptToMove})( [^ ]+)?`, "g"), - '$1$3$2$4'); + '$1$3$2$4')}; } function isCodecFirst(sdp, codec) { @@ -34,8 +34,7 @@ const stream = await navigator.mediaDevices.getUserMedia({ video: true }); const sender = pc1.addTrack(stream.getTracks()[0]); await pc1.setLocalDescription(); - let mungedOffer = pc1.localDescription; - mungedOffer.sdp = makeCodecTopPriority(mungedOffer.sdp, codec); + const mungedOffer = makeCodecTopPriority(pc1.localDescription, codec); await pc2.setRemoteDescription(mungedOffer); await pc2.setLocalDescription(); await pc1.setRemoteDescription(pc2.localDescription); @@ -51,8 +50,7 @@ await pc1.setLocalDescription(); await pc2.setRemoteDescription(pc1.localDescription); await pc2.setLocalDescription(); - let mungedAnswer = pc2.localDescription; - mungedAnswer.sdp = makeCodecTopPriority(mungedAnswer.sdp, codec); + const mungedAnswer = makeCodecTopPriority(pc2.localDescription, codec); await pc1.setRemoteDescription(mungedAnswer); is(isCodecFirst((await pc1.createOffer()).sdp, codec), !isPseudoCodec, "Top-priority codecs should come first in reoffers, unless they are pseudo codecs (eg; ulpfec)"); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html index cdc328fd2b..5146d423a6 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html @@ -23,6 +23,8 @@ if (!("mediaDevices" in navigator)) { await pushPrefs( ['media.peerconnection.nat_simulator.filtering_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'ENDPOINT_INDEPENDENT'], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); const test = new PeerConnectionTest(options); makeOffererNonTrickle(test.chain); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html index 75f0d12463..8627590b65 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html @@ -98,10 +98,10 @@ runNetworkTest(async () => { //info("Original OFFER: " + JSON.stringify(offer)); offer.sdp = sdputils.removeBundle(offer.sdp); //info("OFFER w/o BUNDLE: " + JSON.stringify(offer)); - const offerAudio = new RTCSessionDescription(JSON.parse(JSON.stringify(offer))); + const offerAudio = JSON.parse(JSON.stringify(offer)); offerAudio.sdp = offerAudio.sdp.replace('m=video 9', 'm=video 0'); //info("offerAudio: " + JSON.stringify(offerAudio)); - const offerVideo = new RTCSessionDescription(JSON.parse(JSON.stringify(offer))); + const offerVideo = JSON.parse(JSON.stringify(offer)); offerVideo.sdp = offerVideo.sdp.replace('m=audio 9', 'm=audio 0'); //info("offerVideo: " + JSON.stringify(offerVideo)); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html index c476c60161..342b72a5e1 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html @@ -11,7 +11,12 @@ title: "Throw in PeerConnection callbacks" }); -runNetworkTest(function () { +runNetworkTest(async function () { + // Tests trigger warnings + await SpecialPowers.pushPrefEnv({ + set: [['media.peerconnection.treat_warnings_as_errors', false]] + }); + let finish; const onfinished = new Promise(r => finish = async () => { window.onerror = oldOnError; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html index f685f7c99a..8c12a50942 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html @@ -8,49 +8,70 @@ <script type="application/javascript"> createHTML({ bug: "1264479", - title: "PeerConnection verify current and pending descriptions" + title: "PeerConnection verify current and pending descriptions in legacy mode" }); - const pc1 = new RTCPeerConnection(); - const pc2 = new RTCPeerConnection(); + // Mostly covered by WPT tests these days, except for in our legacy mode - var add = (pc, can, failed) => can && pc.addIceCandidate(can).catch(failed); - pc1.onicecandidate = e => add(pc2, e.candidate, generateErrorCallback()); - pc2.onicecandidate = e => add(pc1, e.candidate, generateErrorCallback()); + const isSimilarDescription = (d1, d2, msg) => { + isnot(d1, d2, msg + " — not the same object"); + is(d1.type, d2.type, msg + " — but same type"); + is(d1.sdp, d2.sdp, msg + " — and same sdp"); + } + runNetworkTest(async () => { + await SpecialPowers.pushPrefEnv({ + set: [['media.peerconnection.description.legacy.enabled', true]] + }); + + const pc1 = new RTCPeerConnection(); + const pc2 = new RTCPeerConnection(); + + const add = (pc, can, failed) => can && pc.addIceCandidate(can).catch(failed); + pc1.onicecandidate = e => add(pc2, e.candidate, generateErrorCallback()); + pc2.onicecandidate = e => add(pc1, e.candidate, generateErrorCallback()); - runNetworkTest(function() { const v1 = createMediaElement('video', 'v1'); const v2 = createMediaElement('video', 'v2'); - return navigator.mediaDevices.getUserMedia({ video: true, audio: true }) - .then(stream => (v1.srcObject = stream).getTracks().forEach(t => pc1.addTrack(t, stream))) - .then(() => pc1.createOffer({})) // check that createOffer accepts arg. - .then(offer => pc1.setLocalDescription(offer)) - .then(() => { - ok(!pc1.currentLocalDescription, "pc1 currentLocalDescription is empty"); - ok(pc1.pendingLocalDescription, "pc1 pendingLocalDescription is set"); - ok(pc1.localDescription, "pc1 localDescription is set"); - }) - .then(() => pc2.setRemoteDescription(pc1.localDescription)) - .then(() => { - ok(!pc2.currentRemoteDescription, "pc2 currentRemoteDescription is empty"); - ok(pc2.pendingRemoteDescription, "pc2 pendingRemoteDescription is set"); - ok(pc2.remoteDescription, "pc2 remoteDescription is set"); - }) - .then(() => pc2.createAnswer({})) // check that createAnswer accepts arg. - .then(answer => pc2.setLocalDescription(answer)) - .then(() => { - ok(pc2.currentLocalDescription, "pc2 currentLocalDescription is set"); - ok(!pc2.pendingLocalDescription, "pc2 pendingLocalDescription is empty"); - ok(pc2.localDescription, "pc2 localDescription is set"); - }) - .then(() => pc1.setRemoteDescription(pc2.localDescription)) - .then(() => { - ok(pc1.currentRemoteDescription, "pc1 currentRemoteDescription is set"); - ok(!pc1.pendingRemoteDescription, "pc1 pendingRemoteDescription is empty"); - ok(pc1.remoteDescription, "pc1 remoteDescription is set"); + const stream = await navigator.mediaDevices.getUserMedia({ + video: true, audio: true }); + v1.srcObject = stream; + for (const track of stream.getTracks()) { + pc1.addTrack(track, stream); + } + await pc1.setLocalDescription(); + is(pc1.currentLocalDescription, null, "pc1 currentLocalDescription is null"); + ok(pc1.pendingLocalDescription, "pc1 pendingLocalDescription is set"); + ok(pc1.localDescription, "pc1 localDescription is set"); + isSimilarDescription(pc1.pendingLocalDescription, pc1.pendingLocalDescription, "pendingLocalDescription"); + isSimilarDescription(pc1.localDescription, pc1.localDescription, "localDescription"); + isSimilarDescription(pc1.localDescription, pc1.pendingLocalDescription, "local and pending"); + + await pc2.setRemoteDescription(pc1.localDescription); + is(pc2.currentRemoteDescription, null, "pc2 currentRemoteDescription is null"); + ok(pc2.pendingRemoteDescription, "pc2 pendingRemoteDescription is set"); + ok(pc2.remoteDescription, "pc2 remoteDescription is set"); + isSimilarDescription(pc2.pendingRemoteDescription, pc2.pendingRemoteDescription, "pendingRemoteDescription"); + isSimilarDescription(pc2.remoteDescription, pc2.remoteDescription, "remoteDescription"); + isSimilarDescription(pc2.remoteDescription, pc2.pendingRemoteDescription, "remote and pending"); + + await pc2.setLocalDescription(); + ok(pc2.currentLocalDescription, "pc2 currentLocalDescription is set"); + is(pc2.pendingLocalDescription, null, "pc2 pendingLocalDescription is null"); + ok(pc2.localDescription, "pc2 localDescription is set"); + isSimilarDescription(pc2.currentLocalDescription, pc2.currentLocalDescription, "currentLocalDescription"); + isSimilarDescription(pc2.localDescription, pc2.localDescription, "localDescription"); + isSimilarDescription(pc2.localDescription, pc2.currentLocalDescription, "local and current"); + + await pc1.setRemoteDescription(pc2.localDescription); + ok(pc1.currentRemoteDescription, "pc1 currentRemoteDescription is set"); + is(pc1.pendingRemoteDescription, null, "pc1 pendingRemoteDescription is null"); + ok(pc1.remoteDescription, "pc1 remoteDescription is set"); + isSimilarDescription(pc1.currentRemoteDescription, pc1.currentRemoteDescription, "currentRemoteDescription"); + isSimilarDescription(pc1.remoteDescription, pc1.remoteDescription, "remoteDescription"); + isSimilarDescription(pc1.remoteDescription, pc1.currentRemoteDescription, "remote and current"); }); </script> </pre> diff --git a/dom/media/webrtc/third_party_build/default_config_env b/dom/media/webrtc/third_party_build/default_config_env index af8b77bba6..7013520a30 100644 --- a/dom/media/webrtc/third_party_build/default_config_env +++ b/dom/media/webrtc/third_party_build/default_config_env @@ -5,41 +5,41 @@ export MOZ_LIBWEBRTC_SRC=$STATE_DIR/moz-libwebrtc # The previous fast-forward bug number is used for some error messaging. -export MOZ_PRIOR_FASTFORWARD_BUG="1867099" +export MOZ_PRIOR_FASTFORWARD_BUG="1871981" # Fast-forwarding each Chromium version of libwebrtc should be done # under a separate bugzilla bug. This bug number is used when crafting # the commit summary as each upstream commit is vendored into the # mercurial repository. The bug used for the v106 fast-forward was # 1800920. -export MOZ_FASTFORWARD_BUG="1871981" +export MOZ_FASTFORWARD_BUG="1876843" # MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are # not used during fast-forward processing, but facilitate generating this # default config. To generate an default config for the next update, run # bash dom/media/webrtc/third_party_build/update_default_config_env.sh -export MOZ_NEXT_LIBWEBRTC_MILESTONE=120 -export MOZ_NEXT_FIREFOX_REL_TARGET=124 +export MOZ_NEXT_LIBWEBRTC_MILESTONE=121 +export MOZ_NEXT_FIREFOX_REL_TARGET=125 # For Chromium release branches, see: # https://chromiumdash.appspot.com/branches -# Chromium's v119 release branch was 6045. This is used to pre-stack +# Chromium's v120 release branch was 6099. This is used to pre-stack # the previous release branch's commits onto the appropriate base commit # (the first common commit between trunk and the release branch). -export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="6045" +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="6099" -# New target release branch for v120 is branch-heads/6099. This is used +# New target release branch for v121 is branch-heads/6167. This is used # to calculate the next upstream commit. -export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/6099" +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/6167" # For local development 'mozpatches' is fine for a branch name, but when # pushing the patch stack to github, it should be named something like -# 'moz-mods-chr120-for-rel124'. +# 'moz-mods-chr121-for-rel125'. export MOZ_LIBWEBRTC_BRANCH="mozpatches" # After elm has been merged to mozilla-central, the patch stack in # moz-libwebrtc should be pushed to github. The script # push_official_branch.sh uses this branch name when pushing to the # public repo. -export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr120-for-rel124" +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr121-for-rel125" diff --git a/dom/media/webrtc/third_party_build/filter_git_changes.py b/dom/media/webrtc/third_party_build/filter_git_changes.py index ee6cdbcbd9..86358b0b01 100644 --- a/dom/media/webrtc/third_party_build/filter_git_changes.py +++ b/dom/media/webrtc/third_party_build/filter_git_changes.py @@ -48,7 +48,7 @@ def filter_git_changes(github_path, commit_sha, diff_filter): # out the excluded directory paths (note the lack of trailing '$' # in the regex). regex_excludes = "|".join( - ["^(M|A|D|R\d\d\d)\t{}".format(i) for i in exclude_dir_list] + ["^(M|A|D|R\\d\\d\\d)\t{}".format(i) for i in exclude_dir_list] ) files_not_excluded = [ path for path in changed_files if not re.findall(regex_excludes, path) diff --git a/dom/media/webrtc/third_party_build/prep_repo.sh b/dom/media/webrtc/third_party_build/prep_repo.sh index b601ebc808..8cd9ff6816 100644 --- a/dom/media/webrtc/third_party_build/prep_repo.sh +++ b/dom/media/webrtc/third_party_build/prep_repo.sh @@ -69,6 +69,12 @@ rm -f *.patch CHERRY_PICK_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` echo "common commit: $CHERRY_PICK_BASE" +# find the last upstream commit used by the previous update, so we don't +# accidentally grab release branch commits that were added after we started +# the previous update. +LAST_UPSTREAM_COMMIT_SHA=`tail -1 $CURRENT_DIR/third_party/libwebrtc/README.moz-ff-commit` +echo "previous update's last commit: $LAST_UPSTREAM_COMMIT_SHA" + # create a new branch at the common commit and checkout the new branch ERROR_HELP=$" Unable to create branch '$MOZ_LIBWEBRTC_BRANCH'. This probably means @@ -89,7 +95,7 @@ git checkout $MOZ_LIBWEBRTC_BRANCH rm -f $TMP_DIR/*.patch $TMP_DIR/*.patch.bak # grab the patches for all the commits in chrome's release branch for libwebrtc -git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM +git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..$LAST_UPSTREAM_COMMIT_SHA # tweak the release branch commit summaries to show they were cherry picked sed -i.bak -e "/^Subject: / s/^Subject: /Subject: (cherry-pick-branch-heads\/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM) /" $TMP_DIR/*.patch git am $TMP_DIR/*.patch # applies to branch mozpatches diff --git a/dom/media/webrtc/third_party_build/save_patch_stack.py b/dom/media/webrtc/third_party_build/save_patch_stack.py index 9b080ac0ff..907b90f593 100644 --- a/dom/media/webrtc/third_party_build/save_patch_stack.py +++ b/dom/media/webrtc/third_party_build/save_patch_stack.py @@ -52,7 +52,7 @@ def save_patch_stack( # remove the commit summary from the file name patches_to_rename = os.listdir(patch_directory) for file in patches_to_rename: - shortened_name = re.sub("^(\d\d\d\d)-.*\.patch", "\\1.patch", file) + shortened_name = re.sub(r"^(\d\d\d\d)-.*\.patch", "\\1.patch", file) os.rename( os.path.join(patch_directory, file), os.path.join(patch_directory, shortened_name), diff --git a/dom/media/webrtc/third_party_build/verify_vendoring.sh b/dom/media/webrtc/third_party_build/verify_vendoring.sh index 869a2c8c67..008ab0d5db 100644 --- a/dom/media/webrtc/third_party_build/verify_vendoring.sh +++ b/dom/media/webrtc/third_party_build/verify_vendoring.sh @@ -17,12 +17,12 @@ echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" -TIP_SHA=`hg id -r tip | awk '{ print $1; }'` -echo "TIP_SHA: $TIP_SHA" +CURRENT_SHA=`hg id -r . | awk '{ print $1; }'` +echo "CURRENT_SHA: $CURRENT_SHA" # we grab the entire firstline description for convenient logging -LAST_PATCHSTACK_UPDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ - --include "third_party/libwebrtc/moz-patch-stack/*.patch" | head -1` +LAST_PATCHSTACK_UPDATE_COMMIT=`hg log -r ::. --template "{node|short} {desc|firstline}\n" \ + --include "third_party/libwebrtc/moz-patch-stack/*.patch" | tail -1` echo "LAST_PATCHSTACK_UPDATE_COMMIT: $LAST_PATCHSTACK_UPDATE_COMMIT" LAST_PATCHSTACK_UPDATE_COMMIT_SHA=`echo $LAST_PATCHSTACK_UPDATE_COMMIT \ @@ -31,7 +31,7 @@ echo "LAST_PATCHSTACK_UPDATE_COMMIT_SHA: $LAST_PATCHSTACK_UPDATE_COMMIT_SHA" # grab the oldest, non "Vendor from libwebrtc" line OLDEST_CANDIDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ - -r $LAST_PATCHSTACK_UPDATE_COMMIT_SHA::tip \ + -r $LAST_PATCHSTACK_UPDATE_COMMIT_SHA::. \ | grep -v "Vendor libwebrtc from" | head -1` echo "OLDEST_CANDIDATE_COMMIT: $OLDEST_CANDIDATE_COMMIT" @@ -39,9 +39,9 @@ OLDEST_CANDIDATE_SHA=`echo $OLDEST_CANDIDATE_COMMIT \ | awk '{ print $1; }'` echo "OLDEST_CANDIDATE_SHA: $OLDEST_CANDIDATE_SHA" -EXTRACT_COMMIT_RANGE="{start-commit-sha}::tip" -if [ "x$TIP_SHA" != "x$OLDEST_CANDIDATE_SHA" ]; then - EXTRACT_COMMIT_RANGE="$OLDEST_CANDIDATE_SHA::tip" +EXTRACT_COMMIT_RANGE="{start-commit-sha}::." +if [ "x$CURRENT_SHA" != "x$OLDEST_CANDIDATE_SHA" ]; then + EXTRACT_COMMIT_RANGE="$OLDEST_CANDIDATE_SHA::." echo "EXTRACT_COMMIT_RANGE: $EXTRACT_COMMIT_RANGE" fi diff --git a/dom/media/webrtc/transport/nricectx.cpp b/dom/media/webrtc/transport/nricectx.cpp index f30c2734c2..7c71c0ab06 100644 --- a/dom/media/webrtc/transport/nricectx.cpp +++ b/dom/media/webrtc/transport/nricectx.cpp @@ -257,14 +257,14 @@ nsresult NrIceTurnServer::ToNicerTurnStruct(nr_ice_turn_server* server) const { } NrIceCtx::NrIceCtx(const std::string& name) - : connection_state_(ICE_CTX_INIT), - gathering_state_(ICE_CTX_GATHER_INIT), - name_(name), + : name_(name), ice_controlling_set_(false), ctx_(nullptr), peer_(nullptr), ice_handler_vtbl_(nullptr), ice_handler_(nullptr), + ice_gather_handler_vtbl_(nullptr), + ice_gather_handler_(nullptr), trickle_(true), config_(), nat_(nullptr), @@ -343,13 +343,10 @@ void NrIceCtx::DestroyStream(const std::string& id) { auto it = streams_.find(id); if (it != streams_.end()) { auto preexisting_stream = it->second; + SignalConnectionStateChange(preexisting_stream, ICE_CTX_CLOSED); streams_.erase(it); preexisting_stream->Close(); } - - if (streams_.empty()) { - SetGatheringState(ICE_CTX_GATHER_INIT); - } } // Handler callbacks @@ -377,6 +374,7 @@ int NrIceCtx::stream_ready(void* obj, nr_ice_media_stream* stream) { MOZ_ASSERT(s); s->Ready(stream); + ctx->SignalConnectionStateChange(s, ICE_CTX_CONNECTED); return 0; } @@ -393,44 +391,101 @@ int NrIceCtx::stream_failed(void* obj, nr_ice_media_stream* stream) { // Streams which do not exist should never fail. MOZ_ASSERT(s); - ctx->SetConnectionState(ICE_CTX_FAILED); + if (!ctx->dumped_rlog_) { + // Do this at most once per ctx + ctx->dumped_rlog_ = true; + MOZ_MTLOG(ML_INFO, + "NrIceCtx(" << ctx->name_ << "): dumping r_log ringbuffer... "); + std::deque<std::string> logs; + RLogConnector::GetInstance()->GetAny(0, &logs); + for (auto& log : logs) { + MOZ_MTLOG(ML_INFO, log); + } + } + s->Failed(); + ctx->SignalConnectionStateChange(s, ICE_CTX_FAILED); return 0; } -int NrIceCtx::ice_checking(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_checking called"); +int NrIceCtx::stream_checking(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_checking called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); // Get the ICE ctx NrIceCtx* ctx = static_cast<NrIceCtx*>(obj); + RefPtr<NrIceMediaStream> s = ctx->FindStream(stream); - ctx->SetConnectionState(ICE_CTX_CHECKING); + MOZ_ASSERT(s); + if (!s->AnyGenerationIsConnected()) { + // the checking state only applies if we aren't connected + ctx->SignalConnectionStateChange(s, ICE_CTX_CHECKING); + } return 0; } -int NrIceCtx::ice_connected(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_connected called"); +int NrIceCtx::stream_disconnected(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_disconnected called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); // Get the ICE ctx NrIceCtx* ctx = static_cast<NrIceCtx*>(obj); + RefPtr<NrIceMediaStream> s = ctx->FindStream(stream); - // This is called even on failed contexts. - if (ctx->connection_state() != ICE_CTX_FAILED) { - ctx->SetConnectionState(ICE_CTX_CONNECTED); - } + MOZ_ASSERT(s); + ctx->SignalConnectionStateChange(s, ICE_CTX_DISCONNECTED); return 0; } -int NrIceCtx::ice_disconnected(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_disconnected called"); +int NrIceCtx::stream_gathering(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_gathering called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); + + // Get the ICE ctx + NrIceCtx* ctx = static_cast<NrIceCtx*>(obj); + RefPtr<NrIceMediaStream> s = ctx->FindStream(stream); + + MOZ_ASSERT(s); + + s->OnGatheringStarted(stream); + return 0; +} + +int NrIceCtx::stream_gathered(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_gathered called"); + MOZ_ASSERT(!stream->local_stream); // Get the ICE ctx NrIceCtx* ctx = static_cast<NrIceCtx*>(obj); + RefPtr<NrIceMediaStream> s = ctx->FindStream(stream); - ctx->SetConnectionState(ICE_CTX_DISCONNECTED); + // We get this callback for destroyed streams in some cases + if (s) { + s->OnGatheringComplete(stream); + } + return 0; +} + +int NrIceCtx::ice_checking(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_checking called"); + // We don't use this; we react to the stream-specific callbacks instead + return 0; +} + +int NrIceCtx::ice_connected(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_connected called"); + // We don't use this; we react to the stream-specific callbacks instead + return 0; +} +int NrIceCtx::ice_disconnected(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_disconnected called"); + // We don't use this; we react to the stream-specific callbacks instead return 0; } @@ -466,7 +521,6 @@ void NrIceCtx::trickle_cb(void* arg, nr_ice_ctx* ice_ctx, } if (!candidate) { - s->SignalCandidate(s, "", stream->ufrag, "", ""); return; } @@ -587,11 +641,20 @@ void NrIceCtx::SetStunAddrs(const nsTArray<NrIceStunAddr>& addrs) { } bool NrIceCtx::Initialize() { + // Create the gather handler objects + ice_gather_handler_vtbl_ = new nr_ice_gather_handler_vtbl(); + ice_gather_handler_vtbl_->stream_gathering = &NrIceCtx::stream_gathering; + ice_gather_handler_vtbl_->stream_gathered = &NrIceCtx::stream_gathered; + ice_gather_handler_ = new nr_ice_gather_handler(); + ice_gather_handler_->vtbl = ice_gather_handler_vtbl_; + ice_gather_handler_->obj = this; + // Create the ICE context int r; UINT4 flags = NR_ICE_CTX_FLAGS_AGGRESSIVE_NOMINATION; - r = nr_ice_ctx_create(const_cast<char*>(name_.c_str()), flags, &ctx_); + r = nr_ice_ctx_create(const_cast<char*>(name_.c_str()), flags, + ice_gather_handler_, &ctx_); if (r) { MOZ_MTLOG(ML_ERROR, "Couldn't create ICE ctx for '" << name_ << "'"); @@ -634,6 +697,16 @@ bool NrIceCtx::Initialize() { ice_handler_vtbl_->select_pair = &NrIceCtx::select_pair; ice_handler_vtbl_->stream_ready = &NrIceCtx::stream_ready; ice_handler_vtbl_->stream_failed = &NrIceCtx::stream_failed; + ice_handler_vtbl_->stream_checking = &NrIceCtx::stream_checking; + ice_handler_vtbl_->stream_disconnected = &NrIceCtx::stream_disconnected; + // stream_gathering and stream_gathered do not go here, since those are tied + // to the _local_ nr_ice_media_stream in nICEr. nICEr allows a local + // nr_ice_media_stream (which has a single set of candidates, and therefore a + // single gathering state) to be associated with multiple remote + // nr_ice_media_streams (which each have their own ICE connection state) + // because it allows forking. We never encounter forking, so these will be + // one-to-one in practice, but the architecture in nICEr means we have to set + // up these callbacks on the nr_ice_ctx, not the nr_ice_peer_ctx. ice_handler_vtbl_->ice_connected = &NrIceCtx::ice_connected; ice_handler_vtbl_->msg_recvd = &NrIceCtx::msg_recvd; ice_handler_vtbl_->ice_checking = &NrIceCtx::ice_checking; @@ -720,8 +793,13 @@ NrIceStats NrIceCtx::Destroy() { delete ice_handler_vtbl_; delete ice_handler_; + delete ice_gather_handler_vtbl_; + delete ice_gather_handler_; + ice_handler_vtbl_ = nullptr; ice_handler_ = nullptr; + ice_gather_handler_vtbl_ = nullptr; + ice_gather_handler_ = nullptr; proxy_config_ = nullptr; streams_.clear(); @@ -854,15 +932,10 @@ nsresult NrIceCtx::StartGathering(bool default_route_only, // finished. int r = nr_ice_gather(ctx_, &NrIceCtx::gather_cb, this); - if (!r) { - SetGatheringState(ICE_CTX_GATHER_COMPLETE); - } else if (r == R_WOULDBLOCK) { - SetGatheringState(ICE_CTX_GATHER_STARTED); - } else { - SetGatheringState(ICE_CTX_GATHER_COMPLETE); + if (r && r != R_WOULDBLOCK) { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't gather ICE candidates for '" << name_ << "', error=" << r); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } @@ -940,7 +1013,7 @@ nsresult NrIceCtx::StartChecks() { r = nr_ice_peer_ctx_pair_candidates(peer_); if (r) { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't pair candidates on " << name_); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } @@ -952,7 +1025,7 @@ nsresult NrIceCtx::StartChecks() { } else { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't start peer checks on " << name_); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } } @@ -961,18 +1034,21 @@ nsresult NrIceCtx::StartChecks() { } void NrIceCtx::gather_cb(NR_SOCKET s, int h, void* arg) { - NrIceCtx* ctx = static_cast<NrIceCtx*>(arg); + MOZ_MTLOG(ML_DEBUG, "gather_cb called"); + // We don't use this; we react to the stream-specific callbacks instead +} - ctx->SetGatheringState(ICE_CTX_GATHER_COMPLETE); +void NrIceCtx::SignalAllStreamsFailed() { + for (auto& [id, stream] : streams_) { + Unused << id; + stream->Failed(); + SignalConnectionStateChange(stream, ICE_CTX_FAILED); + } } void NrIceCtx::UpdateNetworkState(bool online) { MOZ_MTLOG(ML_NOTICE, "NrIceCtx(" << name_ << "): updating network state to " << (online ? "online" : "offline")); - if (connection_state_ == ICE_CTX_CLOSED) { - return; - } - if (online) { nr_ice_peer_ctx_refresh_consent_all_streams(peer_); } else { @@ -980,36 +1056,6 @@ void NrIceCtx::UpdateNetworkState(bool online) { } } -void NrIceCtx::SetConnectionState(ConnectionState state) { - if (state == connection_state_) return; - - MOZ_MTLOG(ML_INFO, "NrIceCtx(" << name_ << "): state " << connection_state_ - << "->" << state); - connection_state_ = state; - - if (connection_state_ == ICE_CTX_FAILED) { - MOZ_MTLOG(ML_INFO, - "NrIceCtx(" << name_ << "): dumping r_log ringbuffer... "); - std::deque<std::string> logs; - RLogConnector::GetInstance()->GetAny(0, &logs); - for (auto& log : logs) { - MOZ_MTLOG(ML_INFO, log); - } - } - - SignalConnectionStateChange(this, state); -} - -void NrIceCtx::SetGatheringState(GatheringState state) { - if (state == gathering_state_) return; - - MOZ_MTLOG(ML_DEBUG, "NrIceCtx(" << name_ << "): gathering state " - << gathering_state_ << "->" << state); - gathering_state_ = state; - - SignalGatheringStateChange(this, state); -} - void NrIceCtx::GenerateObfuscatedAddress(nr_ice_candidate* candidate, std::string* mdns_address, std::string* actual_address) { diff --git a/dom/media/webrtc/transport/nricectx.h b/dom/media/webrtc/transport/nricectx.h index a0a0b5b772..01ad6b5dbd 100644 --- a/dom/media/webrtc/transport/nricectx.h +++ b/dom/media/webrtc/transport/nricectx.h @@ -74,6 +74,8 @@ typedef struct nr_ice_peer_ctx_ nr_ice_peer_ctx; typedef struct nr_ice_media_stream_ nr_ice_media_stream; typedef struct nr_ice_handler_ nr_ice_handler; typedef struct nr_ice_handler_vtbl_ nr_ice_handler_vtbl; +typedef struct nr_ice_gather_handler_ nr_ice_gather_handler; +typedef struct nr_ice_gather_handler_vtbl_ nr_ice_gather_handler_vtbl; typedef struct nr_ice_candidate_ nr_ice_candidate; typedef struct nr_ice_cand_pair_ nr_ice_cand_pair; typedef struct nr_ice_stun_server_ nr_ice_stun_server; @@ -200,12 +202,6 @@ class NrIceCtx { ICE_CTX_CLOSED }; - enum GatheringState { - ICE_CTX_GATHER_INIT, - ICE_CTX_GATHER_STARTED, - ICE_CTX_GATHER_COMPLETE - }; - enum Controlling { ICE_CONTROLLING, ICE_CONTROLLED }; enum Policy { ICE_POLICY_RELAY, ICE_POLICY_NO_HOST, ICE_POLICY_ALL }; @@ -294,12 +290,6 @@ class NrIceCtx { // The name of the ctx const std::string& name() const { return name_; } - // Current state - ConnectionState connection_state() const { return connection_state_; } - - // Current state - GatheringState gathering_state() const { return gathering_state_; } - // Get the global attributes std::vector<std::string> GetGlobalAttributes(); @@ -351,9 +341,7 @@ class NrIceCtx { // Signals to indicate events. API users can (and should) // register for these. - sigslot::signal2<NrIceCtx*, NrIceCtx::GatheringState> - SignalGatheringStateChange; - sigslot::signal2<NrIceCtx*, NrIceCtx::ConnectionState> + sigslot::signal2<NrIceMediaStream*, NrIceCtx::ConnectionState> SignalConnectionStateChange; // The thread to direct method calls to @@ -375,7 +363,11 @@ class NrIceCtx { static int select_pair(void* obj, nr_ice_media_stream* stream, int component_id, nr_ice_cand_pair** potentials, int potential_ct); + static int stream_gathering(void* obj, nr_ice_media_stream* stream); + static int stream_gathered(void* obj, nr_ice_media_stream* stream); + static int stream_checking(void* obj, nr_ice_media_stream* stream); static int stream_ready(void* obj, nr_ice_media_stream* stream); + static int stream_disconnected(void* obj, nr_ice_media_stream* stream); static int stream_failed(void* obj, nr_ice_media_stream* stream); static int ice_checking(void* obj, nr_ice_peer_ctx* pctx); static int ice_connected(void* obj, nr_ice_peer_ctx* pctx); @@ -387,28 +379,25 @@ class NrIceCtx { nr_ice_media_stream* stream, int component_id, nr_ice_candidate* candidate); + void SignalAllStreamsFailed(); + // Find a media stream by stream ptr. Gross RefPtr<NrIceMediaStream> FindStream(nr_ice_media_stream* stream); - // Set the state - void SetConnectionState(ConnectionState state); - - // Set the state - void SetGatheringState(GatheringState state); - void GenerateObfuscatedAddress(nr_ice_candidate* candidate, std::string* mdns_address, std::string* actual_address); - ConnectionState connection_state_; - GatheringState gathering_state_; + bool dumped_rlog_ = false; const std::string name_; bool ice_controlling_set_; std::map<std::string, RefPtr<NrIceMediaStream>> streams_; nr_ice_ctx* ctx_; nr_ice_peer_ctx* peer_; - nr_ice_handler_vtbl* ice_handler_vtbl_; // Must be pointer - nr_ice_handler* ice_handler_; // Must be pointer + nr_ice_handler_vtbl* ice_handler_vtbl_; // Must be pointer + nr_ice_handler* ice_handler_; // Must be pointer + nr_ice_gather_handler_vtbl* ice_gather_handler_vtbl_; // Must be pointer + nr_ice_gather_handler* ice_gather_handler_; // Must be pointer bool trickle_; nsCOMPtr<nsIEventTarget> sts_target_; // The thread to run on Config config_; diff --git a/dom/media/webrtc/transport/nricemediastream.cpp b/dom/media/webrtc/transport/nricemediastream.cpp index 426aee230e..a3a0c147c9 100644 --- a/dom/media/webrtc/transport/nricemediastream.cpp +++ b/dom/media/webrtc/transport/nricemediastream.cpp @@ -204,11 +204,17 @@ nsresult NrIceMediaStream::ConnectToPeer( MOZ_ASSERT(stream_); if (Matches(old_stream_, ufrag, pwd)) { + bool wasGathering = !AllGenerationsDoneGathering(); // (We swap before we close so we never have stream_ == nullptr) MOZ_MTLOG(ML_DEBUG, "Rolling back to old stream ufrag=" << ufrag << " " << name_); std::swap(stream_, old_stream_); CloseStream(&old_stream_); + if (wasGathering && AllGenerationsDoneGathering()) { + // Special case; we do not need to send another empty candidate, but we + // do need to handle the transition from gathering to complete. + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_COMPLETE); + } } else if (old_stream_) { // Right now we wait for ICE to complete before closing the old stream. // It might be worth it to close it sooner, but we don't want to close it @@ -273,6 +279,10 @@ nsresult NrIceMediaStream::SetIceCredentials(const std::string& ufrag, } state_ = ICE_CONNECTING; + + MOZ_MTLOG(ML_WARNING, + "SetIceCredentials new=" << stream_ << " old=" << old_stream_); + return NS_OK; } @@ -661,6 +671,21 @@ void NrIceMediaStream::Failed() { } } +void NrIceMediaStream::OnGatheringStarted(nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_WARNING, "OnGatheringStarted called for " << stream); + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_STARTED); +} + +void NrIceMediaStream::OnGatheringComplete(nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_WARNING, "OnGatheringComplete called for " << stream); + // Spec says to queue two separate tasks; one for the empty candidate, and + // the next for the state change. + SignalCandidate(this, "", stream->ufrag, "", ""); + if (AllGenerationsDoneGathering()) { + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_COMPLETE); + } +} + void NrIceMediaStream::Close() { MOZ_MTLOG(ML_DEBUG, "Marking stream closed '" << name_ << "'"); state_ = ICE_CLOSED; @@ -709,4 +734,34 @@ nr_ice_media_stream* NrIceMediaStream::GetStreamForRemoteUfrag( return nullptr; } +bool NrIceMediaStream::AllGenerationsDoneGathering() const { + if (stream_ && !nr_ice_media_stream_is_done_gathering(stream_)) { + return false; + } + if (old_stream_ && !nr_ice_media_stream_is_done_gathering(old_stream_)) { + return false; + } + return true; +} + +bool NrIceMediaStream::AnyGenerationIsConnected() const { + nr_ice_media_stream* peer_stream = nullptr; + + if (stream_ && + !nr_ice_peer_ctx_find_pstream(ctx_->peer(), stream_, &peer_stream)) { + if (peer_stream->ice_state == NR_ICE_MEDIA_STREAM_CHECKS_CONNECTED && + !peer_stream->disconnected) { + return true; + } + } + + if (old_stream_ && + !nr_ice_peer_ctx_find_pstream(ctx_->peer(), old_stream_, &peer_stream)) { + if (peer_stream->ice_state == NR_ICE_MEDIA_STREAM_CHECKS_CONNECTED && + !peer_stream->disconnected) { + return true; + } + } + return false; +} } // namespace mozilla diff --git a/dom/media/webrtc/transport/nricemediastream.h b/dom/media/webrtc/transport/nricemediastream.h index 406373573c..d434dd34dd 100644 --- a/dom/media/webrtc/transport/nricemediastream.h +++ b/dom/media/webrtc/transport/nricemediastream.h @@ -126,6 +126,12 @@ struct NrIceCandidatePair { class NrIceMediaStream { public: + enum GatheringState { + ICE_STREAM_GATHER_INIT, + ICE_STREAM_GATHER_STARTED, + ICE_STREAM_GATHER_COMPLETE + }; + NrIceMediaStream(NrIceCtx* ctx, const std::string& id, const std::string& name, size_t components); @@ -182,6 +188,9 @@ class NrIceMediaStream { void Ready(nr_ice_media_stream* stream); void Failed(); + void OnGatheringStarted(nr_ice_media_stream* stream); + void OnGatheringComplete(nr_ice_media_stream* stream); + // Close the stream. Called by the NrIceCtx. // Different from the destructor because other people // might be holding RefPtrs but we want those writes to fail once @@ -192,9 +201,14 @@ class NrIceMediaStream { // the candidate belongs to. const std::string& GetId() const { return id_; } + bool AllGenerationsDoneGathering() const; + bool AnyGenerationIsConnected() const; + sigslot::signal5<NrIceMediaStream*, const std::string&, const std::string&, const std::string&, const std::string&> SignalCandidate; // A new ICE candidate: + sigslot::signal2<const std::string&, NrIceMediaStream::GatheringState> + SignalGatheringStateChange; sigslot::signal1<NrIceMediaStream*> SignalReady; // Candidate pair ready. sigslot::signal1<NrIceMediaStream*> SignalFailed; // Candidate pair failed. diff --git a/dom/media/webrtc/transport/test/ice_unittest.cpp b/dom/media/webrtc/transport/test/ice_unittest.cpp index 4d097fafa3..50febb3cdd 100644 --- a/dom/media/webrtc/transport/test/ice_unittest.cpp +++ b/dom/media/webrtc/transport/test/ice_unittest.cpp @@ -304,6 +304,8 @@ class SchedulableTrickleCandidate { } void Schedule(unsigned int ms) { + std::cerr << "Scheduling " << Candidate() << " in " << ms << "ms" + << std::endl; test_utils_->SyncDispatchToSTS( WrapRunnable(this, &SchedulableTrickleCandidate::Schedule_s, ms)); } @@ -355,10 +357,7 @@ class IceTestPeer : public sigslot::has_slots<> { offerer_(offerer), stream_counter_(0), shutting_down_(false), - gathering_complete_(false), ready_ct_(0), - ice_connected_(false), - ice_failed_(false), ice_reached_checking_(false), received_(0), sent_(0), @@ -372,8 +371,6 @@ class IceTestPeer : public sigslot::has_slots<> { simulate_ice_lite_(false), nat_(new TestNat), test_utils_(utils) { - ice_ctx_->SignalGatheringStateChange.connect( - this, &IceTestPeer::GatheringStateChange); ice_ctx_->SignalConnectionStateChange.connect( this, &IceTestPeer::ConnectionStateChange); @@ -426,6 +423,10 @@ class IceTestPeer : public sigslot::has_slots<> { stream->SignalReady.connect(this, &IceTestPeer::StreamReady); stream->SignalFailed.connect(this, &IceTestPeer::StreamFailed); stream->SignalPacketReceived.connect(this, &IceTestPeer::PacketReceived); + stream->SignalGatheringStateChange.connect( + this, &IceTestPeer::GatheringStateChange); + mConnectionStates[id] = NrIceCtx::ICE_CTX_INIT; + mGatheringStates[id] = NrIceMediaStream::ICE_STREAM_GATHER_INIT; } void AddStream(int components) { @@ -434,7 +435,10 @@ class IceTestPeer : public sigslot::has_slots<> { } void RemoveStream_s(size_t index) { - ice_ctx_->DestroyStream(MakeTransportId(index)); + const std::string id = MakeTransportId(index); + ice_ctx_->DestroyStream(id); + mConnectionStates.erase(id); + mGatheringStates.erase(id); } void RemoveStream(size_t index) { @@ -650,7 +654,15 @@ class IceTestPeer : public sigslot::has_slots<> { return host_net; } - bool gathering_complete() { return gathering_complete_; } + bool gathering_complete() { + for (const auto& [id, state] : mGatheringStates) { + Unused << id; + if (state != NrIceMediaStream::ICE_STREAM_GATHER_COMPLETE) { + return false; + } + } + return true; + } int ready_ct() { return ready_ct_; } bool is_ready_s(size_t index) { auto media_stream = GetStream_s(index); @@ -666,8 +678,33 @@ class IceTestPeer : public sigslot::has_slots<> { WrapRunnableRet(&result, this, &IceTestPeer::is_ready_s, stream)); return result; } - bool ice_connected() { return ice_connected_; } - bool ice_failed() { return ice_failed_; } + bool ice_connected() { + for (const auto& [id, state] : mConnectionStates) { + if (state != NrIceCtx::ICE_CTX_CONNECTED) { + return false; + } + } + return true; + } + bool ice_failed() { + for (const auto& [id, state] : mConnectionStates) { + if (state == NrIceCtx::ICE_CTX_FAILED) { + return true; + } + } + return false; + } + bool ice_checking() { + if (ice_failed() || ice_connected()) { + return false; + } + for (const auto& [id, state] : mConnectionStates) { + if (state == NrIceCtx::ICE_CTX_CHECKING) { + return true; + } + } + return false; + } bool ice_reached_checking() { return ice_reached_checking_; } size_t received() { return received_; } size_t sent() { return sent_; } @@ -680,13 +717,16 @@ class IceTestPeer : public sigslot::has_slots<> { void RestartIce_s() { for (auto& stream : ice_ctx_->GetStreams()) { SetIceCredentials_s(*stream); + mConnectionStates[stream->GetId()] = NrIceCtx::ICE_CTX_INIT; + mGatheringStates[stream->GetId()] = + NrIceMediaStream::ICE_STREAM_GATHER_INIT; } // take care of some local bookkeeping ready_ct_ = 0; - gathering_complete_ = false; - ice_connected_ = false; - ice_failed_ = false; - ice_reached_checking_ = false; + // We do not unset ice_reached_checking_ here, since we do not expect + // ICE to return to checking in an ICE restart, because the ICE stack + // continues using the old streams (which are probably connected) until the + // new ones are connected. remote_ = nullptr; } @@ -709,9 +749,6 @@ class IceTestPeer : public sigslot::has_slots<> { remote_ = remote; trickle_mode_ = trickle_mode; - ice_connected_ = false; - ice_failed_ = false; - ice_reached_checking_ = false; res = ice_ctx_->ParseGlobalAttributes(remote->GetGlobalAttributes()); ASSERT_FALSE(remote->simulate_ice_lite_ && (ice_ctx_->GetControlling() == NrIceCtx::ICE_CONTROLLED)); @@ -793,8 +830,11 @@ class IceTestPeer : public sigslot::has_slots<> { auto stream = GetStream_s(index); if (!stream) { // stream might have gone away before the trickle timer popped + std::cerr << "Trickle candidate has no stream: " << index << std::endl; return NS_OK; } + std::cerr << "Trickle candidate for " << index << " (" << stream->GetId() + << "):" << candidate << std::endl; return stream->ParseTrickleCandidate(candidate, ufrag, ""); } @@ -940,16 +980,18 @@ class IceTestPeer : public sigslot::has_slots<> { } // Handle events - void GatheringStateChange(NrIceCtx* ctx, NrIceCtx::GatheringState state) { + void GatheringStateChange(const std::string& aTransportId, + NrIceMediaStream::GatheringState state) { if (shutting_down_) { return; } - if (state != NrIceCtx::ICE_CTX_GATHER_COMPLETE) { + mGatheringStates[aTransportId] = state; + + if (!gathering_complete()) { return; } std::cerr << name_ << " Gathering complete" << std::endl; - gathering_complete_ = true; std::cerr << name_ << " ATTRIBUTES:" << std::endl; for (const auto& stream : ice_ctx_->GetStreams()) { @@ -973,9 +1015,9 @@ class IceTestPeer : public sigslot::has_slots<> { if (candidate.empty()) { return; } - std::cerr << "Candidate for stream " << stream->name() + std::cerr << "Candidate for stream " << stream->GetId() << " initialized: " << candidate << std::endl; - candidates_[stream->name()].push_back(candidate); + candidates_[stream->GetId()].push_back(candidate); // If we are connected, then try to trickle to the other side. if (remote_ && remote_->remote_ && (trickle_mode_ != TRICKLE_SIMULATE)) { @@ -990,7 +1032,7 @@ class IceTestPeer : public sigslot::has_slots<> { return; } } - ADD_FAILURE() << "No matching stream found for " << stream; + ADD_FAILURE() << "No matching stream found for " << stream->GetId(); } } @@ -1133,32 +1175,45 @@ class IceTestPeer : public sigslot::has_slots<> { DumpCandidatePairs_s(stream); } - void ConnectionStateChange(NrIceCtx* ctx, NrIceCtx::ConnectionState state) { - (void)ctx; + void ConnectionStateChange(NrIceMediaStream* stream, + NrIceCtx::ConnectionState state) { + mConnectionStates[stream->GetId()] = state; + if (ice_checking()) { + ice_reached_checking_ = true; + } + switch (state) { case NrIceCtx::ICE_CTX_INIT: break; case NrIceCtx::ICE_CTX_CHECKING: - std::cerr << name_ << " ICE reached checking" << std::endl; - ice_reached_checking_ = true; + std::cerr << name_ << " ICE reached checking (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_CONNECTED: - std::cerr << name_ << " ICE connected" << std::endl; - ice_connected_ = true; + std::cerr << name_ << " ICE reached connected (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_COMPLETED: - std::cerr << name_ << " ICE completed" << std::endl; + std::cerr << name_ << " ICE reached completed (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_FAILED: - std::cerr << name_ << " ICE failed" << std::endl; - ice_failed_ = true; + std::cerr << name_ << " ICE reached failed (" << stream->GetId() << ")" + << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_DISCONNECTED: - std::cerr << name_ << " ICE disconnected" << std::endl; - ice_connected_ = false; + std::cerr << name_ << " ICE reached disconnected (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); + break; + case NrIceCtx::ICE_CTX_CLOSED: + std::cerr << name_ << " ICE reached closed (" << stream->GetId() << ")" + << std::endl; break; - default: - MOZ_CRASH(); } } @@ -1326,10 +1381,9 @@ class IceTestPeer : public sigslot::has_slots<> { std::map<std::string, std::pair<std::string, std::string>> mOldIceCredentials; size_t stream_counter_; bool shutting_down_; - bool gathering_complete_; + std::map<std::string, NrIceCtx::ConnectionState> mConnectionStates; + std::map<std::string, NrIceMediaStream::GatheringState> mGatheringStates; int ready_ct_; - bool ice_connected_; - bool ice_failed_; bool ice_reached_checking_; size_t received_; size_t sent_; @@ -1686,10 +1740,8 @@ class WebRtcIceConnectTest : public StunTest { TrickleMode mode = TRICKLE_NONE) { ASSERT_TRUE(caller->ready_ct() == 0); ASSERT_TRUE(caller->ice_connected() == 0); - ASSERT_TRUE(caller->ice_reached_checking() == 0); ASSERT_TRUE(callee->ready_ct() == 0); ASSERT_TRUE(callee->ice_connected() == 0); - ASSERT_TRUE(callee->ice_reached_checking() == 0); // IceTestPeer::Connect grabs attributes from the first arg, and // gives them to |this|, meaning that callee->Connect(caller, ...) @@ -3361,6 +3413,8 @@ TEST_F(WebRtcIceConnectTest, TestConnectTrickleAddStreamDuringICE) { RealisticTrickleDelay(p1_->ControlTrickle(0)); RealisticTrickleDelay(p2_->ControlTrickle(0)); AddStream(1); + ASSERT_TRUE(Gather()); + ConnectTrickle(); RealisticTrickleDelay(p1_->ControlTrickle(1)); RealisticTrickleDelay(p2_->ControlTrickle(1)); WaitForConnected(1000); diff --git a/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp b/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp index b55b05f10c..20754f033b 100644 --- a/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp +++ b/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp @@ -73,7 +73,8 @@ class IcePeer { peer_ctx_(nullptr), nat_(nat), test_utils_(test_utils) { - nr_ice_ctx_create(const_cast<char*>(name_.c_str()), flags, &ice_ctx_); + nr_ice_ctx_create(const_cast<char*>(name_.c_str()), flags, nullptr, + &ice_ctx_); if (nat_) { nr_socket_factory* factory; diff --git a/dom/media/webrtc/transport/test/transport_unittests.cpp b/dom/media/webrtc/transport/test/transport_unittests.cpp index 7729151ade..0e9702ced8 100644 --- a/dom/media/webrtc/transport/test/transport_unittests.cpp +++ b/dom/media/webrtc/transport/test/transport_unittests.cpp @@ -586,16 +586,15 @@ class TransportTestPeer : public sigslot::has_slots<> { void InitIce() { nsresult res; - // Attach our slots - ice_ctx_->SignalGatheringStateChange.connect( - this, &TransportTestPeer::GatheringStateChange); - char name[100]; snprintf(name, sizeof(name), "%s:stream%d", name_.c_str(), (int)streams_.size()); // Create the media stream RefPtr<NrIceMediaStream> stream = ice_ctx_->CreateStream(name, name, 1); + // Attach our slots + stream->SignalGatheringStateChange.connect( + this, &TransportTestPeer::GatheringStateChange); ASSERT_TRUE(stream != nullptr); stream->SetIceCredentials("ufrag", "pass"); @@ -639,9 +638,12 @@ class TransportTestPeer : public sigslot::has_slots<> { << std::endl; } - void GatheringStateChange(NrIceCtx* ctx, NrIceCtx::GatheringState state) { - (void)ctx; - if (state == NrIceCtx::ICE_CTX_GATHER_COMPLETE) { + void GatheringStateChange(const std::string& aTransportId, + NrIceMediaStream::GatheringState state) { + // We only use one stream, no need to check whether all streams are done + // gathering. + Unused << aTransportId; + if (state == NrIceMediaStream::ICE_STREAM_GATHER_COMPLETE) { GatheringComplete(); } } diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c index 0d498845a4..b428264e5a 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c @@ -325,8 +325,9 @@ int nr_ice_fetch_turn_servers(int ct, nr_ice_turn_server **out) #endif /* USE_TURN */ #define MAXADDRS 100 /* Ridiculously high */ -int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) - { + int nr_ice_ctx_create(char* label, UINT4 flags, + nr_ice_gather_handler* gather_handler, + nr_ice_ctx** ctxp) { nr_ice_ctx *ctx=0; int r,_status; @@ -341,6 +342,8 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) if(!(ctx->label=r_strdup(label))) ABORT(R_NO_MEMORY); + ctx->gather_handler = gather_handler; + /* Get the STUN servers */ if(r=NR_reg_get_child_count(NR_ICE_REG_STUN_SRV_PRFX, (unsigned int *)&ctx->stun_server_ct_cfg)||ctx->stun_server_ct_cfg==0) { @@ -442,7 +445,7 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) int i; nr_ice_stun_id *id1,*id2; - ctx->done_cb = 0; + ctx->gather_done_cb = 0; ctx->trickle_cb = 0; STAILQ_FOREACH_SAFE(s1, &ctx->streams, entry, s2){ @@ -452,6 +455,8 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) RFREE(ctx->label); + ctx->gather_handler = 0; + RFREE(ctx->stun_servers_cfg); RFREE(ctx->local_addrs); @@ -539,20 +544,26 @@ void nr_ice_gather_finished_cb(NR_SOCKET s, int h, void *cb_arg) } } - if (nr_ice_media_stream_is_done_gathering(stream) && - ctx->trickle_cb) { - ctx->trickle_cb(ctx->trickle_cb_arg, ctx, stream, component_id, NULL); + if (nr_ice_media_stream_is_done_gathering(stream)) { + if (ctx->gather_handler && ctx->gather_handler->vtbl->stream_gathered) { + ctx->gather_handler->vtbl->stream_gathered(ctx->gather_handler->obj, + stream); + } + if (ctx->trickle_cb) { + ctx->trickle_cb(ctx->trickle_cb_arg, ctx, stream, component_id, NULL); + } } if(ctx->uninitialized_candidates==0){ + assert(nr_ice_media_stream_is_done_gathering(stream)); r_log(LOG_ICE, LOG_INFO, "ICE(%s): All candidates initialized", ctx->label); - if (ctx->done_cb) { - ctx->done_cb(0,0,ctx->cb_arg); - } - else { + if (ctx->gather_done_cb) { + ctx->gather_done_cb(0, 0, ctx->cb_arg); + } else { r_log(LOG_ICE, LOG_INFO, - "ICE(%s): No done_cb. We were probably destroyed.", ctx->label); + "ICE(%s): No gather_done_cb. We were probably destroyed.", + ctx->label); } } else { @@ -850,8 +861,7 @@ int nr_ice_set_target_for_default_local_address_lookup(nr_ice_ctx *ctx, const ch return(_status); } -int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg) - { + int nr_ice_gather(nr_ice_ctx* ctx, NR_async_cb gather_done_cb, void* cb_arg) { int r,_status; nr_ice_media_stream *stream; nr_local_addr stun_addrs[MAXADDRS]; @@ -872,7 +882,7 @@ int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg) } r_log(LOG_ICE,LOG_DEBUG,"ICE(%s): Initializing candidates",ctx->label); - ctx->done_cb=done_cb; + ctx->gather_done_cb = gather_done_cb; ctx->cb_arg=cb_arg; /* Initialize all the media stream/component pairs */ diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h index 8b3081f567..4039c741ec 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h @@ -97,9 +97,23 @@ typedef struct nr_ice_stats_ { UINT2 turn_438s; } nr_ice_stats; +typedef struct nr_ice_gather_handler_vtbl_ { + /* This media stream is gathering */ + int (*stream_gathering)(void* obj, nr_ice_media_stream* stream); + + /* This media stream has finished gathering */ + int (*stream_gathered)(void* obj, nr_ice_media_stream* stream); +} nr_ice_gather_handler_vtbl; + +typedef struct nr_ice_gather_handler_ { + void* obj; + nr_ice_gather_handler_vtbl* vtbl; +} nr_ice_gather_handler; + struct nr_ice_ctx_ { UINT4 flags; char *label; + nr_ice_gather_handler* gather_handler; UINT4 Ta; @@ -129,7 +143,7 @@ struct nr_ice_ctx_ { nr_ice_peer_ctx_head peers; nr_ice_stun_id_head ids; - NR_async_cb done_cb; + NR_async_cb gather_done_cb; void *cb_arg; nr_ice_trickle_candidate_cb trickle_cb; @@ -141,7 +155,8 @@ struct nr_ice_ctx_ { nr_transport_addr *target_for_default_local_address_lookup; }; -int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp); +int nr_ice_ctx_create(char* label, UINT4 flags, + nr_ice_gather_handler* gather_handler, nr_ice_ctx** ctxp); int nr_ice_ctx_create_with_credentials(char *label, UINT4 flags, char* ufrag, char* pwd, nr_ice_ctx **ctxp); #define NR_ICE_CTX_FLAGS_AGGRESSIVE_NOMINATION (1) #define NR_ICE_CTX_FLAGS_LITE (1<<1) @@ -156,7 +171,7 @@ void nr_ice_ctx_remove_flags(nr_ice_ctx *ctx, UINT4 flags); void nr_ice_ctx_destroy(nr_ice_ctx** ctxp); int nr_ice_set_local_addresses(nr_ice_ctx *ctx, nr_local_addr* stun_addrs, int stun_addr_ct); int nr_ice_set_target_for_default_local_address_lookup(nr_ice_ctx *ctx, const char *target_ip, UINT2 target_port); -int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg); +int nr_ice_gather(nr_ice_ctx* ctx, NR_async_cb gather_done_cb, void* cb_arg); int nr_ice_add_candidate(nr_ice_ctx *ctx, nr_ice_candidate *cand); void nr_ice_gather_finished_cb(NR_SOCKET s, int h, void *cb_arg); int nr_ice_add_media_stream(nr_ice_ctx *ctx,const char *label,const char *ufrag,const char *pwd,int components, nr_ice_media_stream **streamp); diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h index 5a0690adad..ab3e41ef2d 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h @@ -56,9 +56,15 @@ int component_id, nr_ice_cand_pair **potentials,int potential_ct); */ int (*stream_ready)(void *obj, nr_ice_media_stream *stream); + /* This media stream is checking */ + int (*stream_checking)(void* obj, nr_ice_media_stream* stream); + /* This media stream has failed */ int (*stream_failed)(void *obj, nr_ice_media_stream *stream); + /* This media stream has disconnected */ + int (*stream_disconnected)(void* obj, nr_ice_media_stream* stream); + /* ICE is connected for this peer ctx */ int (*ice_connected)(void *obj, nr_ice_peer_ctx *pctx); diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c index 62bfbad629..90e278bedb 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c @@ -80,6 +80,7 @@ int nr_ice_media_stream_create(nr_ice_ctx *ctx,const char *label,const char *ufr stream->component_ct=components; stream->ice_state = NR_ICE_MEDIA_STREAM_UNPAIRED; stream->obsolete = 0; + stream->actually_started_checking = 0; stream->r2l_user = 0; stream->l2r_user = 0; stream->flags = ctx->flags; @@ -177,8 +178,20 @@ int nr_ice_media_stream_initialize(nr_ice_ctx *ctx, nr_ice_media_stream *stream) comp=STAILQ_NEXT(comp,entry); } + if (!nr_ice_media_stream_is_done_gathering(stream) && ctx->gather_handler && + ctx->gather_handler->vtbl->stream_gathering) { + ctx->gather_handler->vtbl->stream_gathering(ctx->gather_handler->obj, + stream); + } + _status=0; abort: + if (_status) { + if (ctx->gather_handler && ctx->gather_handler->vtbl->stream_gathered) { + ctx->gather_handler->vtbl->stream_gathered(ctx->gather_handler->obj, + stream); + } + } return(_status); } @@ -413,6 +426,19 @@ static void nr_ice_media_stream_check_timer_cb(NR_SOCKET s, int h, void *cb_arg) if(pair){ nr_ice_candidate_pair_start(pair->pctx,pair); /* Ignore failures */ + + /* stream->ice_state goes to checking when we decide that it is ok to + * start checking, which can happen before we get remote candidates. We + * want to fire this event when we _actually_ start sending checks. */ + if (!stream->actually_started_checking) { + stream->actually_started_checking = 1; + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_checking) { + stream->pctx->handler->vtbl->stream_checking( + stream->pctx->handler->obj, stream->local_stream); + } + } + NR_ASYNC_TIMER_SET(timer_val,nr_ice_media_stream_check_timer_cb,cb_arg,&stream->timer); } else { @@ -729,9 +755,21 @@ void nr_ice_media_stream_set_disconnected(nr_ice_media_stream *stream, int disco if (disconnected == NR_ICE_MEDIA_STREAM_DISCONNECTED) { if (!stream->local_stream->obsolete) { + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_disconnected) { + stream->pctx->handler->vtbl->stream_disconnected( + stream->pctx->handler->obj, stream->local_stream); + } nr_ice_peer_ctx_disconnected(stream->pctx); } } else { + if (!stream->local_stream->obsolete) { + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_ready) { + stream->pctx->handler->vtbl->stream_ready(stream->pctx->handler->obj, + stream->local_stream); + } + } nr_ice_peer_ctx_check_if_connected(stream->pctx); } } diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h index 99f906c100..3da20e3b5e 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h @@ -72,6 +72,7 @@ struct nr_ice_media_stream_ { * processing. If this stream is connected already, traffic can continue to * flow for a limited time while the new stream gets ready. */ int obsolete; + int actually_started_checking; #define NR_ICE_MEDIA_STREAM_UNPAIRED 1 #define NR_ICE_MEDIA_STREAM_CHECKS_FROZEN 2 diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c index 0bf97eb984..bfa1fcf44b 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c @@ -671,6 +671,12 @@ void nr_ice_peer_ctx_refresh_consent_all_streams(nr_ice_peer_ctx *pctx) void nr_ice_peer_ctx_disconnected(nr_ice_peer_ctx *pctx) { + if (pctx->connected_cb_timer) { + /* Whoops, never mind */ + NR_async_timer_cancel(pctx->connected_cb_timer); + pctx->connected_cb_timer = 0; + } + if (pctx->reported_connected && pctx->handler && pctx->handler->vtbl->ice_disconnected) { diff --git a/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp b/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp index fc1a51dd0d..275e1bf45a 100644 --- a/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp +++ b/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp @@ -85,23 +85,27 @@ bool MediaPipelineFilter::Filter(const webrtc::RTPHeader& header) { // Remote SSRC based filtering // - if (remote_ssrc_set_.count(header.ssrc)) { + if (!remote_ssrc_set_.empty()) { + if (remote_ssrc_set_.count(header.ssrc)) { + DEBUG_LOG( + ("MediaPipelineFilter SSRC: %u matched remote SSRC set." + " passing packet", + header.ssrc)); + return true; + } DEBUG_LOG( - ("MediaPipelineFilter SSRC: %u matched remote SSRC set." - " passing packet", - header.ssrc)); - return true; + ("MediaPipelineFilter SSRC: %u did not match any of %zu" + " remote SSRCS.", + header.ssrc, remote_ssrc_set_.size())); + return false; } - DEBUG_LOG( - ("MediaPipelineFilter SSRC: %u did not match any of %zu" - " remote SSRCS.", - header.ssrc, remote_ssrc_set_.size())); // - // PT, payload type, last ditch effort filtering + // PT, payload type, last ditch effort filtering. We only try this if we do + // not have any ssrcs configured (either by learning them, or negotiation). // - if (payload_type_set_.count(header.payloadType)) { + if (receive_payload_type_set_.count(header.payloadType)) { DEBUG_LOG( ("MediaPipelineFilter payload-type: %u matched %zu" " unique payload type. learning ssrc. passing packet", @@ -114,7 +118,7 @@ bool MediaPipelineFilter::Filter(const webrtc::RTPHeader& header) { DEBUG_LOG( ("MediaPipelineFilter payload-type: %u did not match any of %zu" " unique payload-types.", - header.payloadType, payload_type_set_.size())); + header.payloadType, receive_payload_type_set_.size())); DEBUG_LOG( ("MediaPipelineFilter packet failed to match any criteria." " ignoring packet")); @@ -125,8 +129,8 @@ void MediaPipelineFilter::AddRemoteSSRC(uint32_t ssrc) { remote_ssrc_set_.insert(ssrc); } -void MediaPipelineFilter::AddUniquePT(uint8_t payload_type) { - payload_type_set_.insert(payload_type); +void MediaPipelineFilter::AddUniqueReceivePT(uint8_t payload_type) { + receive_payload_type_set_.insert(payload_type); } void MediaPipelineFilter::Update(const MediaPipelineFilter& filter_update) { @@ -143,7 +147,7 @@ void MediaPipelineFilter::Update(const MediaPipelineFilter& filter_update) { mRemoteMid = filter_update.mRemoteMid; mRemoteMidBindings = filter_update.mRemoteMidBindings; } - payload_type_set_ = filter_update.payload_type_set_; + receive_payload_type_set_ = filter_update.receive_payload_type_set_; // Use extmapping from new filter mExtMap = filter_update.mExtMap; diff --git a/dom/media/webrtc/transportbridge/MediaPipelineFilter.h b/dom/media/webrtc/transportbridge/MediaPipelineFilter.h index d6bb7abd43..62d4b63ab8 100644 --- a/dom/media/webrtc/transportbridge/MediaPipelineFilter.h +++ b/dom/media/webrtc/transportbridge/MediaPipelineFilter.h @@ -66,7 +66,7 @@ class MediaPipelineFilter { void SetRemoteMediaStreamId(const Maybe<std::string>& aMid); // When a payload type id is unique to our media section, add it here. - void AddUniquePT(uint8_t payload_type); + void AddUniqueReceivePT(uint8_t payload_type); void Update(const MediaPipelineFilter& filter_update); @@ -76,7 +76,7 @@ class MediaPipelineFilter { // The number of filters we manage here is quite small, so I am optimizing // for readability. std::set<uint32_t> remote_ssrc_set_; - std::set<uint8_t> payload_type_set_; + std::set<uint8_t> receive_payload_type_set_; Maybe<std::string> mRemoteMid; std::set<uint32_t> mRemoteMidBindings; // RID extension can be set by tests and is sticky, the rest of diff --git a/dom/media/webspeech/recognition/SpeechRecognition.cpp b/dom/media/webspeech/recognition/SpeechRecognition.cpp index 75d1ba7709..7239a88237 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.cpp +++ b/dom/media/webspeech/recognition/SpeechRecognition.cpp @@ -437,12 +437,13 @@ uint32_t SpeechRecognition::ProcessAudioSegment(AudioSegment* aSegment, // we need to call the nsISpeechRecognitionService::ProcessAudioSegment // in a separate thread so that any eventual encoding or pre-processing // of the audio does not block the main thread - nsresult rv = mEncodeTaskQueue->Dispatch( - NewRunnableMethod<StoreCopyPassByPtr<AudioSegment>, TrackRate>( - "nsISpeechRecognitionService::ProcessAudioSegment", - mRecognitionService, - &nsISpeechRecognitionService::ProcessAudioSegment, - std::move(*aSegment), aTrackRate)); + nsresult rv = mEncodeTaskQueue->Dispatch(NS_NewRunnableFunction( + "nsISpeechRecognitionService::ProcessAudioSegment", + [=, service = mRecognitionService, + segment = std::move(*aSegment)]() mutable { + service->ProcessAudioSegment(&segment, aTrackRate); + })); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; return samples; diff --git a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl index 1898bf68c1..8192eff045 100644 --- a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl +++ b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl @@ -65,18 +65,6 @@ interface nsISynthVoiceRegistry : nsISupports }; %{C++ -#define NS_SYNTHVOICEREGISTRY_CID \ - { /* {7090524d-5574-4492-a77f-d8d558ced59d} */ \ - 0x7090524d, \ - 0x5574, \ - 0x4492, \ - { 0xa7, 0x7f, 0xd8, 0xd5, 0x58, 0xce, 0xd5, 0x9d } \ - } - #define NS_SYNTHVOICEREGISTRY_CONTRACTID \ "@mozilla.org/synth-voice-registry;1" - -#define NS_SYNTHVOICEREGISTRY_CLASSNAME \ - "Speech Synthesis Voice Registry" - %} diff --git a/dom/media/webvtt/WebVTTListener.cpp b/dom/media/webvtt/WebVTTListener.cpp index 3f8d99a8f3..b0691491e3 100644 --- a/dom/media/webvtt/WebVTTListener.cpp +++ b/dom/media/webvtt/WebVTTListener.cpp @@ -65,7 +65,7 @@ nsresult WebVTTListener::LoadResource() { if (IsCanceled()) { return NS_OK; } - // Exit if we failed to create the WebVTTParserWrapper (vtt.jsm) + // Exit if we failed to create the WebVTTParserWrapper (vtt.sys.mjs) NS_ENSURE_SUCCESS(mParserWrapperError, mParserWrapperError); mElement->SetReadyState(TextTrackReadyState::Loading); diff --git a/dom/media/webvtt/test/mochitest/manifest.js b/dom/media/webvtt/test/mochitest/manifest.js index 91c481feb9..dc72b55ffa 100644 --- a/dom/media/webvtt/test/mochitest/manifest.js +++ b/dom/media/webvtt/test/mochitest/manifest.js @@ -11,7 +11,7 @@ function removeNodeAndSource(n) { } function once(target, name, cb) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { target.addEventListener( name, function () { diff --git a/dom/media/webvtt/test/mochitest/mochitest.toml b/dom/media/webvtt/test/mochitest/mochitest.toml index 6ec6042407..6fd4c9e7ba 100644 --- a/dom/media/webvtt/test/mochitest/mochitest.toml +++ b/dom/media/webvtt/test/mochitest/mochitest.toml @@ -18,15 +18,14 @@ support-files = [ ["test_bug883173.html"] skip-if = [ - "android_version == '25' && debug", # android(bug 1232305) - "os == 'linux' && (asan || tsan || debug)", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && asan", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && tsan", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && debug", # Bug 1829511 ] ["test_bug895091.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug957847.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug1018933.html"] @@ -39,10 +38,7 @@ skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_texttrack_cors_preload_none.html"] ["test_texttrack_mode_change_during_loading.html"] -skip-if = [ - "true", - "os == 'android'", # Bug 1636572, android(bug 1562021) -] +skip-if = ["true"] ["test_texttrack_moz.html"] diff --git a/dom/media/webvtt/update-webvtt.js b/dom/media/webvtt/update-webvtt.js index 20a3e2669f..8350cff5dc 100644 --- a/dom/media/webvtt/update-webvtt.js +++ b/dom/media/webvtt/update-webvtt.js @@ -6,7 +6,7 @@ var gift = require("gift"), fs = require("fs"), argv = require("optimist") .usage( - "Update vtt.jsm with the latest from a vtt.js directory.\nUsage:" + + "Update vtt.sys.mjs with the latest from a vtt.js directory.\nUsage:" + " $0 -d [dir]" ) .demand("d") @@ -22,7 +22,7 @@ var gift = require("gift"), .options("w", { alias: "write", describe: "Path to file to write to.", - default: "./vtt.jsm", + default: "./vtt.sys.mjs", }).argv; var repo = gift(argv.d); |