diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /dom/media/utils | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/utils')
-rw-r--r-- | dom/media/utils/MediaElementEventRunners.cpp | 140 | ||||
-rw-r--r-- | dom/media/utils/MediaElementEventRunners.h | 190 | ||||
-rw-r--r-- | dom/media/utils/PerformanceRecorder.cpp | 308 | ||||
-rw-r--r-- | dom/media/utils/PerformanceRecorder.h | 408 | ||||
-rw-r--r-- | dom/media/utils/TelemetryProbesReporter.cpp | 708 | ||||
-rw-r--r-- | dom/media/utils/TelemetryProbesReporter.h | 179 | ||||
-rw-r--r-- | dom/media/utils/gtest/TestPerformanceRecorder.cpp | 110 | ||||
-rw-r--r-- | dom/media/utils/gtest/moz.build | 15 | ||||
-rw-r--r-- | dom/media/utils/moz.build | 26 |
9 files changed, 2084 insertions, 0 deletions
diff --git a/dom/media/utils/MediaElementEventRunners.cpp b/dom/media/utils/MediaElementEventRunners.cpp new file mode 100644 index 0000000000..57be04528c --- /dev/null +++ b/dom/media/utils/MediaElementEventRunners.cpp @@ -0,0 +1,140 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaElementEventRunners.h" + +#include "mozilla/dom/HTMLMediaElement.h" + +extern mozilla::LazyLogModule gMediaElementEventsLog; +#define LOG_EVENT(type, msg) MOZ_LOG(gMediaElementEventsLog, type, msg) + +namespace mozilla::dom { + +nsMediaEventRunner::nsMediaEventRunner(const nsAString& aName, + HTMLMediaElement* aElement, + const nsAString& aEventName) + : mElement(aElement), + mName(aName), + mEventName(aEventName), + mLoadID(mElement->GetCurrentLoadID()) {} + +bool nsMediaEventRunner::IsCancelled() { + return !mElement || mElement->GetCurrentLoadID() != mLoadID; +} + +nsresult nsMediaEventRunner::DispatchEvent(const nsAString& aName) { + return mElement ? mElement->DispatchEvent(aName) : NS_OK; +} + +NS_IMPL_CYCLE_COLLECTION(nsMediaEventRunner, mElement) +NS_IMPL_CYCLE_COLLECTING_ADDREF(nsMediaEventRunner) +NS_IMPL_CYCLE_COLLECTING_RELEASE(nsMediaEventRunner) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsMediaEventRunner) + NS_INTERFACE_MAP_ENTRY(nsINamed) + NS_INTERFACE_MAP_ENTRY(nsIRunnable) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIRunnable) +NS_INTERFACE_MAP_END + +NS_IMETHODIMP nsAsyncEventRunner::Run() { + // Silently cancel if our load has been cancelled or element has been CCed. + return IsCancelled() ? NS_OK : DispatchEvent(mEventName); +} + +nsResolveOrRejectPendingPlayPromisesRunner:: + nsResolveOrRejectPendingPlayPromisesRunner( + HTMLMediaElement* aElement, nsTArray<RefPtr<PlayPromise>>&& aPromises, + nsresult aError) + : nsMediaEventRunner(u"nsResolveOrRejectPendingPlayPromisesRunner"_ns, + aElement), + mPromises(std::move(aPromises)), + mError(aError) { + mElement->mPendingPlayPromisesRunners.AppendElement(this); +} + +void nsResolveOrRejectPendingPlayPromisesRunner::ResolveOrReject() { + if (NS_SUCCEEDED(mError)) { + PlayPromise::ResolvePromisesWithUndefined(mPromises); + } else { + PlayPromise::RejectPromises(mPromises, mError); + } +} + +NS_IMETHODIMP nsResolveOrRejectPendingPlayPromisesRunner::Run() { + if (!IsCancelled()) { + ResolveOrReject(); + } + + mElement->mPendingPlayPromisesRunners.RemoveElement(this); + return NS_OK; +} + +NS_IMETHODIMP nsNotifyAboutPlayingRunner::Run() { + if (!IsCancelled()) { + DispatchEvent(u"playing"_ns); + } + return nsResolveOrRejectPendingPlayPromisesRunner::Run(); +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(nsResolveOrRejectPendingPlayPromisesRunner, + nsMediaEventRunner, mPromises) +NS_IMPL_ADDREF_INHERITED(nsResolveOrRejectPendingPlayPromisesRunner, + nsMediaEventRunner) +NS_IMPL_RELEASE_INHERITED(nsResolveOrRejectPendingPlayPromisesRunner, + nsMediaEventRunner) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION( + nsResolveOrRejectPendingPlayPromisesRunner) +NS_INTERFACE_MAP_END_INHERITING(nsMediaEventRunner) + +NS_IMETHODIMP nsSourceErrorEventRunner::Run() { + // Silently cancel if our load has been cancelled. + if (IsCancelled()) { + return NS_OK; + } + LOG_EVENT(LogLevel::Debug, + ("%p Dispatching simple event source error", mElement.get())); + return nsContentUtils::DispatchTrustedEvent(mElement->OwnerDoc(), mSource, + u"error"_ns, CanBubble::eNo, + Cancelable::eNo); +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(nsSourceErrorEventRunner, nsMediaEventRunner, + mSource) +NS_IMPL_ADDREF_INHERITED(nsSourceErrorEventRunner, nsMediaEventRunner) +NS_IMPL_RELEASE_INHERITED(nsSourceErrorEventRunner, nsMediaEventRunner) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsSourceErrorEventRunner) +NS_INTERFACE_MAP_END_INHERITING(nsMediaEventRunner) + +NS_IMETHODIMP nsTimeupdateRunner::Run() { + if (IsCancelled() || !ShouldDispatchTimeupdate()) { + return NS_OK; + } + // After dispatching `timeupdate`, if the timeupdate event listener takes lots + // of time then we end up spending all time handling just timeupdate events. + // The spec is vague in this situation, so we choose to update time after we + // dispatch the event in order to solve that issue. + nsresult rv = DispatchEvent(mEventName); + if (NS_WARN_IF(NS_FAILED(rv))) { + LOG_EVENT(LogLevel::Debug, + ("%p Failed to dispatch 'timeupdate'", mElement.get())); + } else { + mElement->UpdateLastTimeupdateDispatchTime(); + } + return rv; +} + +bool nsTimeupdateRunner::ShouldDispatchTimeupdate() const { + if (mIsMandatory) { + return true; + } + + // If the main thread is busy, tasks may be delayed and dispatched at + // unexpected times. Ensure we don't dispatch `timeupdate` more often + // than once per `TIMEUPDATE_MS`. + const TimeStamp& lastTime = mElement->LastTimeupdateDispatchTime(); + return lastTime.IsNull() || TimeStamp::Now() - lastTime > + TimeDuration::FromMilliseconds(TIMEUPDATE_MS); +} + +#undef LOG_EVENT +} // namespace mozilla::dom diff --git a/dom/media/utils/MediaElementEventRunners.h b/dom/media/utils/MediaElementEventRunners.h new file mode 100644 index 0000000000..3f13494493 --- /dev/null +++ b/dom/media/utils/MediaElementEventRunners.h @@ -0,0 +1,190 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_media_mediaelementeventrunners_h +#define mozilla_media_mediaelementeventrunners_h + +#include "mozilla/dom/PlayPromise.h" +#include "nsCycleCollectionParticipant.h" +#include "nsIContent.h" +#include "nsINamed.h" +#include "nsIRunnable.h" +#include "nsString.h" +#include "nsISupportsImpl.h" +#include "nsTString.h" + +namespace mozilla::dom { + +class HTMLMediaElement; + +// Under certain conditions there may be no-one holding references to +// a media element from script, DOM parent, etc, but the element may still +// fire meaningful events in the future so we can't destroy it yet: +// 1) If the element is delaying the load event (or would be, if it were +// in a document), then events up to loadeddata or error could be fired, +// so we need to stay alive. +// 2) If the element is not paused and playback has not ended, then +// we will (or might) play, sending timeupdate and ended events and possibly +// audio output, so we need to stay alive. +// 3) if the element is seeking then we will fire seeking events and possibly +// start playing afterward, so we need to stay alive. +// 4) If autoplay could start playback in this element (if we got enough data), +// then we need to stay alive. +// 5) if the element is currently loading, not suspended, and its source is +// not a MediaSource, then script might be waiting for progress events or a +// 'stalled' or 'suspend' event, so we need to stay alive. +// If we're already suspended then (all other conditions being met), +// it's OK to just disappear without firing any more events, +// since we have the freedom to remain suspended indefinitely. Note +// that we could use this 'suspended' loophole to garbage-collect a suspended +// element in case 4 even if it had 'autoplay' set, but we choose not to. +// If someone throws away all references to a loading 'autoplay' element +// sound should still eventually play. +// 6) If the source is a MediaSource, most loading events will not fire unless +// appendBuffer() is called on a SourceBuffer, in which case something is +// already referencing the SourceBuffer, which keeps the associated media +// element alive. Further, a MediaSource will never time out the resource +// fetch, and so should not keep the media element alive if it is +// unreferenced. A pending 'stalled' event keeps the media element alive. +// +// Media elements owned by inactive documents (i.e. documents not contained in +// any document viewer) should never hold a self-reference because none of the +// above conditions are allowed: the element will stop loading and playing +// and never resume loading or playing unless its owner document changes to +// an active document (which can only happen if there is an external reference +// to the element). +// Media elements with no owner doc should be able to hold a self-reference. +// Something native must have created the element and may expect it to +// stay alive to play. + +// It's very important that any change in state which could change the value of +// needSelfReference in AddRemoveSelfReference be followed by a call to +// AddRemoveSelfReference before this element could die! +// It's especially important if needSelfReference would change to 'true', +// since if we neglect to add a self-reference, this element might be +// garbage collected while there are still event listeners that should +// receive events. If we neglect to remove the self-reference then the element +// just lives longer than it needs to. + +class nsMediaEventRunner : public nsIRunnable, public nsINamed { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_CLASS_AMBIGUOUS(nsMediaEventRunner, nsIRunnable) + + explicit nsMediaEventRunner(const nsAString& aName, + HTMLMediaElement* aElement, + const nsAString& aEventName = u"unknown"_ns); + + void Cancel() { mElement = nullptr; } + NS_IMETHODIMP GetName(nsACString& aName) override { + aName = NS_ConvertUTF16toUTF8(mName).get(); + return NS_OK; + } + nsString Name() const { return mName; } + nsString EventName() const { return mEventName; } + + protected: + virtual ~nsMediaEventRunner() = default; + bool IsCancelled(); + nsresult DispatchEvent(const nsAString& aName); + + RefPtr<HTMLMediaElement> mElement; + nsString mName; + nsString mEventName; + uint32_t mLoadID; +}; + +/** + * This runner is used to dispatch async event on media element. + */ +class nsAsyncEventRunner : public nsMediaEventRunner { + public: + nsAsyncEventRunner(const nsAString& aEventName, HTMLMediaElement* aElement) + : nsMediaEventRunner(u"nsAsyncEventRunner"_ns, aElement, aEventName) {} + NS_IMETHOD Run() override; +}; + +/** + * These runners are used to handle `playing` event and address play promise. + * + * If no error is passed while constructing an instance, the instance will + * resolve the passed promises with undefined; otherwise, the instance will + * reject the passed promises with the passed error. + * + * The constructor appends the constructed instance into the passed media + * element's mPendingPlayPromisesRunners member and once the the runner is run + * (whether fulfilled or canceled), it removes itself from + * mPendingPlayPromisesRunners. + */ +class nsResolveOrRejectPendingPlayPromisesRunner : public nsMediaEventRunner { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED( + nsResolveOrRejectPendingPlayPromisesRunner, nsMediaEventRunner) + + nsResolveOrRejectPendingPlayPromisesRunner( + HTMLMediaElement* aElement, nsTArray<RefPtr<PlayPromise>>&& aPromises, + nsresult aError = NS_OK); + void ResolveOrReject(); + NS_IMETHOD Run() override; + + protected: + virtual ~nsResolveOrRejectPendingPlayPromisesRunner() = default; + + private: + nsTArray<RefPtr<PlayPromise>> mPromises; + nsresult mError; +}; + +class nsNotifyAboutPlayingRunner + : public nsResolveOrRejectPendingPlayPromisesRunner { + public: + nsNotifyAboutPlayingRunner( + HTMLMediaElement* aElement, + nsTArray<RefPtr<PlayPromise>>&& aPendingPlayPromises) + : nsResolveOrRejectPendingPlayPromisesRunner( + aElement, std::move(aPendingPlayPromises)) {} + NS_IMETHOD Run() override; +}; + +/** + * This runner is used to dispatch a source error event, which would happen when + * loading resource failed. + */ +class nsSourceErrorEventRunner : public nsMediaEventRunner { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(nsSourceErrorEventRunner, + nsMediaEventRunner) + nsSourceErrorEventRunner(HTMLMediaElement* aElement, nsIContent* aSource) + : nsMediaEventRunner(u"nsSourceErrorEventRunner"_ns, aElement), + mSource(aSource) {} + NS_IMETHOD Run() override; + + private: + virtual ~nsSourceErrorEventRunner() = default; + nsCOMPtr<nsIContent> mSource; +}; + +/** + * This runner is used to dispatch `timeupdate` event and ensure we don't + * dispatch `timeupdate` more often than once per `TIMEUPDATE_MS` if that is not + * a mandatory event. + */ +class nsTimeupdateRunner : public nsMediaEventRunner { + public: + nsTimeupdateRunner(HTMLMediaElement* aElement, bool aIsMandatory) + : nsMediaEventRunner(u"nsTimeupdateRunner"_ns, aElement, + u"timeupdate"_ns), + mIsMandatory(aIsMandatory) {} + NS_IMETHOD Run() override; + + private: + bool ShouldDispatchTimeupdate() const; + bool mIsMandatory; +}; + +} // namespace mozilla::dom + +#endif // mozilla_media_mediaelementeventrunners_h diff --git a/dom/media/utils/PerformanceRecorder.cpp b/dom/media/utils/PerformanceRecorder.cpp new file mode 100644 index 0000000000..d6124e8cf6 --- /dev/null +++ b/dom/media/utils/PerformanceRecorder.cpp @@ -0,0 +1,308 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PerformanceRecorder.h" + +#include "base/process_util.h" +#include "mozilla/Logging.h" +#include "mozilla/gfx/Types.h" +#include "nsPrintfCString.h" + +namespace mozilla { + +static const char* SourceToStr(TrackingId::Source aSource) { + switch (aSource) { + case TrackingId::Source::Unimplemented: + MOZ_ASSERT_UNREACHABLE("Unimplemented TrackingId Source"); + return "Unimplemented"; + case TrackingId::Source::AudioDestinationNode: + return "AudioDestinationNode"; + case TrackingId::Source::Camera: + return "CameraCapture"; + case TrackingId::Source::Canvas: + return "CanvasCapture"; + case TrackingId::Source::ChannelDecoder: + return "ChannelDecoder"; + case TrackingId::Source::HLSDecoder: + return "HLSDecoder"; + case TrackingId::Source::MediaCapabilities: + return "MediaCapabilities"; + case TrackingId::Source::MediaElementDecoder: + return "MediaElementDecoderCapture"; + case TrackingId::Source::MediaElementStream: + return "MediaElementStreamCapture"; + case TrackingId::Source::MSEDecoder: + return "MSEDecoder"; + case TrackingId::Source::RTCRtpReceiver: + return "RTCRtpReceiver"; + case TrackingId::Source::Screen: + return "ScreenCapture"; + case TrackingId::Source::Tab: + return "TabCapture"; + case TrackingId::Source::Window: + return "WindowCapture"; + case TrackingId::Source::LAST: + MOZ_ASSERT_UNREACHABLE("Invalid TrackingId Source"); + return "Invalid"; + } + MOZ_ASSERT_UNREACHABLE("Unexpected TrackingId Source"); + return "Unexpected"; +} + +TrackingId::TrackingId() : mSource(Source::Unimplemented), mUniqueInProcId(0) {} + +TrackingId::TrackingId( + Source aSource, uint32_t aUniqueInProcId, + TrackAcrossProcesses aTrack /* = TrackAcrossProcesses::NO */) + : mSource(aSource), + mUniqueInProcId(aUniqueInProcId), + mProcId(aTrack == TrackAcrossProcesses::Yes + ? Some(base::GetCurrentProcId()) + : Nothing()) {} + +nsCString TrackingId::ToString() const { + if (mProcId) { + return nsPrintfCString("%s-%u-%u", SourceToStr(mSource), *mProcId, + mUniqueInProcId); + } + return nsPrintfCString("%s-%u", SourceToStr(mSource), mUniqueInProcId); +} + +static const char* StageToStr(MediaStage aStage) { + switch (aStage) { + case MediaStage::RequestData: + return "RequestData"; + case MediaStage::RequestDemux: + return "RequestDemux"; + case MediaStage::CopyDemuxedData: + return "CopyDemuxedData"; + case MediaStage::RequestDecode: + return "RequestDecode"; + case MediaStage::CopyDecodedVideo: + return "CopyDecodedVideo"; + default: + return "InvalidStage"; + } +} + +static void AppendMediaInfoFlagToName(nsCString& aName, MediaInfoFlag aFlag) { + if (aFlag & MediaInfoFlag::KeyFrame) { + aName.Append("kf,"); + } + // Decoding + if (aFlag & MediaInfoFlag::SoftwareDecoding) { + aName.Append("sw,"); + } else if (aFlag & MediaInfoFlag::HardwareDecoding) { + aName.Append("hw,"); + } + // Codec type + if (aFlag & MediaInfoFlag::VIDEO_AV1) { + aName.Append("av1,"); + } else if (aFlag & MediaInfoFlag::VIDEO_H264) { + aName.Append("h264,"); + } else if (aFlag & MediaInfoFlag::VIDEO_VP8) { + aName.Append("vp8,"); + } else if (aFlag & MediaInfoFlag::VIDEO_VP9) { + aName.Append("vp9,"); + } else if (aFlag & MediaInfoFlag::VIDEO_THEORA) { + aName.Append("theora,"); + } +} + +static void AppendImageFormatToName(nsCString& aName, + DecodeStage::ImageFormat aFormat) { + aName.Append([&] { + switch (aFormat) { + case DecodeStage::YUV420P: + return "yuv420p,"; + case DecodeStage::YUV422P: + return "yuv422p,"; + case DecodeStage::YUV444P: + return "yuv444p,"; + case DecodeStage::NV12: + return "nv12,"; + case DecodeStage::YV12: + return "yv12,"; + case DecodeStage::NV21: + return "nv21,"; + case DecodeStage::P010: + return "p010,"; + case DecodeStage::P016: + return "p016,"; + case DecodeStage::RGBA32: + return "rgba32,"; + case DecodeStage::RGB24: + return "rgb24,"; + case DecodeStage::GBRP: + return "gbrp,"; + case DecodeStage::ANDROID_SURFACE: + return "android.Surface,"; + } + MOZ_ASSERT_UNREACHABLE("Unhandled DecodeStage::ImageFormat"); + return ""; + }()); +} + +static void AppendYUVColorSpaceToName(nsCString& aName, + gfx::YUVColorSpace aSpace) { + aName.Append([&] { + switch (aSpace) { + case gfx::YUVColorSpace::BT601: + return "space=BT.601,"; + case gfx::YUVColorSpace::BT709: + return "space=BT.709,"; + case gfx::YUVColorSpace::BT2020: + return "space=BT.2020,"; + case gfx::YUVColorSpace::Identity: + return "space=Identity,"; + } + MOZ_ASSERT_UNREACHABLE("Unhandled gfx::YUVColorSpace"); + return ""; + }()); +} + +static void AppendColorRangeToName(nsCString& aName, gfx::ColorRange aRange) { + aName.Append([&] { + switch (aRange) { + case gfx::ColorRange::LIMITED: + return "range=Limited,"; + case gfx::ColorRange::FULL: + return "range=Full,"; + } + MOZ_ASSERT_UNREACHABLE("Unhandled gfx::ColorRange"); + return ""; + }()); +} + +static void AppendColorDepthToName(nsCString& aName, gfx::ColorDepth aDepth) { + aName.Append([&] { + switch (aDepth) { + case gfx::ColorDepth::COLOR_8: + return "depth=8,"; + case gfx::ColorDepth::COLOR_10: + return "depth=10,"; + case gfx::ColorDepth::COLOR_12: + return "depth=12,"; + case gfx::ColorDepth::COLOR_16: + return "depth=16,"; + } + MOZ_ASSERT_UNREACHABLE("Unhandled gfx::ColorDepth"); + return ""; + }()); +} + +/* static */ +const char* FindMediaResolution(int32_t aHeight) { + static const struct { + const int32_t mH; + const nsCString mRes; + } sResolutions[] = {{0, "A:0"_ns}, // other followings are for video + {240, "V:0<h<=240"_ns}, + {480, "V:240<h<=480"_ns}, + {576, "V:480<h<=576"_ns}, + {720, "V:576<h<=720"_ns}, + {1080, "V:720<h<=1080"_ns}, + {1440, "V:1080<h<=1440"_ns}, + {2160, "V:1440<h<=2160"_ns}, + {INT_MAX, "V:h>2160"_ns}}; + const char* resolution = sResolutions[0].mRes.get(); + for (auto&& res : sResolutions) { + if (aHeight <= res.mH) { + resolution = res.mRes.get(); + break; + } + } + return resolution; +} + +/* static */ +bool PerformanceRecorderBase::IsMeasurementEnabled() { + return profiler_thread_is_being_profiled_for_markers() || + PerformanceRecorderBase::sEnableMeasurementForTesting; +} + +/* static */ +TimeStamp PerformanceRecorderBase::GetCurrentTimeForMeasurement() { + // The system call to get the clock is rather expensive on Windows. As we + // only report the measurement report via markers, if the marker isn't enabled + // then we won't do any measurement in order to save CPU time. + return IsMeasurementEnabled() ? TimeStamp::Now() : TimeStamp(); +} + +ProfilerString8View PlaybackStage::Name() const { + if (!mName) { + mName.emplace(StageToStr(mStage)); + mName->Append(":"); + mName->Append(FindMediaResolution(mHeight)); + mName->Append(":"); + AppendMediaInfoFlagToName(*mName, mFlag); + } + return *mName; +} + +ProfilerString8View CaptureStage::Name() const { + if (!mName) { + auto imageTypeToStr = [](ImageType aType) -> const char* { + switch (aType) { + case ImageType::I420: + return "I420"; + case ImageType::YUY2: + return "YUY2"; + case ImageType::YV12: + return "YV12"; + case ImageType::UYVY: + return "UYVY"; + case ImageType::NV12: + return "NV12"; + case ImageType::NV21: + return "NV21"; + case ImageType::MJPEG: + return "MJPEG"; + case ImageType::Unknown: + return "(unknown image type)"; + default: + return "(unimplemented image type)"; + }; + }; + mName = Some(nsPrintfCString( + "CaptureVideoFrame %s %dx%d %s %s", mSource.Data(), mWidth, mHeight, + imageTypeToStr(mImageType), mTrackingId.ToString().get())); + } + return *mName; +} + +ProfilerString8View CopyVideoStage::Name() const { + if (!mName) { + mName = + Some(nsPrintfCString("CopyVideoFrame %s %dx%d %s", mSource.Data(), + mWidth, mHeight, mTrackingId.ToString().get())); + } + return *mName; +} + +ProfilerString8View DecodeStage::Name() const { + if (!mName) { + nsCString extras; + AppendMediaInfoFlagToName(extras, mFlag); + mImageFormat.apply( + [&](ImageFormat aFormat) { AppendImageFormatToName(extras, aFormat); }); + mColorDepth.apply([&](gfx::ColorDepth aDepth) { + AppendColorDepthToName(extras, aDepth); + }); + mColorRange.apply([&](gfx::ColorRange aRange) { + AppendColorRangeToName(extras, aRange); + }); + mYUVColorSpace.apply([&](gfx::YUVColorSpace aColorSpace) { + AppendYUVColorSpaceToName(extras, aColorSpace); + }); + mName = Some(nsPrintfCString("DecodeFrame %s %dx%d %s %s", mSource.Data(), + mWidth.valueOr(-1), mHeight.valueOr(-1), + extras.get(), mTrackingId.ToString().get())); + } + return *mName; +} + +} // namespace mozilla diff --git a/dom/media/utils/PerformanceRecorder.h b/dom/media/utils/PerformanceRecorder.h new file mode 100644 index 0000000000..34c6676b01 --- /dev/null +++ b/dom/media/utils/PerformanceRecorder.h @@ -0,0 +1,408 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_PerformanceRecorder_h +#define mozilla_PerformanceRecorder_h + +#include <type_traits> + +#include "mozilla/Attributes.h" +#include "mozilla/BaseProfilerMarkersPrerequisites.h" +#include "mozilla/Maybe.h" +#include "mozilla/Mutex.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/TypedEnumBits.h" +#include "nsStringFwd.h" +#include "nsTPriorityQueue.h" +#include "mozilla/ProfilerMarkers.h" + +namespace mozilla { +namespace gfx { +enum class YUVColorSpace : uint8_t; +enum class ColorDepth : uint8_t; +enum class ColorRange : uint8_t; +} // namespace gfx + +struct TrackingId { + enum class Source : uint8_t { + Unimplemented, + AudioDestinationNode, + Camera, + Canvas, + ChannelDecoder, + HLSDecoder, + MediaCapabilities, + MediaElementDecoder, + MediaElementStream, + MSEDecoder, + RTCRtpReceiver, + Screen, + Tab, + Window, + LAST + }; + enum class TrackAcrossProcesses : uint8_t { + Yes, + No, + }; + TrackingId(); + TrackingId(Source aSource, uint32_t aUniqueInProcId, + TrackAcrossProcesses aTrack = TrackAcrossProcesses::No); + + nsCString ToString() const; + + Source mSource; + uint32_t mUniqueInProcId; + Maybe<uint32_t> mProcId; +}; + +enum class MediaInfoFlag : uint16_t { + None = (0 << 0), + NonKeyFrame = (1 << 0), + KeyFrame = (1 << 1), + SoftwareDecoding = (1 << 2), + HardwareDecoding = (1 << 3), + VIDEO_AV1 = (1 << 4), + VIDEO_H264 = (1 << 5), + VIDEO_VP8 = (1 << 6), + VIDEO_VP9 = (1 << 7), + VIDEO_THEORA = (1 << 8), + VIDEO_HEVC = (1 << 9), +}; +MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(MediaInfoFlag) + +/** + * This represents the different stages that a media data will go through + * within the playback journey. + * + * |---| |---| |------| + * Copy Demuxed Copy Demuxed Copy Decoded + * Data Data Video + * |------------- | |-----------------------------------| + * Request Demux Request Decode + * |-----------------------------------------------------------| + * Request Data + * + * RequestData : Record the time where MediaDecoderStateMachine(MDSM) starts + * asking for a decoded data to MDSM receives a decoded data. + * + * RequestDemux : Record the time where MediaFormatReader(MFR) starts asking + * a demuxed sample to MFR received a demuxed sample. This stage is a sub- + * stage of RequestData. + * + * CopyDemuxedData : On some situations, we will need to copy the demuxed + * data, which is still not decoded yet so its size is still small. This + * records the time which we spend on copying data. This stage could happen + * multiple times, either being a sub-stage of RequestDemux (in MSE case), + * or being a sub-stage of RequestDecode (when sending data via IPC). + * + * RequestDecode : Record the time where MFR starts asking decoder to return + * a decoded data to MFR receives a decoded data. As the decoder might be + * remote, this stage might include the time spending on IPC trips. This + * stage is a sub-stage of RequestData. + * + * CopyDecodedVideo : If we can't reuse same decoder texture to the + * compositor, then we have to copy video data to to another sharable + * texture. This records the time which we spend on copying data. This stage + * is a sub- stage of RequestDecode. + */ +enum class MediaStage : uint8_t { + Invalid, + RequestData, + RequestDemux, + CopyDemuxedData, + RequestDecode, + CopyDecodedVideo, +}; + +class PlaybackStage { + public: + explicit PlaybackStage(MediaStage aStage, int32_t aHeight = 0, + MediaInfoFlag aFlag = MediaInfoFlag::None) + : mStage(aStage), mHeight(aHeight), mFlag(aFlag) { + MOZ_ASSERT(aStage != MediaStage::Invalid); + } + + ProfilerString8View Name() const; + const MarkerCategory& Category() const { + return baseprofiler::category::MEDIA_PLAYBACK; + } + + MediaStage mStage; + int32_t mHeight; + MediaInfoFlag mFlag; + + private: + mutable Maybe<nsCString> mName; +}; + +class CaptureStage { + public: + enum class ImageType : uint8_t { + Unknown, + I420, + YUY2, + YV12, + UYVY, + NV12, + NV21, + MJPEG, + }; + + CaptureStage(nsCString aSource, TrackingId aTrackingId, int32_t aWidth, + int32_t aHeight, ImageType aImageType) + : mSource(std::move(aSource)), + mTrackingId(std::move(aTrackingId)), + mWidth(aWidth), + mHeight(aHeight), + mImageType(aImageType) {} + + ProfilerString8View Name() const; + const MarkerCategory& Category() const { + return baseprofiler::category::MEDIA_RT; + } + + nsCString mSource; + TrackingId mTrackingId; + int32_t mWidth; + int32_t mHeight; + ImageType mImageType; + + private: + mutable Maybe<nsCString> mName; +}; + +class CopyVideoStage { + public: + CopyVideoStage(nsCString aSource, TrackingId aTrackingId, int32_t aWidth, + int32_t aHeight) + : mSource(std::move(aSource)), + mTrackingId(std::move(aTrackingId)), + mWidth(aWidth), + mHeight(aHeight) {} + + ProfilerString8View Name() const; + const MarkerCategory& Category() const { + return baseprofiler::category::MEDIA_RT; + } + + // The name of the source that performs this stage. + nsCString mSource; + // A unique id identifying the source of the video frame this stage is + // performed for. + TrackingId mTrackingId; + int32_t mWidth; + int32_t mHeight; + + private: + mutable Maybe<nsCString> mName; +}; + +class DecodeStage { + public: + enum ImageFormat : uint8_t { + YUV420P, + YUV422P, + YUV444P, + NV12, + YV12, + NV21, + P010, + P016, + RGBA32, + RGB24, + GBRP, + ANDROID_SURFACE, + }; + + DecodeStage(nsCString aSource, TrackingId aTrackingId, MediaInfoFlag aFlag) + : mSource(std::move(aSource)), + mTrackingId(std::move(aTrackingId)), + mFlag(aFlag) {} + ProfilerString8View Name() const; + const MarkerCategory& Category() const { + return baseprofiler::category::MEDIA_PLAYBACK; + } + + void SetResolution(int aWidth, int aHeight) { + mWidth = Some(aWidth); + mHeight = Some(aHeight); + } + void SetImageFormat(ImageFormat aFormat) { mImageFormat = Some(aFormat); } + void SetYUVColorSpace(gfx::YUVColorSpace aColorSpace) { + mYUVColorSpace = Some(aColorSpace); + } + void SetColorRange(gfx::ColorRange aColorRange) { + mColorRange = Some(aColorRange); + } + void SetColorDepth(gfx::ColorDepth aColorDepth) { + mColorDepth = Some(aColorDepth); + } + + // The name of the source that performs this stage. + nsCString mSource; + // A unique id identifying the source of the video frame this stage is + // performed for. + TrackingId mTrackingId; + MediaInfoFlag mFlag; + Maybe<int> mWidth; + Maybe<int> mHeight; + Maybe<ImageFormat> mImageFormat; + Maybe<gfx::YUVColorSpace> mYUVColorSpace; + Maybe<gfx::ColorRange> mColorRange; + Maybe<gfx::ColorDepth> mColorDepth; + mutable Maybe<nsCString> mName; +}; + +class PerformanceRecorderBase { + public: + static bool IsMeasurementEnabled(); + static TimeStamp GetCurrentTimeForMeasurement(); + + // Return the resolution range for the given height. Eg. V:1080<h<=1440. + static const char* FindMediaResolution(int32_t aHeight); + + protected: + // We would enable the measurement on testing. + static inline bool sEnableMeasurementForTesting = false; +}; + +template <typename StageType> +class PerformanceRecorderImpl : public PerformanceRecorderBase { + public: + ~PerformanceRecorderImpl() = default; + + PerformanceRecorderImpl(PerformanceRecorderImpl&& aRhs) noexcept + : mStages(std::move(aRhs.mStages)) {} + PerformanceRecorderImpl& operator=(PerformanceRecorderImpl&&) = delete; + PerformanceRecorderImpl(const PerformanceRecorderImpl&) = delete; + PerformanceRecorderImpl& operator=(const PerformanceRecorderImpl&) = delete; + + protected: + PerformanceRecorderImpl() = default; + + // Stores the stage with the current time as its start time, associated with + // aId. + template <typename... Args> + void Start(int64_t aId, Args... aArgs) { + if (IsMeasurementEnabled()) { + MutexAutoLock lock(mMutex); + mStages.Push(std::make_tuple(aId, GetCurrentTimeForMeasurement(), + StageType(std::move(aArgs)...))); + } + } + + // Return the passed time since creation of the aId stage in microseconds if + // it has not yet been recorded. Other stages with lower ids will be + // discarded. Otherwise, return 0. + template <typename F> + float Record(int64_t aId, F&& aStageMutator) { + Maybe<Entry> entry; + { + MutexAutoLock lock(mMutex); + while (!mStages.IsEmpty() && std::get<0>(mStages.Top()) < aId) { + mStages.Pop(); + } + if (mStages.IsEmpty()) { + return 0.0; + } + if (std::get<0>(mStages.Top()) != aId) { + return 0.0; + } + entry = Some(mStages.Pop()); + } + const auto& startTime = std::get<1>(*entry); + auto& stage = std::get<2>(*entry); + MOZ_ASSERT(std::get<0>(*entry) == aId); + double elapsedTimeUs = 0.0; + if (!startTime.IsNull() && IsMeasurementEnabled()) { + const auto now = TimeStamp::Now(); + elapsedTimeUs = (now - startTime).ToMicroseconds(); + MOZ_ASSERT(elapsedTimeUs >= 0, "Elapsed time can't be less than 0!"); + aStageMutator(stage); + AUTO_PROFILER_STATS(PROFILER_MARKER_UNTYPED); + profiler_add_marker( + stage.Name(), stage.Category(), + MarkerOptions(MarkerTiming::Interval(startTime, now))); + } + return static_cast<float>(elapsedTimeUs); + } + float Record(int64_t aId) { + return Record(aId, [](auto&) {}); + } + + protected: + using Entry = std::tuple<int64_t, TimeStamp, StageType>; + + struct IdComparator { + bool LessThan(const Entry& aTupleA, const Entry& aTupleB) { + return std::get<0>(aTupleA) < std::get<0>(aTupleB); + } + }; + + Mutex mMutex{"PerformanceRecorder::mMutex"}; + nsTPriorityQueue<Entry, IdComparator> mStages MOZ_GUARDED_BY(mMutex); +}; + +/** + * This class is used to record the time spent on different stages in the media + * pipeline. `Record()` needs to be called explicitly to record a profiler + * marker registering the time passed since creation. A stage may be mutated in + * `Record()` in case data has become available since the recorder started. + * + * This variant is intended to be created on the stack when a stage starts, then + * recorded with `Record()` when the stage is finished. + */ +template <typename StageType> +class PerformanceRecorder : public PerformanceRecorderImpl<StageType> { + using Super = PerformanceRecorderImpl<StageType>; + + public: + template <typename... Args> + explicit PerformanceRecorder(Args... aArgs) { + Start(std::move(aArgs)...); + }; + + private: + template <typename... Args> + void Start(Args... aArgs) { + Super::Start(0, std::move(aArgs)...); + } + + public: + template <typename F> + float Record(F&& aStageMutator) { + return Super::Record(0, std::forward<F>(aStageMutator)); + } + float Record() { return Super::Record(0); } +}; + +/** + * This class is used to record the time spent on different stages in the media + * pipeline. `Start()` and `Record()` needs to be called explicitly to record a + * profiler marker registering the time passed since creation. A stage may be + * mutated in `Record()` in case data has become available since the recorder + * started. + * + * This variant is intended to be kept as a member in a class and supports async + * stages. The async stages may overlap each other. To distinguish different + * stages from each other, an int64_t is used as identifier. This is often a + * timestamp in microseconds, see TimeUnit::ToMicroseconds. + */ +template <typename StageType> +class PerformanceRecorderMulti : public PerformanceRecorderImpl<StageType> { + using Super = PerformanceRecorderImpl<StageType>; + + public: + PerformanceRecorderMulti() = default; + + using Super::Record; + using Super::Start; +}; + +} // namespace mozilla + +#endif // mozilla_PerformanceRecorder_h diff --git a/dom/media/utils/TelemetryProbesReporter.cpp b/dom/media/utils/TelemetryProbesReporter.cpp new file mode 100644 index 0000000000..a2854812ba --- /dev/null +++ b/dom/media/utils/TelemetryProbesReporter.cpp @@ -0,0 +1,708 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "TelemetryProbesReporter.h" + +#include <cmath> + +#include "FrameStatistics.h" +#include "VideoUtils.h" +#include "mozilla/EMEUtils.h" +#include "mozilla/Logging.h" +#include "mozilla/Telemetry.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/glean/GleanMetrics.h" +#include "nsThreadUtils.h" + +namespace mozilla { + +LazyLogModule gTelemetryProbesReporterLog("TelemetryProbesReporter"); +#define LOG(msg, ...) \ + MOZ_LOG(gTelemetryProbesReporterLog, LogLevel::Debug, \ + ("TelemetryProbesReporter=%p, " msg, this, ##__VA_ARGS__)) + +static const char* ToVisibilityStr( + TelemetryProbesReporter::Visibility aVisibility) { + switch (aVisibility) { + case TelemetryProbesReporter::Visibility::eVisible: + return "visible"; + case TelemetryProbesReporter::Visibility::eInvisible: + return "invisible"; + case TelemetryProbesReporter::Visibility::eInitial: + return "initial"; + default: + MOZ_ASSERT_UNREACHABLE("invalid visibility"); + return "unknown"; + } +} +static const char* ToAudibilityStr( + TelemetryProbesReporter::AudibleState aAudibleState) { + switch (aAudibleState) { + case TelemetryProbesReporter::AudibleState::eAudible: + return "audible"; + case TelemetryProbesReporter::AudibleState::eNotAudible: + return "inaudible"; + default: + MOZ_ASSERT_UNREACHABLE("invalid audibility"); + return "unknown"; + } +} + +static const char* ToMutedStr(bool aMuted) { + return aMuted ? "muted" : "unmuted"; +} + +MediaContent TelemetryProbesReporter::MediaInfoToMediaContent( + const MediaInfo& aInfo) { + MediaContent content = MediaContent::MEDIA_HAS_NOTHING; + if (aInfo.HasAudio()) { + content |= MediaContent::MEDIA_HAS_AUDIO; + } + if (aInfo.HasVideo()) { + content |= MediaContent::MEDIA_HAS_VIDEO; + if (aInfo.mVideo.GetAsVideoInfo()->mColorDepth > gfx::ColorDepth::COLOR_8) { + content |= MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8; + } + } + return content; +} + +TelemetryProbesReporter::TelemetryProbesReporter( + TelemetryProbesReporterOwner* aOwner) + : mOwner(aOwner) { + MOZ_ASSERT(mOwner); +} + +void TelemetryProbesReporter::OnPlay(Visibility aVisibility, + MediaContent aMediaContent, + bool aIsMuted) { + LOG("Start time accumulation for total play time"); + + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_VIDEO, + !mTotalVideoPlayTime.IsStarted()); + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_AUDIO, + !mTotalAudioPlayTime.IsStarted()); + + if (aMediaContent & MediaContent::MEDIA_HAS_VIDEO) { + mTotalVideoPlayTime.Start(); + + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8, + !mTotalVideoHDRPlayTime.IsStarted()); + if (aMediaContent & MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8) { + mTotalVideoHDRPlayTime.Start(); + } + } + if (aMediaContent & MediaContent::MEDIA_HAS_AUDIO) { + mTotalAudioPlayTime.Start(); + } + + OnMediaContentChanged(aMediaContent); + OnVisibilityChanged(aVisibility); + OnMutedChanged(aIsMuted); + + mOwner->DispatchAsyncTestingEvent(u"moztotalplaytimestarted"_ns); + + mIsPlaying = true; +} + +void TelemetryProbesReporter::OnPause(Visibility aVisibility) { + if (!mIsPlaying) { + // Not started + LOG("TelemetryProbesReporter::OnPause: not started, early return"); + return; + } + + LOG("Pause time accumulation for total play time"); + + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_VIDEO, + mTotalVideoPlayTime.IsStarted()); + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_AUDIO, + mTotalAudioPlayTime.IsStarted()); + + if (mMediaContent & MediaContent::MEDIA_HAS_VIDEO) { + MOZ_ASSERT_IF(mMediaContent & MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8, + mTotalVideoHDRPlayTime.IsStarted()); + + LOG("Pause video time accumulation for total play time"); + if (mInvisibleVideoPlayTime.IsStarted()) { + LOG("Pause invisible video time accumulation for total play time"); + PauseInvisibleVideoTimeAccumulator(); + } + mTotalVideoPlayTime.Pause(); + mTotalVideoHDRPlayTime.Pause(); + } + if (mMediaContent & MediaContent::MEDIA_HAS_AUDIO) { + LOG("Pause audio time accumulation for total play time"); + if (mInaudibleAudioPlayTime.IsStarted()) { + LOG("Pause audible audio time accumulation for total play time"); + PauseInaudibleAudioTimeAccumulator(); + } + if (mMutedAudioPlayTime.IsStarted()) { + LOG("Pause muted audio time accumulation for total play time"); + PauseMutedAudioTimeAccumulator(); + } + mTotalAudioPlayTime.Pause(); + } + + mOwner->DispatchAsyncTestingEvent(u"moztotalplaytimepaused"_ns); + ReportTelemetry(); + + mIsPlaying = false; +} + +void TelemetryProbesReporter::OnVisibilityChanged(Visibility aVisibility) { + AssertOnMainThreadAndNotShutdown(); + LOG("Corresponding media element visibility change=%s -> %s", + ToVisibilityStr(mMediaElementVisibility), ToVisibilityStr(aVisibility)); + if (aVisibility == Visibility::eInvisible) { + StartInvisibleVideoTimeAccumulator(); + } else { + if (aVisibility != Visibility::eInitial) { + PauseInvisibleVideoTimeAccumulator(); + } else { + LOG("Visibility was initial, not pausing."); + } + } + mMediaElementVisibility = aVisibility; +} + +void TelemetryProbesReporter::OnAudibleChanged(AudibleState aAudibleState) { + AssertOnMainThreadAndNotShutdown(); + LOG("Audibility changed, now %s", ToAudibilityStr(aAudibleState)); + if (aAudibleState == AudibleState::eNotAudible) { + if (!mInaudibleAudioPlayTime.IsStarted()) { + StartInaudibleAudioTimeAccumulator(); + } + } else { + // This happens when starting playback, no need to pause, because it hasn't + // been started yet. + if (mInaudibleAudioPlayTime.IsStarted()) { + PauseInaudibleAudioTimeAccumulator(); + } + } +} + +void TelemetryProbesReporter::OnMutedChanged(bool aMuted) { + // There are multiple ways to mute an element: + // - volume = 0 + // - muted = true + // - set the enabled property of the playing AudioTrack to false + // Muted -> Muted "transisition" can therefore happen, and we can't add + // asserts here. + AssertOnMainThreadAndNotShutdown(); + if (!(mMediaContent & MediaContent::MEDIA_HAS_AUDIO)) { + return; + } + LOG("Muted changed, was %s now %s", ToMutedStr(mIsMuted), ToMutedStr(aMuted)); + if (aMuted) { + if (!mMutedAudioPlayTime.IsStarted()) { + StartMutedAudioTimeAccumulator(); + } + } else { + // This happens when starting playback, no need to pause, because it hasn't + // been started yet. + if (mMutedAudioPlayTime.IsStarted()) { + PauseMutedAudioTimeAccumulator(); + } + } + mIsMuted = aMuted; +} + +void TelemetryProbesReporter::OnMediaContentChanged(MediaContent aContent) { + AssertOnMainThreadAndNotShutdown(); + if (aContent == mMediaContent) { + return; + } + if (mMediaContent & MediaContent::MEDIA_HAS_VIDEO && + !(aContent & MediaContent::MEDIA_HAS_VIDEO)) { + LOG("Video track removed from media."); + if (mInvisibleVideoPlayTime.IsStarted()) { + PauseInvisibleVideoTimeAccumulator(); + } + if (mTotalVideoPlayTime.IsStarted()) { + mTotalVideoPlayTime.Pause(); + mTotalVideoHDRPlayTime.Pause(); + } + } + if (mMediaContent & MediaContent::MEDIA_HAS_AUDIO && + !(aContent & MediaContent::MEDIA_HAS_AUDIO)) { + LOG("Audio track removed from media."); + if (mTotalAudioPlayTime.IsStarted()) { + mTotalAudioPlayTime.Pause(); + } + if (mInaudibleAudioPlayTime.IsStarted()) { + mInaudibleAudioPlayTime.Pause(); + } + if (mMutedAudioPlayTime.IsStarted()) { + mMutedAudioPlayTime.Pause(); + } + } + if (!(mMediaContent & MediaContent::MEDIA_HAS_VIDEO) && + aContent & MediaContent::MEDIA_HAS_VIDEO) { + LOG("Video track added to media."); + if (mIsPlaying) { + mTotalVideoPlayTime.Start(); + if (mMediaElementVisibility == Visibility::eInvisible) { + StartInvisibleVideoTimeAccumulator(); + } + } + } + if (!(mMediaContent & MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8) && + aContent & MediaContent::MEDIA_HAS_COLOR_DEPTH_ABOVE_8) { + if (mIsPlaying) { + mTotalVideoHDRPlayTime.Start(); + } + } + if (!(mMediaContent & MediaContent::MEDIA_HAS_AUDIO) && + aContent & MediaContent::MEDIA_HAS_AUDIO) { + LOG("Audio track added to media."); + if (mIsPlaying) { + mTotalAudioPlayTime.Start(); + if (mIsMuted) { + StartMutedAudioTimeAccumulator(); + } + } + } + + mMediaContent = aContent; +} + +void TelemetryProbesReporter::OnDecodeSuspended() { + AssertOnMainThreadAndNotShutdown(); + // Suspended time should only be counted after starting accumulating invisible + // time. + if (!mInvisibleVideoPlayTime.IsStarted()) { + return; + } + LOG("Start time accumulation for video decoding suspension"); + mVideoDecodeSuspendedTime.Start(); + mOwner->DispatchAsyncTestingEvent(u"mozvideodecodesuspendedstarted"_ns); +} + +void TelemetryProbesReporter::OnDecodeResumed() { + AssertOnMainThreadAndNotShutdown(); + if (!mVideoDecodeSuspendedTime.IsStarted()) { + return; + } + LOG("Pause time accumulation for video decoding suspension"); + mVideoDecodeSuspendedTime.Pause(); + mOwner->DispatchAsyncTestingEvent(u"mozvideodecodesuspendedpaused"_ns); +} + +void TelemetryProbesReporter::OnShutdown() { + AssertOnMainThreadAndNotShutdown(); + LOG("Shutdown"); + OnPause(Visibility::eInvisible); + mOwner = nullptr; +} + +void TelemetryProbesReporter::StartInvisibleVideoTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + if (!mTotalVideoPlayTime.IsStarted() || mInvisibleVideoPlayTime.IsStarted() || + !HasOwnerHadValidVideo()) { + return; + } + LOG("Start time accumulation for invisible video"); + mInvisibleVideoPlayTime.Start(); + mOwner->DispatchAsyncTestingEvent(u"mozinvisibleplaytimestarted"_ns); +} + +void TelemetryProbesReporter::PauseInvisibleVideoTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + if (!mInvisibleVideoPlayTime.IsStarted()) { + return; + } + OnDecodeResumed(); + LOG("Pause time accumulation for invisible video"); + mInvisibleVideoPlayTime.Pause(); + mOwner->DispatchAsyncTestingEvent(u"mozinvisibleplaytimepaused"_ns); +} + +void TelemetryProbesReporter::StartInaudibleAudioTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT(!mInaudibleAudioPlayTime.IsStarted()); + mInaudibleAudioPlayTime.Start(); + mOwner->DispatchAsyncTestingEvent(u"mozinaudibleaudioplaytimestarted"_ns); +} + +void TelemetryProbesReporter::PauseInaudibleAudioTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT(mInaudibleAudioPlayTime.IsStarted()); + mInaudibleAudioPlayTime.Pause(); + mOwner->DispatchAsyncTestingEvent(u"mozinaudibleaudioplaytimepaused"_ns); +} + +void TelemetryProbesReporter::StartMutedAudioTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT(!mMutedAudioPlayTime.IsStarted()); + mMutedAudioPlayTime.Start(); + mOwner->DispatchAsyncTestingEvent(u"mozmutedaudioplaytimestarted"_ns); +} + +void TelemetryProbesReporter::PauseMutedAudioTimeAccumulator() { + AssertOnMainThreadAndNotShutdown(); + MOZ_ASSERT(mMutedAudioPlayTime.IsStarted()); + mMutedAudioPlayTime.Pause(); + mOwner->DispatchAsyncTestingEvent(u"mozmutedeaudioplaytimepaused"_ns); +} + +bool TelemetryProbesReporter::HasOwnerHadValidVideo() const { + // Checking both image and display dimensions helps address cases such as + // suspending, where we use a null decoder. In that case a null decoder + // produces 0x0 video frames, which might cause layout to resize the display + // size, but the image dimensions would be still non-null. + const VideoInfo info = mOwner->GetMediaInfo().mVideo; + return (info.mDisplay.height > 0 && info.mDisplay.width > 0) || + (info.mImage.height > 0 && info.mImage.width > 0); +} + +bool TelemetryProbesReporter::HasOwnerHadValidMedia() const { + return mMediaContent != MediaContent::MEDIA_HAS_NOTHING; +} + +void TelemetryProbesReporter::AssertOnMainThreadAndNotShutdown() const { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mOwner, "Already shutdown?"); +} + +void TelemetryProbesReporter::ReportTelemetry() { + AssertOnMainThreadAndNotShutdown(); + // ReportResultForAudio needs to be called first, because it can use the video + // play time, that is reset in ReportResultForVideo. + ReportResultForAudio(); + ReportResultForVideo(); + mOwner->DispatchAsyncTestingEvent(u"mozreportedtelemetry"_ns); +} + +void TelemetryProbesReporter::ReportResultForVideo() { + // We don't want to know the result for video without valid video frames. + if (!HasOwnerHadValidVideo()) { + return; + } + + const double totalVideoPlayTimeS = mTotalVideoPlayTime.GetAndClearTotal(); + const double invisiblePlayTimeS = mInvisibleVideoPlayTime.GetAndClearTotal(); + const double videoDecodeSuspendTimeS = + mVideoDecodeSuspendedTime.GetAndClearTotal(); + const double totalVideoHDRPlayTimeS = + mTotalVideoHDRPlayTime.GetAndClearTotal(); + + // No need to report result for video that didn't start playing. + if (totalVideoPlayTimeS == 0.0) { + return; + } + MOZ_ASSERT(totalVideoPlayTimeS >= invisiblePlayTimeS); + + LOG("VIDEO_PLAY_TIME_S = %f", totalVideoPlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_PLAY_TIME_MS, + SECONDS_TO_MS(totalVideoPlayTimeS)); + + LOG("VIDEO_HIDDEN_PLAY_TIME_S = %f", invisiblePlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_MS, + SECONDS_TO_MS(invisiblePlayTimeS)); + + // We only want to accumulate non-zero samples for HDR playback. + // This is different from the other timings tracked here, but + // we don't need 0-length play times to do our calculations. + if (totalVideoHDRPlayTimeS > 0.0) { + LOG("VIDEO_HDR_PLAY_TIME_S = %f", totalVideoHDRPlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_HDR_PLAY_TIME_MS, + SECONDS_TO_MS(totalVideoHDRPlayTimeS)); + } + + if (mOwner->IsEncrypted()) { + LOG("VIDEO_ENCRYPTED_PLAY_TIME_S = %f", totalVideoPlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_ENCRYPTED_PLAY_TIME_MS, + SECONDS_TO_MS(totalVideoPlayTimeS)); + } + + // Report result for video using CDM + auto keySystem = mOwner->GetKeySystem(); + if (keySystem) { + if (IsClearkeyKeySystem(*keySystem)) { + LOG("VIDEO_CLEARKEY_PLAY_TIME_S = %f", totalVideoPlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_CLEARKEY_PLAY_TIME_MS, + SECONDS_TO_MS(totalVideoPlayTimeS)); + + } else if (IsWidevineKeySystem(*keySystem)) { + LOG("VIDEO_WIDEVINE_PLAY_TIME_S = %f", totalVideoPlayTimeS); + Telemetry::Accumulate(Telemetry::VIDEO_WIDEVINE_PLAY_TIME_MS, + SECONDS_TO_MS(totalVideoPlayTimeS)); + } + } + + // Keyed by audio+video or video alone, and by a resolution range. + const MediaInfo& info = mOwner->GetMediaInfo(); + nsCString key; + DetermineResolutionForTelemetry(info, key); + + auto visiblePlayTimeS = totalVideoPlayTimeS - invisiblePlayTimeS; + LOG("VIDEO_VISIBLE_PLAY_TIME = %f, keys: '%s' and 'All'", visiblePlayTimeS, + key.get()); + Telemetry::Accumulate(Telemetry::VIDEO_VISIBLE_PLAY_TIME_MS, key, + SECONDS_TO_MS(visiblePlayTimeS)); + // Also accumulate result in an "All" key. + Telemetry::Accumulate(Telemetry::VIDEO_VISIBLE_PLAY_TIME_MS, "All"_ns, + SECONDS_TO_MS(visiblePlayTimeS)); + + const uint32_t hiddenPercentage = + lround(invisiblePlayTimeS / totalVideoPlayTimeS * 100.0); + Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE, key, + hiddenPercentage); + // Also accumulate all percentages in an "All" key. + Telemetry::Accumulate(Telemetry::VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE, "All"_ns, + hiddenPercentage); + LOG("VIDEO_HIDDEN_PLAY_TIME_PERCENTAGE = %u, keys: '%s' and 'All'", + hiddenPercentage, key.get()); + + const uint32_t videoDecodeSuspendPercentage = + lround(videoDecodeSuspendTimeS / totalVideoPlayTimeS * 100.0); + Telemetry::Accumulate(Telemetry::VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE, + key, videoDecodeSuspendPercentage); + Telemetry::Accumulate(Telemetry::VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE, + "All"_ns, videoDecodeSuspendPercentage); + LOG("VIDEO_INFERRED_DECODE_SUSPEND_PERCENTAGE = %u, keys: '%s' and 'All'", + videoDecodeSuspendPercentage, key.get()); + + ReportResultForVideoFrameStatistics(totalVideoPlayTimeS, key); +#ifdef MOZ_WMF_CDM + if (mOwner->IsUsingWMFCDM()) { + ReportResultForMFCDMPlaybackIfNeeded(totalVideoPlayTimeS, key); + } +#endif +} + +#ifdef MOZ_WMF_CDM +void TelemetryProbesReporter::ReportResultForMFCDMPlaybackIfNeeded( + double aTotalPlayTimeS, const nsCString& aResolution) { + const auto keySystem = mOwner->GetKeySystem(); + if (!keySystem) { + NS_WARNING("Can not find key system to report telemetry for MFCDM!!"); + return; + } + glean::mfcdm::EmePlaybackExtra extraData; + extraData.keySystem = Some(NS_ConvertUTF16toUTF8(*keySystem)); + extraData.videoCodec = Some(mOwner->GetMediaInfo().mVideo.mMimeType); + extraData.resolution = Some(aResolution); + extraData.playedTime = Some(aTotalPlayTimeS); + + Maybe<uint64_t> renderedFrames; + Maybe<uint64_t> droppedFrames; + if (auto* stats = mOwner->GetFrameStatistics()) { + renderedFrames = Some(stats->GetPresentedFrames()); + droppedFrames = Some(stats->GetDroppedFrames()); + extraData.renderedFrames = Some(*renderedFrames); + extraData.droppedFrames = Some(*droppedFrames); + } + if (MOZ_LOG_TEST(gTelemetryProbesReporterLog, LogLevel::Debug)) { + nsPrintfCString logMessage{ + "MFCDM EME_Playback event, keySystem=%s, videoCodec=%s, resolution=%s, " + "playedTime=%s", + NS_ConvertUTF16toUTF8(*keySystem).get(), + mOwner->GetMediaInfo().mVideo.mMimeType.get(), aResolution.get(), + std::to_string(aTotalPlayTimeS).c_str()}; + if (renderedFrames) { + logMessage.Append( + nsPrintfCString{", renderedFrames=%" PRIu64, *renderedFrames}); + } + if (droppedFrames) { + logMessage.Append( + nsPrintfCString{", droppedFrames=%" PRIu64, *droppedFrames}); + } + LOG("%s", logMessage.get()); + } + glean::mfcdm::eme_playback.Record(Some(extraData)); +} +#endif + +void TelemetryProbesReporter::ReportResultForAudio() { + // Don't record telemetry for a media that didn't have a valid audio or video + // to play, or hasn't played. + if (!HasOwnerHadValidMedia() || (mTotalAudioPlayTime.PeekTotal() == 0.0 && + mTotalVideoPlayTime.PeekTotal() == 0.0)) { + return; + } + + nsCString key; + nsCString avKey; + const double totalAudioPlayTimeS = mTotalAudioPlayTime.GetAndClearTotal(); + const double inaudiblePlayTimeS = mInaudibleAudioPlayTime.GetAndClearTotal(); + const double mutedPlayTimeS = mMutedAudioPlayTime.GetAndClearTotal(); + const double audiblePlayTimeS = totalAudioPlayTimeS - inaudiblePlayTimeS; + const double unmutedPlayTimeS = totalAudioPlayTimeS - mutedPlayTimeS; + const uint32_t audiblePercentage = + lround(audiblePlayTimeS / totalAudioPlayTimeS * 100.0); + const uint32_t unmutedPercentage = + lround(unmutedPlayTimeS / totalAudioPlayTimeS * 100.0); + const double totalVideoPlayTimeS = mTotalVideoPlayTime.PeekTotal(); + + // Key semantics: + // - AV: Audible audio + video + // - IV: Inaudible audio + video + // - MV: Muted audio + video + // - A: Audible audio-only + // - I: Inaudible audio-only + // - M: Muted audio-only + // - V: Video-only + if (mMediaContent & MediaContent::MEDIA_HAS_AUDIO) { + if (audiblePercentage == 0) { + // Media element had an audio track, but it was inaudible throughout + key.AppendASCII("I"); + } else if (unmutedPercentage == 0) { + // Media element had an audio track, but it was muted throughout + key.AppendASCII("M"); + } else { + // Media element had an audible audio track + key.AppendASCII("A"); + } + avKey.AppendASCII("A"); + } + if (mMediaContent & MediaContent::MEDIA_HAS_VIDEO) { + key.AppendASCII("V"); + avKey.AppendASCII("V"); + } + + LOG("Key: %s", key.get()); + + if (mMediaContent & MediaContent::MEDIA_HAS_AUDIO) { + LOG("Audio:\ntotal: %lf\naudible: %lf\ninaudible: %lf\nmuted: " + "%lf\npercentage audible: " + "%u\npercentage unmuted: %u\n", + totalAudioPlayTimeS, audiblePlayTimeS, inaudiblePlayTimeS, + mutedPlayTimeS, audiblePercentage, unmutedPercentage); + Telemetry::Accumulate(Telemetry::MEDIA_PLAY_TIME_MS, key, + SECONDS_TO_MS(totalAudioPlayTimeS)); + Telemetry::Accumulate(Telemetry::MUTED_PLAY_TIME_PERCENT, avKey, + 100 - unmutedPercentage); + Telemetry::Accumulate(Telemetry::AUDIBLE_PLAY_TIME_PERCENT, avKey, + audiblePercentage); + } else { + MOZ_ASSERT(mMediaContent & MediaContent::MEDIA_HAS_VIDEO); + Telemetry::Accumulate(Telemetry::MEDIA_PLAY_TIME_MS, key, + SECONDS_TO_MS(totalVideoPlayTimeS)); + } +} + +void TelemetryProbesReporter::ReportResultForVideoFrameStatistics( + double aTotalPlayTimeS, const nsCString& key) { + FrameStatistics* stats = mOwner->GetFrameStatistics(); + if (!stats) { + return; + } + + FrameStatisticsData data = stats->GetFrameStatisticsData(); + if (data.mInterKeyframeCount != 0) { + const uint32_t average_ms = uint32_t( + std::min<uint64_t>(lround(double(data.mInterKeyframeSum_us) / + double(data.mInterKeyframeCount) / 1000.0), + UINT32_MAX)); + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_AVERAGE_MS, key, + average_ms); + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_AVERAGE_MS, "All"_ns, + average_ms); + LOG("VIDEO_INTER_KEYFRAME_AVERAGE_MS = %u, keys: '%s' and 'All'", + average_ms, key.get()); + + const uint32_t max_ms = uint32_t(std::min<uint64_t>( + (data.mInterKeyFrameMax_us + 500) / 1000, UINT32_MAX)); + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, key, max_ms); + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, "All"_ns, + max_ms); + LOG("VIDEO_INTER_KEYFRAME_MAX_MS = %u, keys: '%s' and 'All'", max_ms, + key.get()); + } else { + // Here, we have played *some* of the video, but didn't get more than 1 + // keyframe. Report '0' if we have played for longer than the video- + // decode-suspend delay (showing recovery would be difficult). + const uint32_t suspendDelay_ms = + StaticPrefs::media_suspend_background_video_delay_ms(); + if (uint32_t(aTotalPlayTimeS * 1000.0) > suspendDelay_ms) { + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, key, 0); + Telemetry::Accumulate(Telemetry::VIDEO_INTER_KEYFRAME_MAX_MS, "All"_ns, + 0); + LOG("VIDEO_INTER_KEYFRAME_MAX_MS = 0 (only 1 keyframe), keys: '%s' and " + "'All'", + key.get()); + } + } + + const uint64_t parsedFrames = stats->GetParsedFrames(); + if (parsedFrames) { + const uint64_t droppedFrames = stats->GetDroppedFrames(); + MOZ_ASSERT(droppedFrames <= parsedFrames); + // Dropped frames <= total frames, so 'percentage' cannot be higher than + // 100 and therefore can fit in a uint32_t (that Telemetry takes). + const uint32_t percentage = 100 * droppedFrames / parsedFrames; + LOG("DROPPED_FRAMES_IN_VIDEO_PLAYBACK = %u", percentage); + Telemetry::Accumulate(Telemetry::VIDEO_DROPPED_FRAMES_PROPORTION, + percentage); + const uint32_t proportion = 10000 * droppedFrames / parsedFrames; + Telemetry::Accumulate( + Telemetry::VIDEO_DROPPED_FRAMES_PROPORTION_EXPONENTIAL, proportion); + + { + const uint64_t droppedFrames = stats->GetDroppedDecodedFrames(); + const uint32_t proportion = 10000 * droppedFrames / parsedFrames; + Telemetry::Accumulate( + Telemetry::VIDEO_DROPPED_DECODED_FRAMES_PROPORTION_EXPONENTIAL, + proportion); + } + { + const uint64_t droppedFrames = stats->GetDroppedSinkFrames(); + const uint32_t proportion = 10000 * droppedFrames / parsedFrames; + Telemetry::Accumulate( + Telemetry::VIDEO_DROPPED_SINK_FRAMES_PROPORTION_EXPONENTIAL, + proportion); + } + { + const uint64_t droppedFrames = stats->GetDroppedCompositorFrames(); + const uint32_t proportion = 10000 * droppedFrames / parsedFrames; + Telemetry::Accumulate( + Telemetry::VIDEO_DROPPED_COMPOSITOR_FRAMES_PROPORTION_EXPONENTIAL, + proportion); + } + } +} + +double TelemetryProbesReporter::GetTotalVideoPlayTimeInSeconds() const { + return mTotalVideoPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetTotalVideoHDRPlayTimeInSeconds() const { + return mTotalVideoHDRPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetVisibleVideoPlayTimeInSeconds() const { + return GetTotalVideoPlayTimeInSeconds() - + GetInvisibleVideoPlayTimeInSeconds(); +} + +double TelemetryProbesReporter::GetInvisibleVideoPlayTimeInSeconds() const { + return mInvisibleVideoPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetVideoDecodeSuspendedTimeInSeconds() const { + return mVideoDecodeSuspendedTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetTotalAudioPlayTimeInSeconds() const { + return mTotalAudioPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetInaudiblePlayTimeInSeconds() const { + return mInaudibleAudioPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetMutedPlayTimeInSeconds() const { + return mMutedAudioPlayTime.PeekTotal(); +} + +double TelemetryProbesReporter::GetAudiblePlayTimeInSeconds() const { + return GetTotalAudioPlayTimeInSeconds() - GetInaudiblePlayTimeInSeconds(); +} + +#undef LOG +} // namespace mozilla diff --git a/dom/media/utils/TelemetryProbesReporter.h b/dom/media/utils/TelemetryProbesReporter.h new file mode 100644 index 0000000000..51ae2f12bb --- /dev/null +++ b/dom/media/utils/TelemetryProbesReporter.h @@ -0,0 +1,179 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_TelemetryProbesReporter_H_ +#define DOM_TelemetryProbesReporter_H_ + +#include "MediaInfo.h" +#include "mozilla/Maybe.h" +#include "mozilla/AwakeTimeStamp.h" +#include "AudioChannelService.h" +#include "nsISupportsImpl.h" + +namespace mozilla { +class FrameStatistics; + +class TelemetryProbesReporterOwner { + public: + virtual Maybe<nsAutoString> GetKeySystem() const = 0; + virtual MediaInfo GetMediaInfo() const = 0; + virtual FrameStatistics* GetFrameStatistics() const = 0; + virtual bool IsEncrypted() const = 0; + virtual void DispatchAsyncTestingEvent(const nsAString& aName) = 0; +#ifdef MOZ_WMF_CDM + virtual bool IsUsingWMFCDM() const = 0; +#endif +}; + +enum class MediaContent : uint8_t { + MEDIA_HAS_NOTHING = (0 << 0), + MEDIA_HAS_VIDEO = (1 << 0), + MEDIA_HAS_AUDIO = (1 << 1), + MEDIA_HAS_COLOR_DEPTH_ABOVE_8 = (1 << 2), +}; + +MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(MediaContent) + +/** + * This class is used for collecting and reporting telemetry probes for + * its owner which should inherit from TelemetryProbesReporterOwner. We use it + * for HTMLMediaElement, and each element has one corresponding reporter. + */ +class TelemetryProbesReporter final { + public: + explicit TelemetryProbesReporter(TelemetryProbesReporterOwner* aOwner); + ~TelemetryProbesReporter() = default; + + enum class Visibility { + eInitial, + eVisible, + eInvisible, + }; + + static MediaContent MediaInfoToMediaContent(const MediaInfo& aInfo); + + using AudibleState = dom::AudioChannelService::AudibleState; + + // State transitions + void OnPlay(Visibility aVisibility, MediaContent aContent, bool aIsMuted); + void OnPause(Visibility aVisibility); + void OnShutdown(); + + void OnVisibilityChanged(Visibility aVisibility); + void OnAudibleChanged(AudibleState aAudible); + void OnMediaContentChanged(MediaContent aContent); + void OnMutedChanged(bool aMuted); + void OnDecodeSuspended(); + void OnDecodeResumed(); + + double GetTotalVideoPlayTimeInSeconds() const; + double GetTotalVideoHDRPlayTimeInSeconds() const; + double GetVisibleVideoPlayTimeInSeconds() const; + double GetInvisibleVideoPlayTimeInSeconds() const; + double GetVideoDecodeSuspendedTimeInSeconds() const; + + double GetTotalAudioPlayTimeInSeconds() const; + double GetInaudiblePlayTimeInSeconds() const; + double GetAudiblePlayTimeInSeconds() const; + double GetMutedPlayTimeInSeconds() const; + + private: + void StartInvisibleVideoTimeAccumulator(); + void PauseInvisibleVideoTimeAccumulator(); + void StartInaudibleAudioTimeAccumulator(); + void PauseInaudibleAudioTimeAccumulator(); + void StartMutedAudioTimeAccumulator(); + void PauseMutedAudioTimeAccumulator(); + bool HasOwnerHadValidVideo() const; + bool HasOwnerHadValidMedia() const; + void AssertOnMainThreadAndNotShutdown() const; + + void ReportTelemetry(); + void ReportResultForVideo(); + void ReportResultForAudio(); + void ReportResultForVideoFrameStatistics(double aTotalPlayTimeS, + const nsCString& key); +#ifdef MOZ_WMF_CDM + void ReportResultForMFCDMPlaybackIfNeeded(double aTotalPlayTimeS, + const nsCString& aResolution); +#endif + + // Helper class to measure times for playback telemetry stats + class TimeDurationAccumulator { + public: + TimeDurationAccumulator() = default; + void Start() { + if (IsStarted()) { + return; + } + mStartTime = Some(AwakeTimeStamp::NowLoRes()); + } + void Pause() { + if (!IsStarted()) { + return; + } + mSum = (AwakeTimeStamp::NowLoRes() - mStartTime.value()); + mStartTime = Nothing(); + } + bool IsStarted() const { return mStartTime.isSome(); } + + double GetAndClearTotal() { + MOZ_ASSERT(!IsStarted(), "only call this when accumulator is paused"); + double total = mSum.ToSeconds(); + mStartTime = Nothing(); + mSum = AwakeTimeDuration(); + return total; + } + + double PeekTotal() const { + if (!IsStarted()) { + return mSum.ToSeconds(); + } + return (AwakeTimeStamp::NowLoRes() - mStartTime.value()).ToSeconds(); + } + + private: + Maybe<AwakeTimeStamp> mStartTime; + AwakeTimeDuration mSum; + }; + + // The owner is HTMLMediaElement that is guaranteed being always alive during + // our whole life cycle. + TelemetryProbesReporterOwner* mOwner; + + // Total time an element has spent on playing video. + TimeDurationAccumulator mTotalVideoPlayTime; + + // Total time an element has spent on playing video that has a color depth + // greater than 8, which is likely HDR video. + TimeDurationAccumulator mTotalVideoHDRPlayTime; + + // Total time an element has spent on playing audio + TimeDurationAccumulator mTotalAudioPlayTime; + + // Total time a VIDEO element has spent playing while the corresponding media + // element is invisible. + TimeDurationAccumulator mInvisibleVideoPlayTime; + + // Total time an element has spent playing audio that was not audible + TimeDurationAccumulator mInaudibleAudioPlayTime; + + // Total time an element with an audio track has spent muted + TimeDurationAccumulator mMutedAudioPlayTime; + + // Total time a VIDEO has spent in video-decode-suspend mode. + TimeDurationAccumulator mVideoDecodeSuspendedTime; + + Visibility mMediaElementVisibility = Visibility::eInitial; + + MediaContent mMediaContent = MediaContent::MEDIA_HAS_NOTHING; + + bool mIsPlaying = false; + + bool mIsMuted = false; +}; + +} // namespace mozilla + +#endif // DOM_TelemetryProbesReporter_H_ diff --git a/dom/media/utils/gtest/TestPerformanceRecorder.cpp b/dom/media/utils/gtest/TestPerformanceRecorder.cpp new file mode 100644 index 0000000000..ae5d22a916 --- /dev/null +++ b/dom/media/utils/gtest/TestPerformanceRecorder.cpp @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include <chrono> +#include <thread> + +#include "PerformanceRecorder.h" +#include "gtest/gtest.h" +#include "nsString.h" + +using namespace mozilla; + +class PerformanceRecorderWrapper : public PerformanceRecorder<PlaybackStage> { + public: + PerformanceRecorderWrapper(MediaStage aStage, int32_t aHeight) + : PerformanceRecorder(aStage, aHeight) {} + + static void EnableMeasurementOnNonMarkerSituation() { + sEnableMeasurementForTesting = true; + } +}; + +TEST(PerformanceRecorder, TestResolution) +{ + PerformanceRecorderWrapper::EnableMeasurementOnNonMarkerSituation(); + + static const struct { + const int32_t mH; + const char* mRes; + } resolutions[] = {{0, "A:0"}, + {240, "V:0<h<=240"}, + {480, "V:240<h<=480"}, + {576, "V:480<h<=576"}, + {720, "V:576<h<=720"}, + {1080, "V:720<h<=1080"}, + {1440, "V:1080<h<=1440"}, + {2160, "V:1440<h<=2160"}, + {4320, "V:h>2160"}}; + + const MediaStage stage = MediaStage::RequestDecode; + for (auto&& res : resolutions) { + PerformanceRecorderWrapper w(stage, res.mH); + nsCString name; + w.Record([&](auto& aStage) { name = nsCString(aStage.Name()); }); + ASSERT_NE(name.Find(res.mRes), kNotFound); + } +} + +TEST(PerformanceRecorder, TestMoveOperation) +{ + PerformanceRecorderWrapper::EnableMeasurementOnNonMarkerSituation(); + + const MediaStage stage = MediaStage::RequestDecode; + const uint32_t resolution = 1080; + PerformanceRecorderWrapper w1(stage, resolution); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + // w1 has been moved which won't continue measuring data. + PerformanceRecorderWrapper w2(std::move(w1)); + ASSERT_DOUBLE_EQ(w1.Record(), 0.0); + ASSERT_TRUE(w2.Record() > 0.0); +} + +TEST(PerformanceRecorder, TestRecordInvalidation) +{ + PerformanceRecorderWrapper::EnableMeasurementOnNonMarkerSituation(); + + const MediaStage stage = MediaStage::RequestDecode; + const uint32_t resolution = 1080; + PerformanceRecorderWrapper w(stage, resolution); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + ASSERT_TRUE(w.Record() > 0.0); + + w.Record(); + // w has been recorded and won't continue measuring data. + ASSERT_DOUBLE_EQ(w.Record(), 0.0); +} + +TEST(PerformanceRecorder, TestMultipleRecords) +{ + PerformanceRecorderWrapper::EnableMeasurementOnNonMarkerSituation(); + + const MediaStage stage = MediaStage::RequestDecode; + PerformanceRecorderMulti<PlaybackStage> r; + + r.Start(1, stage, 1); + r.Start(2, stage, 2); + r.Start(3, stage, 3); + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + // id 0 wasn't started + EXPECT_DOUBLE_EQ(r.Record(0), 0.0); + + // id 1 gets recorded normally + EXPECT_TRUE(r.Record(1) > 0.0); + + // id 1 was already recorded + EXPECT_DOUBLE_EQ(r.Record(1), 0.0); + + // id 2 gets recorded normally + EXPECT_TRUE(r.Record(2) > 0.0); + + // id 4 wasn't started + EXPECT_DOUBLE_EQ(r.Record(4), 0.0); + + // All lower ids got discarded + EXPECT_DOUBLE_EQ(r.Record(3), 0.0); +} diff --git a/dom/media/utils/gtest/moz.build b/dom/media/utils/gtest/moz.build new file mode 100644 index 0000000000..b046869f40 --- /dev/null +++ b/dom/media/utils/gtest/moz.build @@ -0,0 +1,15 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "TestPerformanceRecorder.cpp", +] + +LOCAL_INCLUDES += [ + "/dom/media/utils", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/dom/media/utils/moz.build b/dom/media/utils/moz.build new file mode 100644 index 0000000000..e503c29949 --- /dev/null +++ b/dom/media/utils/moz.build @@ -0,0 +1,26 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("test/**"): + BUG_COMPONENT = ("Core", "Audio/Video: Playback") + +EXPORTS += [ + "MediaElementEventRunners.h", + "PerformanceRecorder.h", + "TelemetryProbesReporter.h", +] + +UNIFIED_SOURCES += [ + "MediaElementEventRunners.cpp", + "PerformanceRecorder.cpp", + "TelemetryProbesReporter.cpp", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +if CONFIG["ENABLE_TESTS"]: + DIRS += ["gtest"] + +FINAL_LIBRARY = "xul" |