summaryrefslogtreecommitdiffstats
path: root/xpcom/base/MemoryTelemetry.cpp
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--xpcom/base/MemoryTelemetry.cpp520
1 files changed, 520 insertions, 0 deletions
diff --git a/xpcom/base/MemoryTelemetry.cpp b/xpcom/base/MemoryTelemetry.cpp
new file mode 100644
index 0000000000..e9b6bff80e
--- /dev/null
+++ b/xpcom/base/MemoryTelemetry.cpp
@@ -0,0 +1,520 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MemoryTelemetry.h"
+#include "nsMemoryReporterManager.h"
+
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/Result.h"
+#include "mozilla/ResultExtensions.h"
+#include "mozilla/Services.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/SimpleEnumerator.h"
+#include "mozilla/Telemetry.h"
+#include "mozilla/TimeStamp.h"
+#include "mozilla/dom/ContentParent.h"
+#include "mozilla/dom/ScriptSettings.h"
+#include "nsContentUtils.h"
+#include "nsIBrowserDOMWindow.h"
+#include "nsIDOMChromeWindow.h"
+#include "nsIMemoryReporter.h"
+#include "nsIWindowMediator.h"
+#include "nsImportModule.h"
+#include "nsITelemetry.h"
+#include "nsNetCID.h"
+#include "nsObserverService.h"
+#include "nsReadableUtils.h"
+#include "nsThreadUtils.h"
+#include "nsXULAppAPI.h"
+#include "xpcpublic.h"
+
+#include <cstdlib>
+
+using namespace mozilla;
+
+using mozilla::dom::AutoJSAPI;
+using mozilla::dom::ContentParent;
+
+// Do not gather data more than once a minute (ms)
+static constexpr uint32_t kTelemetryInterval = 60 * 1000;
+
+static constexpr const char* kTopicCycleCollectorBegin =
+ "cycle-collector-begin";
+
+namespace {
+
+enum class PrevValue : uint32_t {
+#ifdef XP_WIN
+ LOW_MEMORY_EVENTS_VIRTUAL,
+ LOW_MEMORY_EVENTS_COMMIT_SPACE,
+ LOW_MEMORY_EVENTS_PHYSICAL,
+#endif
+#if defined(XP_LINUX) && !defined(ANDROID)
+ PAGE_FAULTS_HARD,
+#endif
+ SIZE_,
+};
+
+} // anonymous namespace
+
+constexpr uint32_t kUninitialized = ~0;
+
+static uint32_t gPrevValues[uint32_t(PrevValue::SIZE_)];
+
+static uint32_t PrevValueIndex(Telemetry::HistogramID aId) {
+ switch (aId) {
+#ifdef XP_WIN
+ case Telemetry::LOW_MEMORY_EVENTS_VIRTUAL:
+ return uint32_t(PrevValue::LOW_MEMORY_EVENTS_VIRTUAL);
+ case Telemetry::LOW_MEMORY_EVENTS_COMMIT_SPACE:
+ return uint32_t(PrevValue::LOW_MEMORY_EVENTS_COMMIT_SPACE);
+ case Telemetry::LOW_MEMORY_EVENTS_PHYSICAL:
+ return uint32_t(PrevValue::LOW_MEMORY_EVENTS_PHYSICAL);
+#endif
+#if defined(XP_LINUX) && !defined(ANDROID)
+ case Telemetry::PAGE_FAULTS_HARD:
+ return uint32_t(PrevValue::PAGE_FAULTS_HARD);
+#endif
+ default:
+ MOZ_ASSERT_UNREACHABLE("Unexpected histogram ID");
+ return 0;
+ }
+}
+
+NS_IMPL_ISUPPORTS(MemoryTelemetry, nsIObserver, nsISupportsWeakReference)
+
+MemoryTelemetry::MemoryTelemetry()
+ : mThreadPool(do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID)) {}
+
+void MemoryTelemetry::Init() {
+ for (auto& val : gPrevValues) {
+ val = kUninitialized;
+ }
+
+ if (XRE_IsContentProcess()) {
+ nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
+ MOZ_RELEASE_ASSERT(obs);
+
+ obs->AddObserver(this, "content-child-shutdown", true);
+ }
+}
+
+/* static */ MemoryTelemetry& MemoryTelemetry::Get() {
+ static RefPtr<MemoryTelemetry> sInstance;
+
+ MOZ_ASSERT(NS_IsMainThread());
+
+ if (!sInstance) {
+ sInstance = new MemoryTelemetry();
+ sInstance->Init();
+ ClearOnShutdown(&sInstance);
+ }
+ return *sInstance;
+}
+
+nsresult MemoryTelemetry::DelayedInit() {
+ if (Telemetry::CanRecordExtended()) {
+ nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
+ MOZ_RELEASE_ASSERT(obs);
+
+ obs->AddObserver(this, kTopicCycleCollectorBegin, true);
+ }
+
+ GatherReports();
+
+ return NS_OK;
+}
+
+nsresult MemoryTelemetry::Shutdown() {
+ nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
+ MOZ_RELEASE_ASSERT(obs);
+
+ obs->RemoveObserver(this, kTopicCycleCollectorBegin);
+
+ return NS_OK;
+}
+
+static inline void HandleMemoryReport(Telemetry::HistogramID aId,
+ int32_t aUnits, uint64_t aAmount,
+ const nsCString& aKey = VoidCString()) {
+ uint32_t val;
+ switch (aUnits) {
+ case nsIMemoryReporter::UNITS_BYTES:
+ val = uint32_t(aAmount / 1024);
+ break;
+
+ case nsIMemoryReporter::UNITS_PERCENTAGE:
+ // UNITS_PERCENTAGE amounts are 100x greater than their raw value.
+ val = uint32_t(aAmount / 100);
+ break;
+
+ case nsIMemoryReporter::UNITS_COUNT:
+ val = uint32_t(aAmount);
+ break;
+
+ case nsIMemoryReporter::UNITS_COUNT_CUMULATIVE: {
+ // If the reporter gives us a cumulative count, we'll report the
+ // difference in its value between now and our previous ping.
+
+ uint32_t idx = PrevValueIndex(aId);
+ uint32_t prev = gPrevValues[idx];
+ gPrevValues[idx] = aAmount;
+
+ if (prev == kUninitialized) {
+ // If this is the first time we're reading this reporter, store its
+ // current value but don't report it in the telemetry ping, so we
+ // ignore the effect startup had on the reporter.
+ return;
+ }
+ val = aAmount - prev;
+ break;
+ }
+
+ default:
+ MOZ_ASSERT_UNREACHABLE("Unexpected aUnits value");
+ return;
+ }
+
+ // Note: The reference equality check here should allow the compiler to
+ // optimize this case out at compile time when we weren't given a key,
+ // while IsEmpty() or IsVoid() most likely will not.
+ if (&aKey == &VoidCString()) {
+ Telemetry::Accumulate(aId, val);
+ } else {
+ Telemetry::Accumulate(aId, aKey, val);
+ }
+}
+
+nsresult MemoryTelemetry::GatherReports(
+ const std::function<void()>& aCompletionCallback) {
+ auto cleanup = MakeScopeExit([&]() {
+ if (aCompletionCallback) {
+ aCompletionCallback();
+ }
+ });
+
+ RefPtr<nsMemoryReporterManager> mgr = nsMemoryReporterManager::GetOrCreate();
+ MOZ_DIAGNOSTIC_ASSERT(mgr);
+ NS_ENSURE_TRUE(mgr, NS_ERROR_FAILURE);
+
+#define RECORD(id, metric, units) \
+ do { \
+ int64_t amt; \
+ nsresult rv = mgr->Get##metric(&amt); \
+ if (NS_SUCCEEDED(rv)) { \
+ HandleMemoryReport(Telemetry::id, nsIMemoryReporter::units, amt); \
+ } else if (rv != NS_ERROR_NOT_AVAILABLE) { \
+ NS_WARNING("Failed to retrieve memory telemetry for " #metric); \
+ } \
+ } while (0)
+
+ // GHOST_WINDOWS is opt-out as of Firefox 55
+ RECORD(GHOST_WINDOWS, GhostWindows, UNITS_COUNT);
+
+ // If we're running in the parent process, collect data from all processes for
+ // the MEMORY_TOTAL histogram.
+ if (XRE_IsParentProcess() && !mGatheringTotalMemory) {
+ GatherTotalMemory();
+ }
+
+ if (!Telemetry::CanRecordReleaseData()) {
+ return NS_OK;
+ }
+
+ // Get memory measurements from distinguished amount attributes. We used
+ // to measure "explicit" too, but it could cause hangs, and the data was
+ // always really noisy anyway. See bug 859657.
+ //
+ // test_TelemetrySession.js relies on some of these histograms being
+ // here. If you remove any of the following histograms from here, you'll
+ // have to modify test_TelemetrySession.js:
+ //
+ // * MEMORY_TOTAL,
+ // * MEMORY_JS_GC_HEAP, and
+ // * MEMORY_JS_COMPARTMENTS_SYSTEM.
+ //
+ // The distinguished amount attribute names don't match the telemetry id
+ // names in some cases due to a combination of (a) historical reasons, and
+ // (b) the fact that we can't change telemetry id names without breaking
+ // data continuity.
+
+ // Collect cheap or main-thread only metrics synchronously, on the main
+ // thread.
+ RECORD(MEMORY_JS_GC_HEAP, JSMainRuntimeGCHeap, UNITS_BYTES);
+ RECORD(MEMORY_JS_COMPARTMENTS_SYSTEM, JSMainRuntimeCompartmentsSystem,
+ UNITS_COUNT);
+ RECORD(MEMORY_JS_COMPARTMENTS_USER, JSMainRuntimeCompartmentsUser,
+ UNITS_COUNT);
+ RECORD(MEMORY_JS_REALMS_SYSTEM, JSMainRuntimeRealmsSystem, UNITS_COUNT);
+ RECORD(MEMORY_JS_REALMS_USER, JSMainRuntimeRealmsUser, UNITS_COUNT);
+ RECORD(MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED, ImagesContentUsedUncompressed,
+ UNITS_BYTES);
+ RECORD(MEMORY_STORAGE_SQLITE, StorageSQLite, UNITS_BYTES);
+#ifdef XP_WIN
+ RECORD(LOW_MEMORY_EVENTS_PHYSICAL, LowMemoryEventsPhysical,
+ UNITS_COUNT_CUMULATIVE);
+#endif
+#if defined(XP_LINUX) && !defined(ANDROID)
+ RECORD(PAGE_FAULTS_HARD, PageFaultsHard, UNITS_COUNT_CUMULATIVE);
+#endif
+
+ RefPtr<Runnable> completionRunnable;
+ if (aCompletionCallback) {
+ completionRunnable = NS_NewRunnableFunction(__func__, aCompletionCallback);
+ }
+
+ // Collect expensive metrics that can be calculated off-main-thread
+ // asynchronously, on a background thread.
+ RefPtr<Runnable> runnable = NS_NewRunnableFunction(
+ "MemoryTelemetry::GatherReports", [mgr, completionRunnable]() mutable {
+ Telemetry::AutoTimer<Telemetry::MEMORY_COLLECTION_TIME> autoTimer;
+ RECORD(MEMORY_VSIZE, Vsize, UNITS_BYTES);
+#if !defined(HAVE_64BIT_BUILD) || !defined(XP_WIN)
+ RECORD(MEMORY_VSIZE_MAX_CONTIGUOUS, VsizeMaxContiguous, UNITS_BYTES);
+#endif
+ RECORD(MEMORY_RESIDENT_FAST, ResidentFast, UNITS_BYTES);
+ RECORD(MEMORY_RESIDENT_PEAK, ResidentPeak, UNITS_BYTES);
+// Although we can measure unique memory on MacOS we choose not to, because
+// doing so is too slow for telemetry.
+#ifndef XP_MACOSX
+ RECORD(MEMORY_UNIQUE, ResidentUnique, UNITS_BYTES);
+#endif
+
+#ifdef HAVE_JEMALLOC_STATS
+ jemalloc_stats_t stats;
+ jemalloc_stats(&stats);
+ HandleMemoryReport(Telemetry::MEMORY_HEAP_ALLOCATED,
+ nsIMemoryReporter::UNITS_BYTES,
+ mgr->HeapAllocated(stats));
+ HandleMemoryReport(Telemetry::MEMORY_HEAP_OVERHEAD_FRACTION,
+ nsIMemoryReporter::UNITS_PERCENTAGE,
+ mgr->HeapOverheadFraction(stats));
+#endif
+
+ if (completionRunnable) {
+ NS_DispatchToMainThread(completionRunnable.forget(),
+ NS_DISPATCH_NORMAL);
+ }
+ });
+
+#undef RECORD
+
+ nsresult rv = mThreadPool->Dispatch(runnable.forget(), NS_DISPATCH_NORMAL);
+ if (!NS_WARN_IF(NS_FAILED(rv))) {
+ cleanup.release();
+ }
+
+ return NS_OK;
+}
+
+namespace {
+struct ChildProcessInfo {
+ GeckoProcessType mType;
+#if defined(XP_WIN)
+ HANDLE mHandle;
+#elif defined(XP_MACOSX)
+ task_t mHandle;
+#else
+ pid_t mHandle;
+#endif
+};
+} // namespace
+
+/**
+ * Runs a task on the background thread pool to fetch the memory usage of all
+ * processes.
+ */
+void MemoryTelemetry::GatherTotalMemory() {
+ MOZ_ASSERT(!mGatheringTotalMemory);
+ mGatheringTotalMemory = true;
+
+ nsTArray<ChildProcessInfo> infos;
+ mozilla::ipc::GeckoChildProcessHost::GetAll(
+ [&](mozilla::ipc::GeckoChildProcessHost* aGeckoProcess) {
+ if (!aGeckoProcess->GetChildProcessHandle()) {
+ return;
+ }
+
+ ChildProcessInfo info{};
+ info.mType = aGeckoProcess->GetProcessType();
+
+ // NOTE: For now we ignore non-content processes here for compatibility
+ // with the existing probe. We may want to introduce a new probe in the
+ // future which also collects data for non-content processes.
+ if (info.mType != GeckoProcessType_Content) {
+ return;
+ }
+
+#if defined(XP_WIN)
+ if (!::DuplicateHandle(::GetCurrentProcess(),
+ aGeckoProcess->GetChildProcessHandle(),
+ ::GetCurrentProcess(), &info.mHandle, 0, false,
+ DUPLICATE_SAME_ACCESS)) {
+ return;
+ }
+#elif defined(XP_MACOSX)
+ info.mHandle = aGeckoProcess->GetChildTask();
+ if (mach_port_mod_refs(mach_task_self(), info.mHandle,
+ MACH_PORT_RIGHT_SEND, 1) != KERN_SUCCESS) {
+ return;
+ }
+#else
+ info.mHandle = aGeckoProcess->GetChildProcessId();
+#endif
+
+ infos.AppendElement(info);
+ });
+
+ mThreadPool->Dispatch(NS_NewRunnableFunction(
+ "MemoryTelemetry::GatherTotalMemory", [infos = std::move(infos)] {
+ RefPtr<nsMemoryReporterManager> mgr =
+ nsMemoryReporterManager::GetOrCreate();
+ MOZ_RELEASE_ASSERT(mgr);
+
+ int64_t totalMemory = mgr->ResidentFast();
+ nsTArray<int64_t> childSizes(infos.Length());
+
+ // Use our handle for the remote process to collect resident unique set
+ // size information for that process.
+ for (const auto& info : infos) {
+#ifdef XP_MACOSX
+ int64_t memory =
+ nsMemoryReporterManager::PhysicalFootprint(info.mHandle);
+#else
+ int64_t memory =
+ nsMemoryReporterManager::ResidentUnique(info.mHandle);
+#endif
+ if (memory > 0) {
+ childSizes.AppendElement(memory);
+ totalMemory += memory;
+ }
+
+#if defined(XP_WIN)
+ ::CloseHandle(info.mHandle);
+#elif defined(XP_MACOSX)
+ mach_port_deallocate(mach_task_self(), info.mHandle);
+#endif
+ }
+
+ NS_DispatchToMainThread(NS_NewRunnableFunction(
+ "MemoryTelemetry::FinishGatheringTotalMemory",
+ [totalMemory, childSizes = std::move(childSizes)] {
+ MemoryTelemetry::Get().FinishGatheringTotalMemory(totalMemory,
+ childSizes);
+ }));
+ }));
+}
+
+nsresult MemoryTelemetry::FinishGatheringTotalMemory(
+ int64_t aTotalMemory, const nsTArray<int64_t>& aChildSizes) {
+ mGatheringTotalMemory = false;
+
+ // Total memory usage can be difficult to measure both accurately and fast
+ // enough for telemetry (iterating memory maps can jank whole processes on
+ // MacOS). Therefore this shouldn't be relied on as an absolute measurement
+ // especially on MacOS where it double-counts shared memory. For a more
+ // detailed explaination see:
+ // https://groups.google.com/a/mozilla.org/g/dev-platform/c/WGNOtjHdsdA
+ HandleMemoryReport(Telemetry::MEMORY_TOTAL, nsIMemoryReporter::UNITS_BYTES,
+ aTotalMemory);
+
+ if (aChildSizes.Length() > 1) {
+ int32_t tabsCount;
+ MOZ_TRY_VAR(tabsCount, GetOpenTabsCount());
+
+ nsCString key;
+ if (tabsCount <= 10) {
+ key = "0 - 10 tabs";
+ } else if (tabsCount <= 500) {
+ key = "11 - 500 tabs";
+ } else {
+ key = "more tabs";
+ }
+
+ // Mean of the USS of all the content processes.
+ int64_t mean = 0;
+ for (auto size : aChildSizes) {
+ mean += size;
+ }
+ mean /= aChildSizes.Length();
+
+ // For some users, for unknown reasons (though most likely because they're
+ // in a sandbox without procfs mounted), we wind up with 0 here, which
+ // triggers a floating point exception if we try to calculate values using
+ // it.
+ if (!mean) {
+ return NS_ERROR_UNEXPECTED;
+ }
+
+ // Absolute error of USS for each content process, normalized by the mean
+ // (*100 to get it in percentage). 20% means for a content process that it
+ // is using 20% more or 20% less than the mean.
+ for (auto size : aChildSizes) {
+ int64_t diff = llabs(size - mean) * 100 / mean;
+
+ HandleMemoryReport(Telemetry::MEMORY_DISTRIBUTION_AMONG_CONTENT,
+ nsIMemoryReporter::UNITS_COUNT, diff, key);
+ }
+ }
+
+ // This notification is for testing only.
+ if (nsCOMPtr<nsIObserverService> obs = services::GetObserverService()) {
+ obs->NotifyObservers(nullptr, "gather-memory-telemetry-finished", nullptr);
+ }
+
+ return NS_OK;
+}
+
+/* static */ Result<uint32_t, nsresult> MemoryTelemetry::GetOpenTabsCount() {
+ nsresult rv;
+
+ nsCOMPtr<nsIWindowMediator> windowMediator(
+ do_GetService(NS_WINDOWMEDIATOR_CONTRACTID, &rv));
+ MOZ_TRY(rv);
+
+ nsCOMPtr<nsISimpleEnumerator> enumerator;
+ MOZ_TRY(windowMediator->GetEnumerator(u"navigator:browser",
+ getter_AddRefs(enumerator)));
+
+ uint32_t total = 0;
+ for (auto& window : SimpleEnumerator<nsIDOMChromeWindow>(enumerator)) {
+ nsCOMPtr<nsIBrowserDOMWindow> browserWin;
+ MOZ_TRY(window->GetBrowserDOMWindow(getter_AddRefs(browserWin)));
+
+ NS_ENSURE_TRUE(browserWin, Err(NS_ERROR_UNEXPECTED));
+
+ uint32_t tabCount;
+ MOZ_TRY(browserWin->GetTabCount(&tabCount));
+ total += tabCount;
+ }
+
+ return total;
+}
+
+nsresult MemoryTelemetry::Observe(nsISupports* aSubject, const char* aTopic,
+ const char16_t* aData) {
+ if (strcmp(aTopic, kTopicCycleCollectorBegin) == 0) {
+ auto now = TimeStamp::Now();
+ if (!mLastPoll.IsNull() &&
+ (now - mLastPoll).ToMilliseconds() < kTelemetryInterval) {
+ return NS_OK;
+ }
+
+ mLastPoll = now;
+
+ NS_DispatchToCurrentThreadQueue(
+ NewRunnableMethod<std::function<void()>>(
+ "MemoryTelemetry::GatherReports", this,
+ &MemoryTelemetry::GatherReports, nullptr),
+ EventQueuePriority::Idle);
+ } else if (strcmp(aTopic, "content-child-shutdown") == 0) {
+ if (nsCOMPtr<nsITelemetry> telemetry =
+ do_GetService("@mozilla.org/base/telemetry;1")) {
+ telemetry->FlushBatchedChildTelemetry();
+ }
+ }
+ return NS_OK;
+}