summaryrefslogtreecommitdiffstats
path: root/memory/replace/phc
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--memory/replace/phc/PHC.cpp1698
-rw-r--r--memory/replace/phc/PHC.h102
-rw-r--r--memory/replace/phc/moz.build50
-rw-r--r--memory/replace/phc/test/gtest/TestPHC.cpp305
-rw-r--r--memory/replace/phc/test/gtest/moz.build15
-rw-r--r--memory/replace/phc/test/moz.build9
6 files changed, 2179 insertions, 0 deletions
diff --git a/memory/replace/phc/PHC.cpp b/memory/replace/phc/PHC.cpp
new file mode 100644
index 0000000000..7693499ae0
--- /dev/null
+++ b/memory/replace/phc/PHC.cpp
@@ -0,0 +1,1698 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// PHC is a probabilistic heap checker. A tiny fraction of randomly chosen heap
+// allocations are subject to some expensive checking via the use of OS page
+// access protection. A failed check triggers a crash, whereupon useful
+// information about the failure is put into the crash report. The cost and
+// coverage for each user is minimal, but spread over the entire user base the
+// coverage becomes significant.
+//
+// The idea comes from Chromium, where it is called GWP-ASAN. (Firefox uses PHC
+// as the name because GWP-ASAN is long, awkward, and doesn't have any
+// particular meaning.)
+//
+// In the current implementation up to 64 allocations per process can become
+// PHC allocations. These allocations must be page-sized or smaller. Each PHC
+// allocation gets its own page, and when the allocation is freed its page is
+// marked inaccessible until the page is reused for another allocation. This
+// means that a use-after-free defect (which includes double-frees) will be
+// caught if the use occurs before the page is reused for another allocation.
+// The crash report will contain stack traces for the allocation site, the free
+// site, and the use-after-free site, which is often enough to diagnose the
+// defect.
+//
+// Also, each PHC allocation is followed by a guard page. The PHC allocation is
+// positioned so that its end abuts the guard page (or as close as possible,
+// given alignment constraints). This means that a bounds violation at the end
+// of the allocation (overflow) will be caught. The crash report will contain
+// stack traces for the allocation site and the bounds violation use site,
+// which is often enough to diagnose the defect.
+//
+// (A bounds violation at the start of the allocation (underflow) will not be
+// caught, unless it is sufficiently large to hit the preceding allocation's
+// guard page, which is not that likely. It would be possible to look more
+// assiduously for underflow by randomly placing some allocations at the end of
+// the page and some at the start of the page, and GWP-ASAN does this. PHC does
+// not, however, because overflow is likely to be much more common than
+// underflow in practice.)
+//
+// We use a simple heuristic to categorize a guard page access as overflow or
+// underflow: if the address falls in the lower half of the guard page, we
+// assume it is overflow, otherwise we assume it is underflow. More
+// sophisticated heuristics are possible, but this one is very simple, and it is
+// likely that most overflows/underflows in practice are very close to the page
+// boundary.
+//
+// The design space for the randomization strategy is large. The current
+// implementation has a large random delay before it starts operating, and a
+// small random delay between each PHC allocation attempt. Each freed PHC
+// allocation is quarantined for a medium random delay before being reused, in
+// order to increase the chance of catching UAFs.
+//
+// The basic cost of PHC's operation is as follows.
+//
+// - The physical memory cost is 64 pages plus some metadata (including stack
+// traces) for each page. This amounts to 256 KiB per process on
+// architectures with 4 KiB pages and 1024 KiB on macOS/AArch64 which uses
+// 16 KiB pages.
+//
+// - The virtual memory cost is the physical memory cost plus the guard pages:
+// another 64 pages. This amounts to another 256 KiB per process on
+// architectures with 4 KiB pages and 1024 KiB on macOS/AArch64 which uses
+// 16 KiB pages. PHC is currently only enabled on 64-bit platforms so the
+// impact of the virtual memory usage is negligible.
+//
+// - Every allocation requires a size check and a decrement-and-check of an
+// atomic counter. When the counter reaches zero a PHC allocation can occur,
+// which involves marking a page as accessible and getting a stack trace for
+// the allocation site. Otherwise, mozjemalloc performs the allocation.
+//
+// - Every deallocation requires a range check on the pointer to see if it
+// involves a PHC allocation. (The choice to only do PHC allocations that are
+// a page or smaller enables this range check, because the 64 pages are
+// contiguous. Allowing larger allocations would make this more complicated,
+// and we definitely don't want something as slow as a hash table lookup on
+// every deallocation.) PHC deallocations involve marking a page as
+// inaccessible and getting a stack trace for the deallocation site.
+//
+// Note that calls to realloc(), free(), and malloc_usable_size() will
+// immediately crash if the given pointer falls within a page allocation's
+// page, but does not point to the start of the allocation itself.
+//
+// void* p = malloc(64);
+// free(p + 1); // p+1 doesn't point to the allocation start; crash
+//
+// Such crashes will not have the PHC fields in the crash report.
+//
+// PHC-specific tests can be run with the following commands:
+// - gtests: `./mach gtest '*PHC*'`
+// - xpcshell-tests: `./mach test toolkit/crashreporter/test/unit`
+// - This runs some non-PHC tests as well.
+
+#include "PHC.h"
+
+#include <stdlib.h>
+#include <time.h>
+
+#include <algorithm>
+
+#ifdef XP_WIN
+# include <process.h>
+#else
+# include <sys/mman.h>
+# include <sys/types.h>
+# include <pthread.h>
+# include <unistd.h>
+#endif
+
+#include "replace_malloc.h"
+#include "FdPrintf.h"
+#include "Mutex.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/StackWalk.h"
+#include "mozilla/ThreadLocal.h"
+#include "mozilla/XorShift128PlusRNG.h"
+
+using namespace mozilla;
+
+//---------------------------------------------------------------------------
+// Utilities
+//---------------------------------------------------------------------------
+
+#ifdef ANDROID
+// Android doesn't have pthread_atfork defined in pthread.h.
+extern "C" MOZ_EXPORT int pthread_atfork(void (*)(void), void (*)(void),
+ void (*)(void));
+#endif
+
+#ifndef DISALLOW_COPY_AND_ASSIGN
+# define DISALLOW_COPY_AND_ASSIGN(T) \
+ T(const T&); \
+ void operator=(const T&)
+#endif
+
+static malloc_table_t sMallocTable;
+
+// This class provides infallible operations for the small number of heap
+// allocations that PHC does for itself. It would be nice if we could use the
+// InfallibleAllocPolicy from mozalloc, but PHC cannot use mozalloc.
+class InfallibleAllocPolicy {
+ public:
+ static void AbortOnFailure(const void* aP) {
+ if (!aP) {
+ MOZ_CRASH("PHC failed to allocate");
+ }
+ }
+
+ template <class T>
+ static T* new_() {
+ void* p = sMallocTable.malloc(sizeof(T));
+ AbortOnFailure(p);
+ return new (p) T;
+ }
+};
+
+//---------------------------------------------------------------------------
+// Stack traces
+//---------------------------------------------------------------------------
+
+// This code is similar to the equivalent code within DMD.
+
+class StackTrace : public phc::StackTrace {
+ public:
+ StackTrace() : phc::StackTrace() {}
+
+ void Clear() { mLength = 0; }
+
+ void Fill();
+
+ private:
+ static void StackWalkCallback(uint32_t aFrameNumber, void* aPc, void* aSp,
+ void* aClosure) {
+ StackTrace* st = (StackTrace*)aClosure;
+ MOZ_ASSERT(st->mLength < kMaxFrames);
+ st->mPcs[st->mLength] = aPc;
+ st->mLength++;
+ MOZ_ASSERT(st->mLength == aFrameNumber);
+ }
+};
+
+// WARNING WARNING WARNING: this function must only be called when GMut::sMutex
+// is *not* locked, otherwise we might get deadlocks.
+//
+// How? On Windows, MozStackWalk() can lock a mutex, M, from the shared library
+// loader. Another thread might call malloc() while holding M locked (when
+// loading a shared library) and try to lock GMut::sMutex, causing a deadlock.
+// So GMut::sMutex can't be locked during the call to MozStackWalk(). (For
+// details, see https://bugzilla.mozilla.org/show_bug.cgi?id=374829#c8. On
+// Linux, something similar can happen; see bug 824340. So we just disallow it
+// on all platforms.)
+//
+// In DMD, to avoid this problem we temporarily unlock the equivalent mutex for
+// the MozStackWalk() call. But that's grotty, and things are a bit different
+// here, so we just require that stack traces be obtained before locking
+// GMut::sMutex.
+//
+// Unfortunately, there is no reliable way at compile-time or run-time to ensure
+// this pre-condition. Hence this large comment.
+//
+void StackTrace::Fill() {
+ mLength = 0;
+
+#if defined(XP_WIN) && defined(_M_IX86)
+ // This avoids MozStackWalk(), which causes unusably slow startup on Win32
+ // when it is called during static initialization (see bug 1241684).
+ //
+ // This code is cribbed from the Gecko Profiler, which also uses
+ // FramePointerStackWalk() on Win32: Registers::SyncPopulate() for the
+ // frame pointer, and GetStackTop() for the stack end.
+ CONTEXT context;
+ RtlCaptureContext(&context);
+ void** fp = reinterpret_cast<void**>(context.Ebp);
+
+ PNT_TIB pTib = reinterpret_cast<PNT_TIB>(NtCurrentTeb());
+ void* stackEnd = static_cast<void*>(pTib->StackBase);
+ FramePointerStackWalk(StackWalkCallback, kMaxFrames, this, fp, stackEnd);
+#elif defined(XP_MACOSX)
+ // This avoids MozStackWalk(), which has become unusably slow on Mac due to
+ // changes in libunwind.
+ //
+ // This code is cribbed from the Gecko Profiler, which also uses
+ // FramePointerStackWalk() on Mac: Registers::SyncPopulate() for the frame
+ // pointer, and GetStackTop() for the stack end.
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wframe-address"
+ void** fp = reinterpret_cast<void**>(__builtin_frame_address(1));
+# pragma GCC diagnostic pop
+ void* stackEnd = pthread_get_stackaddr_np(pthread_self());
+ FramePointerStackWalk(StackWalkCallback, kMaxFrames, this, fp, stackEnd);
+#else
+ MozStackWalk(StackWalkCallback, nullptr, kMaxFrames, this);
+#endif
+}
+
+//---------------------------------------------------------------------------
+// Logging
+//---------------------------------------------------------------------------
+
+// Change this to 1 to enable some PHC logging. Useful for debugging.
+#define PHC_LOGGING 0
+
+#if PHC_LOGGING
+
+static size_t GetPid() { return size_t(getpid()); }
+
+static size_t GetTid() {
+# if defined(XP_WIN)
+ return size_t(GetCurrentThreadId());
+# else
+ return size_t(pthread_self());
+# endif
+}
+
+# if defined(XP_WIN)
+# define LOG_STDERR \
+ reinterpret_cast<intptr_t>(GetStdHandle(STD_ERROR_HANDLE))
+# else
+# define LOG_STDERR 2
+# endif
+# define LOG(fmt, ...) \
+ FdPrintf(LOG_STDERR, "PHC[%zu,%zu,~%zu] " fmt, GetPid(), GetTid(), \
+ size_t(GAtomic::Now()), __VA_ARGS__)
+
+#else
+
+# define LOG(fmt, ...)
+
+#endif // PHC_LOGGING
+
+//---------------------------------------------------------------------------
+// Global state
+//---------------------------------------------------------------------------
+
+// Throughout this entire file time is measured as the number of sub-page
+// allocations performed (by PHC and mozjemalloc combined). `Time` is 64-bit
+// because we could have more than 2**32 allocations in a long-running session.
+// `Delay` is 32-bit because the delays used within PHC are always much smaller
+// than 2**32.
+using Time = uint64_t; // A moment in time.
+using Delay = uint32_t; // A time duration.
+
+// PHC only runs if the page size is 4 KiB; anything more is uncommon and would
+// use too much memory. So we hardwire this size for all platforms but macOS
+// on ARM processors. For the latter we make an exception because the minimum
+// page size supported is 16KiB so there's no way to go below that.
+static const size_t kPageSize =
+#if defined(XP_MACOSX) && defined(__aarch64__)
+ 16384
+#else
+ 4096
+#endif
+ ;
+
+// There are two kinds of page.
+// - Allocation pages, from which allocations are made.
+// - Guard pages, which are never touched by PHC.
+//
+// These page kinds are interleaved; each allocation page has a guard page on
+// either side.
+static const size_t kNumAllocPages = kPageSize == 4096 ? 4096 : 1024;
+static const size_t kNumAllPages = kNumAllocPages * 2 + 1;
+
+// The total size of the allocation pages and guard pages.
+static const size_t kAllPagesSize = kNumAllPages * kPageSize;
+
+// The junk value used to fill new allocation in debug builds. It's same value
+// as the one used by mozjemalloc. PHC applies it unconditionally in debug
+// builds. Unlike mozjemalloc, PHC doesn't consult the MALLOC_OPTIONS
+// environment variable to possibly change that behaviour.
+//
+// Also note that, unlike mozjemalloc, PHC doesn't have a poison value for freed
+// allocations because freed allocations are protected by OS page protection.
+#ifdef DEBUG
+const uint8_t kAllocJunk = 0xe4;
+#endif
+
+// The maximum time.
+static const Time kMaxTime = ~(Time(0));
+
+// The average delay before doing any page allocations at the start of a
+// process. Note that roughly 1 million allocations occur in the main process
+// while starting the browser. The delay range is 1..kAvgFirstAllocDelay*2.
+static const Delay kAvgFirstAllocDelay = 64 * 1024;
+
+// The average delay until the next attempted page allocation, once we get past
+// the first delay. The delay range is 1..kAvgAllocDelay*2.
+static const Delay kAvgAllocDelay = 16 * 1024;
+
+// The average delay before reusing a freed page. Should be significantly larger
+// than kAvgAllocDelay, otherwise there's not much point in having it. The delay
+// range is (kAvgAllocDelay / 2)..(kAvgAllocDelay / 2 * 3). This is different to
+// the other delay ranges in not having a minimum of 1, because that's such a
+// short delay that there is a high likelihood of bad stacks in any crash
+// report.
+static const Delay kAvgPageReuseDelay = 256 * 1024;
+
+// Truncate aRnd to the range (1 .. AvgDelay*2). If aRnd is random, this
+// results in an average value of aAvgDelay + 0.5, which is close enough to
+// aAvgDelay. aAvgDelay must be a power-of-two (otherwise it will crash) for
+// speed.
+template <Delay AvgDelay>
+constexpr Delay Rnd64ToDelay(uint64_t aRnd) {
+ static_assert(IsPowerOfTwo(AvgDelay), "must be a power of two");
+
+ return aRnd % (AvgDelay * 2) + 1;
+}
+
+// Maps a pointer to a PHC-specific structure:
+// - Nothing
+// - A guard page (it is unspecified which one)
+// - An allocation page (with an index < kNumAllocPages)
+//
+// The standard way of handling a PtrKind is to check IsNothing(), and if that
+// fails, to check IsGuardPage(), and if that fails, to call AllocPage().
+class PtrKind {
+ private:
+ enum class Tag : uint8_t {
+ Nothing,
+ GuardPage,
+ AllocPage,
+ };
+
+ Tag mTag;
+ uintptr_t mIndex; // Only used if mTag == Tag::AllocPage.
+
+ public:
+ // Detect what a pointer points to. This constructor must be fast because it
+ // is called for every call to free(), realloc(), malloc_usable_size(), and
+ // jemalloc_ptr_info().
+ PtrKind(const void* aPtr, const uint8_t* aPagesStart,
+ const uint8_t* aPagesLimit) {
+ if (!(aPagesStart <= aPtr && aPtr < aPagesLimit)) {
+ mTag = Tag::Nothing;
+ } else {
+ uintptr_t offset = static_cast<const uint8_t*>(aPtr) - aPagesStart;
+ uintptr_t allPageIndex = offset / kPageSize;
+ MOZ_ASSERT(allPageIndex < kNumAllPages);
+ if (allPageIndex & 1) {
+ // Odd-indexed pages are allocation pages.
+ uintptr_t allocPageIndex = allPageIndex / 2;
+ MOZ_ASSERT(allocPageIndex < kNumAllocPages);
+ mTag = Tag::AllocPage;
+ mIndex = allocPageIndex;
+ } else {
+ // Even-numbered pages are guard pages.
+ mTag = Tag::GuardPage;
+ }
+ }
+ }
+
+ bool IsNothing() const { return mTag == Tag::Nothing; }
+ bool IsGuardPage() const { return mTag == Tag::GuardPage; }
+
+ // This should only be called after IsNothing() and IsGuardPage() have been
+ // checked and failed.
+ uintptr_t AllocPageIndex() const {
+ MOZ_RELEASE_ASSERT(mTag == Tag::AllocPage);
+ return mIndex;
+ }
+};
+
+// Shared, atomic, mutable global state.
+class GAtomic {
+ public:
+ static void Init(Delay aFirstDelay) {
+ sAllocDelay = aFirstDelay;
+
+ LOG("Initial sAllocDelay <- %zu\n", size_t(aFirstDelay));
+ }
+
+ static Time Now() { return sNow; }
+
+ static void IncrementNow() { sNow++; }
+
+ // Decrements the delay and returns the decremented value.
+ static int32_t DecrementDelay() { return --sAllocDelay; }
+
+ static void SetAllocDelay(Delay aAllocDelay) { sAllocDelay = aAllocDelay; }
+
+ private:
+ // The current time. Relaxed semantics because it's primarily used for
+ // determining if an allocation can be recycled yet and therefore it doesn't
+ // need to be exact.
+ static Atomic<Time, Relaxed> sNow;
+
+ // Delay until the next attempt at a page allocation. See the comment in
+ // MaybePageAlloc() for an explanation of why it is a signed integer, and why
+ // it uses ReleaseAcquire semantics.
+ static Atomic<Delay, ReleaseAcquire> sAllocDelay;
+};
+
+Atomic<Time, Relaxed> GAtomic::sNow;
+Atomic<Delay, ReleaseAcquire> GAtomic::sAllocDelay;
+
+// Shared, immutable global state. Initialized by replace_init() and never
+// changed after that. replace_init() runs early enough that no synchronization
+// is needed.
+class GConst {
+ private:
+ // The bounds of the allocated pages.
+ uint8_t* const mPagesStart;
+ uint8_t* const mPagesLimit;
+
+ // Allocates the allocation pages and the guard pages, contiguously.
+ uint8_t* AllocAllPages() {
+ // Allocate the pages so that they are inaccessible. They are never freed,
+ // because it would happen at process termination when it would be of little
+ // use.
+ void* pages =
+#ifdef XP_WIN
+ VirtualAlloc(nullptr, kAllPagesSize, MEM_RESERVE, PAGE_NOACCESS);
+#else
+ mmap(nullptr, kAllPagesSize, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
+ 0);
+#endif
+ if (!pages) {
+ MOZ_CRASH();
+ }
+
+ return static_cast<uint8_t*>(pages);
+ }
+
+ public:
+ GConst()
+ : mPagesStart(AllocAllPages()), mPagesLimit(mPagesStart + kAllPagesSize) {
+ LOG("AllocAllPages at %p..%p\n", mPagesStart, mPagesLimit);
+ }
+
+ class PtrKind PtrKind(const void* aPtr) {
+ class PtrKind pk(aPtr, mPagesStart, mPagesLimit);
+ return pk;
+ }
+
+ bool IsInFirstGuardPage(const void* aPtr) {
+ return mPagesStart <= aPtr && aPtr < mPagesStart + kPageSize;
+ }
+
+ // Get the address of the allocation page referred to via an index. Used when
+ // marking the page as accessible/inaccessible.
+ uint8_t* AllocPagePtr(uintptr_t aIndex) {
+ MOZ_ASSERT(aIndex < kNumAllocPages);
+ // Multiply by two and add one to account for allocation pages *and* guard
+ // pages.
+ return mPagesStart + (2 * aIndex + 1) * kPageSize;
+ }
+};
+
+static GConst* gConst;
+
+// On MacOS, the first __thread/thread_local access calls malloc, which leads
+// to an infinite loop. So we use pthread-based TLS instead, which somehow
+// doesn't have this problem.
+#if !defined(XP_DARWIN)
+# define PHC_THREAD_LOCAL(T) MOZ_THREAD_LOCAL(T)
+#else
+# define PHC_THREAD_LOCAL(T) \
+ detail::ThreadLocal<T, detail::ThreadLocalKeyStorage>
+#endif
+
+// Thread-local state.
+class GTls {
+ GTls(const GTls&) = delete;
+
+ const GTls& operator=(const GTls&) = delete;
+
+ // When true, PHC does as little as possible.
+ //
+ // (a) It does not allocate any new page allocations.
+ //
+ // (b) It avoids doing any operations that might call malloc/free/etc., which
+ // would cause re-entry into PHC. (In practice, MozStackWalk() is the
+ // only such operation.) Note that calls to the functions in sMallocTable
+ // are ok.
+ //
+ // For example, replace_malloc() will just fall back to mozjemalloc. However,
+ // operations involving existing allocations are more complex, because those
+ // existing allocations may be page allocations. For example, if
+ // replace_free() is passed a page allocation on a PHC-disabled thread, it
+ // will free the page allocation in the usual way, but it will get a dummy
+ // freeStack in order to avoid calling MozStackWalk(), as per (b) above.
+ //
+ // This single disabling mechanism has two distinct uses.
+ //
+ // - It's used to prevent re-entry into PHC, which can cause correctness
+ // problems. For example, consider this sequence.
+ //
+ // 1. enter replace_free()
+ // 2. which calls PageFree()
+ // 3. which calls MozStackWalk()
+ // 4. which locks a mutex M, and then calls malloc
+ // 5. enter replace_malloc()
+ // 6. which calls MaybePageAlloc()
+ // 7. which calls MozStackWalk()
+ // 8. which (re)locks a mutex M --> deadlock
+ //
+ // We avoid this sequence by "disabling" the thread in PageFree() (at step
+ // 2), which causes MaybePageAlloc() to fail, avoiding the call to
+ // MozStackWalk() (at step 7).
+ //
+ // In practice, realloc or free of a PHC allocation is unlikely on a thread
+ // that is disabled because of this use: MozStackWalk() will probably only
+ // realloc/free allocations that it allocated itself, but those won't be
+ // page allocations because PHC is disabled before calling MozStackWalk().
+ //
+ // (Note that MaybePageAlloc() could safely do a page allocation so long as
+ // it avoided calling MozStackWalk() by getting a dummy allocStack. But it
+ // wouldn't be useful, and it would prevent the second use below.)
+ //
+ // - It's used to prevent PHC allocations in some tests that rely on
+ // mozjemalloc's exact allocation behaviour, which PHC does not replicate
+ // exactly. (Note that (b) isn't necessary for this use -- MozStackWalk()
+ // could be safely called -- but it is necessary for the first use above.)
+ //
+ static PHC_THREAD_LOCAL(bool) tlsIsDisabled;
+
+ public:
+ static void Init() {
+ if (!tlsIsDisabled.init()) {
+ MOZ_CRASH();
+ }
+ }
+
+ static void DisableOnCurrentThread() {
+ MOZ_ASSERT(!GTls::tlsIsDisabled.get());
+ tlsIsDisabled.set(true);
+ }
+
+ static void EnableOnCurrentThread() {
+ MOZ_ASSERT(GTls::tlsIsDisabled.get());
+ tlsIsDisabled.set(false);
+ }
+
+ static bool IsDisabledOnCurrentThread() { return tlsIsDisabled.get(); }
+};
+
+PHC_THREAD_LOCAL(bool) GTls::tlsIsDisabled;
+
+class AutoDisableOnCurrentThread {
+ AutoDisableOnCurrentThread(const AutoDisableOnCurrentThread&) = delete;
+
+ const AutoDisableOnCurrentThread& operator=(
+ const AutoDisableOnCurrentThread&) = delete;
+
+ public:
+ explicit AutoDisableOnCurrentThread() { GTls::DisableOnCurrentThread(); }
+ ~AutoDisableOnCurrentThread() { GTls::EnableOnCurrentThread(); }
+};
+
+// This type is used as a proof-of-lock token, to make it clear which functions
+// require sMutex to be locked.
+using GMutLock = const MutexAutoLock&;
+
+// Shared, mutable global state. Protected by sMutex; all accessing functions
+// take a GMutLock as proof that sMutex is held.
+class GMut {
+ enum class AllocPageState {
+ NeverAllocated = 0,
+ InUse = 1,
+ Freed = 2,
+ };
+
+ // Metadata for each allocation page.
+ class AllocPageInfo {
+ public:
+ AllocPageInfo()
+ : mState(AllocPageState::NeverAllocated),
+ mArenaId(),
+ mBaseAddr(nullptr),
+ mAllocStack(),
+ mFreeStack(),
+ mReuseTime(0) {}
+
+ // The current allocation page state.
+ AllocPageState mState;
+
+ // The arena that the allocation is nominally from. This isn't meaningful
+ // within PHC, which has no arenas. But it is necessary for reallocation of
+ // page allocations as normal allocations, such as in this code:
+ //
+ // p = moz_arena_malloc(arenaId, 4096);
+ // realloc(p, 8192);
+ //
+ // The realloc is more than one page, and thus too large for PHC to handle.
+ // Therefore, if PHC handles the first allocation, it must ask mozjemalloc
+ // to allocate the 8192 bytes in the correct arena, and to do that, it must
+ // call sMallocTable.moz_arena_malloc with the correct arenaId under the
+ // covers. Therefore it must record that arenaId.
+ //
+ // This field is also needed for jemalloc_ptr_info() to work, because it
+ // also returns the arena ID (but only in debug builds).
+ //
+ // - NeverAllocated: must be 0.
+ // - InUse | Freed: can be any valid arena ID value.
+ Maybe<arena_id_t> mArenaId;
+
+ // The starting address of the allocation. Will not be the same as the page
+ // address unless the allocation is a full page.
+ // - NeverAllocated: must be 0.
+ // - InUse | Freed: must be within the allocation page.
+ uint8_t* mBaseAddr;
+
+ // Usable size is computed as the number of bytes between the pointer and
+ // the end of the allocation page. This might be bigger than the requested
+ // size, especially if an outsized alignment is requested.
+ size_t UsableSize() const {
+ return mState == AllocPageState::NeverAllocated
+ ? 0
+ : kPageSize - (reinterpret_cast<uintptr_t>(mBaseAddr) &
+ (kPageSize - 1));
+ }
+
+ // The internal fragmentation for this allocation.
+ size_t FragmentationBytes() const {
+ MOZ_ASSERT(kPageSize >= UsableSize());
+ return mState == AllocPageState::InUse ? kPageSize - UsableSize() : 0;
+ }
+
+ // The allocation stack.
+ // - NeverAllocated: Nothing.
+ // - InUse | Freed: Some.
+ Maybe<StackTrace> mAllocStack;
+
+ // The free stack.
+ // - NeverAllocated | InUse: Nothing.
+ // - Freed: Some.
+ Maybe<StackTrace> mFreeStack;
+
+ // The time at which the page is available for reuse, as measured against
+ // GAtomic::sNow. When the page is in use this value will be kMaxTime.
+ // - NeverAllocated: must be 0.
+ // - InUse: must be kMaxTime.
+ // - Freed: must be > 0 and < kMaxTime.
+ Time mReuseTime;
+ };
+
+ public:
+ // The mutex that protects the other members.
+ static Mutex sMutex MOZ_UNANNOTATED;
+
+ GMut() : mRNG(RandomSeed<0>(), RandomSeed<1>()), mAllocPages() {
+ sMutex.Init();
+ }
+
+ uint64_t Random64(GMutLock) { return mRNG.next(); }
+
+ bool IsPageInUse(GMutLock, uintptr_t aIndex) {
+ return mAllocPages[aIndex].mState == AllocPageState::InUse;
+ }
+
+ // Is the page free? And if so, has enough time passed that we can use it?
+ bool IsPageAllocatable(GMutLock, uintptr_t aIndex, Time aNow) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+ return page.mState != AllocPageState::InUse && aNow >= page.mReuseTime;
+ }
+
+ // Get the address of the allocation page referred to via an index. Used
+ // when checking pointers against page boundaries.
+ uint8_t* AllocPageBaseAddr(GMutLock, uintptr_t aIndex) {
+ return mAllocPages[aIndex].mBaseAddr;
+ }
+
+ Maybe<arena_id_t> PageArena(GMutLock aLock, uintptr_t aIndex) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+ AssertAllocPageInUse(aLock, page);
+
+ return page.mArenaId;
+ }
+
+ size_t PageUsableSize(GMutLock aLock, uintptr_t aIndex) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+ AssertAllocPageInUse(aLock, page);
+
+ return page.UsableSize();
+ }
+
+ // The total fragmentation in PHC
+ size_t FragmentationBytes() const {
+ size_t sum = 0;
+ for (const auto& page : mAllocPages) {
+ sum += page.FragmentationBytes();
+ }
+ return sum;
+ }
+
+ void SetPageInUse(GMutLock aLock, uintptr_t aIndex,
+ const Maybe<arena_id_t>& aArenaId, uint8_t* aBaseAddr,
+ const StackTrace& aAllocStack) {
+ AllocPageInfo& page = mAllocPages[aIndex];
+ AssertAllocPageNotInUse(aLock, page);
+
+ page.mState = AllocPageState::InUse;
+ page.mArenaId = aArenaId;
+ page.mBaseAddr = aBaseAddr;
+ page.mAllocStack = Some(aAllocStack);
+ page.mFreeStack = Nothing();
+ page.mReuseTime = kMaxTime;
+ }
+
+#if PHC_LOGGING
+ Time GetFreeTime(uintptr_t aIndex) const { return mFreeTime[aIndex]; }
+#endif
+
+ void ResizePageInUse(GMutLock aLock, uintptr_t aIndex,
+ const Maybe<arena_id_t>& aArenaId, uint8_t* aNewBaseAddr,
+ const StackTrace& aAllocStack) {
+ AllocPageInfo& page = mAllocPages[aIndex];
+ AssertAllocPageInUse(aLock, page);
+
+ // page.mState is not changed.
+ if (aArenaId.isSome()) {
+ // Crash if the arenas don't match.
+ MOZ_RELEASE_ASSERT(page.mArenaId == aArenaId);
+ }
+ page.mBaseAddr = aNewBaseAddr;
+ // We could just keep the original alloc stack, but the realloc stack is
+ // more recent and therefore seems more useful.
+ page.mAllocStack = Some(aAllocStack);
+ // page.mFreeStack is not changed.
+ // page.mReuseTime is not changed.
+ };
+
+ void SetPageFreed(GMutLock aLock, uintptr_t aIndex,
+ const Maybe<arena_id_t>& aArenaId,
+ const StackTrace& aFreeStack, Delay aReuseDelay) {
+ AllocPageInfo& page = mAllocPages[aIndex];
+ AssertAllocPageInUse(aLock, page);
+
+ page.mState = AllocPageState::Freed;
+
+ // page.mArenaId is left unchanged, for jemalloc_ptr_info() calls that
+ // occur after freeing (e.g. in the PtrInfo test in TestJemalloc.cpp).
+ if (aArenaId.isSome()) {
+ // Crash if the arenas don't match.
+ MOZ_RELEASE_ASSERT(page.mArenaId == aArenaId);
+ }
+
+ // page.musableSize is left unchanged, for reporting on UAF, and for
+ // jemalloc_ptr_info() calls that occur after freeing (e.g. in the PtrInfo
+ // test in TestJemalloc.cpp).
+
+ // page.mAllocStack is left unchanged, for reporting on UAF.
+
+ page.mFreeStack = Some(aFreeStack);
+ Time now = GAtomic::Now();
+#if PHC_LOGGING
+ mFreeTime[aIndex] = now;
+#endif
+ page.mReuseTime = now + aReuseDelay;
+ }
+
+ static void CrashOnGuardPage(void* aPtr) {
+ // An operation on a guard page? This is a bounds violation. Deliberately
+ // touch the page in question, to cause a crash that triggers the usual PHC
+ // machinery.
+ LOG("CrashOnGuardPage(%p), bounds violation\n", aPtr);
+ *static_cast<uint8_t*>(aPtr) = 0;
+ MOZ_CRASH("unreachable");
+ }
+
+ void EnsureValidAndInUse(GMutLock, void* aPtr, uintptr_t aIndex)
+ MOZ_REQUIRES(sMutex) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+
+ // The pointer must point to the start of the allocation.
+ MOZ_RELEASE_ASSERT(page.mBaseAddr == aPtr);
+
+ if (page.mState == AllocPageState::Freed) {
+ LOG("EnsureValidAndInUse(%p), use-after-free\n", aPtr);
+ // An operation on a freed page? This is a particular kind of
+ // use-after-free. Deliberately touch the page in question, in order to
+ // cause a crash that triggers the usual PHC machinery. But unlock sMutex
+ // first, because that self-same PHC machinery needs to re-lock it, and
+ // the crash causes non-local control flow so sMutex won't be unlocked
+ // the normal way in the caller.
+ sMutex.Unlock();
+ *static_cast<uint8_t*>(aPtr) = 0;
+ MOZ_CRASH("unreachable");
+ }
+ }
+
+ void FillAddrInfo(GMutLock, uintptr_t aIndex, const void* aBaseAddr,
+ bool isGuardPage, phc::AddrInfo& aOut) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+ if (isGuardPage) {
+ aOut.mKind = phc::AddrInfo::Kind::GuardPage;
+ } else {
+ switch (page.mState) {
+ case AllocPageState::NeverAllocated:
+ aOut.mKind = phc::AddrInfo::Kind::NeverAllocatedPage;
+ break;
+
+ case AllocPageState::InUse:
+ aOut.mKind = phc::AddrInfo::Kind::InUsePage;
+ break;
+
+ case AllocPageState::Freed:
+ aOut.mKind = phc::AddrInfo::Kind::FreedPage;
+ break;
+
+ default:
+ MOZ_CRASH();
+ }
+ }
+ aOut.mBaseAddr = page.mBaseAddr;
+ aOut.mUsableSize = page.UsableSize();
+ aOut.mAllocStack = page.mAllocStack;
+ aOut.mFreeStack = page.mFreeStack;
+ }
+
+ void FillJemallocPtrInfo(GMutLock, const void* aPtr, uintptr_t aIndex,
+ jemalloc_ptr_info_t* aInfo) {
+ const AllocPageInfo& page = mAllocPages[aIndex];
+ switch (page.mState) {
+ case AllocPageState::NeverAllocated:
+ break;
+
+ case AllocPageState::InUse: {
+ // Only return TagLiveAlloc if the pointer is within the bounds of the
+ // allocation's usable size.
+ uint8_t* base = page.mBaseAddr;
+ uint8_t* limit = base + page.UsableSize();
+ if (base <= aPtr && aPtr < limit) {
+ *aInfo = {TagLiveAlloc, page.mBaseAddr, page.UsableSize(),
+ page.mArenaId.valueOr(0)};
+ return;
+ }
+ break;
+ }
+
+ case AllocPageState::Freed: {
+ // Only return TagFreedAlloc if the pointer is within the bounds of the
+ // former allocation's usable size.
+ uint8_t* base = page.mBaseAddr;
+ uint8_t* limit = base + page.UsableSize();
+ if (base <= aPtr && aPtr < limit) {
+ *aInfo = {TagFreedAlloc, page.mBaseAddr, page.UsableSize(),
+ page.mArenaId.valueOr(0)};
+ return;
+ }
+ break;
+ }
+
+ default:
+ MOZ_CRASH();
+ }
+
+ // Pointers into guard pages will end up here, as will pointers into
+ // allocation pages that aren't within the allocation's bounds.
+ *aInfo = {TagUnknown, nullptr, 0, 0};
+ }
+
+#ifndef XP_WIN
+ static void prefork() MOZ_NO_THREAD_SAFETY_ANALYSIS { sMutex.Lock(); }
+ static void postfork_parent() MOZ_NO_THREAD_SAFETY_ANALYSIS {
+ sMutex.Unlock();
+ }
+ static void postfork_child() { sMutex.Init(); }
+#endif
+
+#if PHC_LOGGING
+ void IncPageAllocHits(GMutLock) { mPageAllocHits++; }
+ void IncPageAllocMisses(GMutLock) { mPageAllocMisses++; }
+#else
+ void IncPageAllocHits(GMutLock) {}
+ void IncPageAllocMisses(GMutLock) {}
+#endif
+
+#if PHC_LOGGING
+ struct PageStats {
+ size_t mNumAlloced = 0;
+ size_t mNumFreed = 0;
+ };
+
+ PageStats GetPageStats(GMutLock) {
+ PageStats stats;
+
+ for (const auto& page : mAllocPages) {
+ stats.mNumAlloced += page.mState == AllocPageState::InUse ? 1 : 0;
+ stats.mNumFreed += page.mState == AllocPageState::Freed ? 1 : 0;
+ }
+
+ return stats;
+ }
+
+ size_t PageAllocHits(GMutLock) { return mPageAllocHits; }
+ size_t PageAllocAttempts(GMutLock) {
+ return mPageAllocHits + mPageAllocMisses;
+ }
+
+ // This is an integer because FdPrintf only supports integer printing.
+ size_t PageAllocHitRate(GMutLock) {
+ return mPageAllocHits * 100 / (mPageAllocHits + mPageAllocMisses);
+ }
+#endif
+
+ private:
+ template <int N>
+ uint64_t RandomSeed() {
+ // An older version of this code used RandomUint64() here, but on Mac that
+ // function uses arc4random(), which can allocate, which would cause
+ // re-entry, which would be bad. So we just use time() and a local variable
+ // address. These are mediocre sources of entropy, but good enough for PHC.
+ static_assert(N == 0 || N == 1, "must be 0 or 1");
+ uint64_t seed;
+ if (N == 0) {
+ time_t t = time(nullptr);
+ seed = t ^ (t << 32);
+ } else {
+ seed = uintptr_t(&seed) ^ (uintptr_t(&seed) << 32);
+ }
+ return seed;
+ }
+
+ void AssertAllocPageInUse(GMutLock, const AllocPageInfo& aPage) {
+ MOZ_ASSERT(aPage.mState == AllocPageState::InUse);
+ // There is nothing to assert about aPage.mArenaId.
+ MOZ_ASSERT(aPage.mBaseAddr);
+ MOZ_ASSERT(aPage.UsableSize() > 0);
+ MOZ_ASSERT(aPage.mAllocStack.isSome());
+ MOZ_ASSERT(aPage.mFreeStack.isNothing());
+ MOZ_ASSERT(aPage.mReuseTime == kMaxTime);
+ }
+
+ void AssertAllocPageNotInUse(GMutLock, const AllocPageInfo& aPage) {
+ // We can assert a lot about `NeverAllocated` pages, but not much about
+ // `Freed` pages.
+#ifdef DEBUG
+ bool isFresh = aPage.mState == AllocPageState::NeverAllocated;
+ MOZ_ASSERT(isFresh || aPage.mState == AllocPageState::Freed);
+ MOZ_ASSERT_IF(isFresh, aPage.mArenaId == Nothing());
+ MOZ_ASSERT(isFresh == (aPage.mBaseAddr == nullptr));
+ MOZ_ASSERT(isFresh == (aPage.mAllocStack.isNothing()));
+ MOZ_ASSERT(isFresh == (aPage.mFreeStack.isNothing()));
+ MOZ_ASSERT(aPage.mReuseTime != kMaxTime);
+#endif
+ }
+
+ // RNG for deciding which allocations to treat specially. It doesn't need to
+ // be high quality.
+ //
+ // This is a raw pointer for the reason explained in the comment above
+ // GMut's constructor. Don't change it to UniquePtr or anything like that.
+ non_crypto::XorShift128PlusRNG mRNG;
+
+ AllocPageInfo mAllocPages[kNumAllocPages];
+#if PHC_LOGGING
+ Time mFreeTime[kNumAllocPages];
+
+ // How many allocations that could have been page allocs actually were? As
+ // constrained kNumAllocPages. If the hit ratio isn't close to 100% it's
+ // likely that the global constants are poorly chosen.
+ size_t mPageAllocHits = 0;
+ size_t mPageAllocMisses = 0;
+#endif
+};
+
+Mutex GMut::sMutex;
+
+static GMut* gMut;
+
+//---------------------------------------------------------------------------
+// Page allocation operations
+//---------------------------------------------------------------------------
+
+// Attempt a page allocation if the time and the size are right. Allocated
+// memory is zeroed if aZero is true. On failure, the caller should attempt a
+// normal allocation via sMallocTable. Can be called in a context where
+// GMut::sMutex is locked.
+static void* MaybePageAlloc(const Maybe<arena_id_t>& aArenaId, size_t aReqSize,
+ size_t aAlignment, bool aZero) {
+ MOZ_ASSERT(IsPowerOfTwo(aAlignment));
+
+ if (aReqSize > kPageSize) {
+ return nullptr;
+ }
+
+ GAtomic::IncrementNow();
+
+ // Decrement the delay. If it's zero, we do a page allocation and reset the
+ // delay to a random number. Because the assignment to the random number isn't
+ // atomic w.r.t. the decrement, we might have a sequence like this:
+ //
+ // Thread 1 Thread 2 Thread 3
+ // -------- -------- --------
+ // (a) newDelay = --sAllocDelay (-> 0)
+ // (b) --sAllocDelay (-> -1)
+ // (c) (newDelay != 0) fails
+ // (d) --sAllocDelay (-> -2)
+ // (e) sAllocDelay = new_random_number()
+ //
+ // It's critical that sAllocDelay has ReleaseAcquire semantics, because that
+ // guarantees that exactly one thread will see sAllocDelay have the value 0.
+ // (Relaxed semantics wouldn't guarantee that.)
+ //
+ // It's also nice that sAllocDelay is signed, given that we can decrement to
+ // below zero. (Strictly speaking, an unsigned integer would also work due
+ // to wrapping, but a signed integer is conceptually cleaner.)
+ //
+ // Finally, note that the decrements that occur between (a) and (e) above are
+ // effectively ignored, because (e) clobbers them. This shouldn't be a
+ // problem; it effectively just adds a little more randomness to
+ // new_random_number(). An early version of this code tried to account for
+ // these decrements by doing `sAllocDelay += new_random_number()`. However, if
+ // new_random_value() is small, the number of decrements between (a) and (e)
+ // can easily exceed it, whereupon sAllocDelay ends up negative after
+ // `sAllocDelay += new_random_number()`, and the zero-check never succeeds
+ // again. (At least, not until sAllocDelay wraps around on overflow, which
+ // would take a very long time indeed.)
+ //
+ int32_t newDelay = GAtomic::DecrementDelay();
+ if (newDelay != 0) {
+ return nullptr;
+ }
+
+ if (GTls::IsDisabledOnCurrentThread()) {
+ return nullptr;
+ }
+
+ // Disable on this thread *before* getting the stack trace.
+ AutoDisableOnCurrentThread disable;
+
+ // Get the stack trace *before* locking the mutex. If we return nullptr then
+ // it was a waste, but it's not so frequent, and doing a stack walk while
+ // the mutex is locked is problematic (see the big comment on
+ // StackTrace::Fill() for details).
+ StackTrace allocStack;
+ allocStack.Fill();
+
+ MutexAutoLock lock(GMut::sMutex);
+
+ Time now = GAtomic::Now();
+ Delay newAllocDelay = Rnd64ToDelay<kAvgAllocDelay>(gMut->Random64(lock));
+
+ // We start at a random page alloc and wrap around, to ensure pages get even
+ // amounts of use.
+ uint8_t* ptr = nullptr;
+ uint8_t* pagePtr = nullptr;
+ for (uintptr_t n = 0, i = size_t(gMut->Random64(lock)) % kNumAllocPages;
+ n < kNumAllocPages; n++, i = (i + 1) % kNumAllocPages) {
+ if (!gMut->IsPageAllocatable(lock, i, now)) {
+ continue;
+ }
+
+#if PHC_LOGGING
+ Time lifetime = 0;
+#endif
+ pagePtr = gConst->AllocPagePtr(i);
+ MOZ_ASSERT(pagePtr);
+ bool ok =
+#ifdef XP_WIN
+ !!VirtualAlloc(pagePtr, kPageSize, MEM_COMMIT, PAGE_READWRITE);
+#else
+ mprotect(pagePtr, kPageSize, PROT_READ | PROT_WRITE) == 0;
+#endif
+
+ if (!ok) {
+ pagePtr = nullptr;
+ continue;
+ }
+
+ size_t usableSize = sMallocTable.malloc_good_size(aReqSize);
+ MOZ_ASSERT(usableSize > 0);
+
+ // Put the allocation as close to the end of the page as possible,
+ // allowing for alignment requirements.
+ ptr = pagePtr + kPageSize - usableSize;
+ if (aAlignment != 1) {
+ ptr = reinterpret_cast<uint8_t*>(
+ (reinterpret_cast<uintptr_t>(ptr) & ~(aAlignment - 1)));
+ }
+
+#if PHC_LOGGING
+ Time then = gMut->GetFreeTime(i);
+ lifetime = then != 0 ? now - then : 0;
+#endif
+
+ gMut->SetPageInUse(lock, i, aArenaId, ptr, allocStack);
+
+ if (aZero) {
+ memset(ptr, 0, usableSize);
+ } else {
+#ifdef DEBUG
+ memset(ptr, kAllocJunk, usableSize);
+#endif
+ }
+
+ gMut->IncPageAllocHits(lock);
+#if PHC_LOGGING
+ GMut::PageStats stats = gMut->GetPageStats(lock);
+#endif
+ LOG("PageAlloc(%zu, %zu) -> %p[%zu]/%p (%zu) (z%zu), sAllocDelay <- %zu, "
+ "fullness %zu/%zu/%zu, hits %zu/%zu (%zu%%), lifetime %zu\n",
+ aReqSize, aAlignment, pagePtr, i, ptr, usableSize, size_t(aZero),
+ size_t(newAllocDelay), stats.mNumAlloced, stats.mNumFreed,
+ kNumAllocPages, gMut->PageAllocHits(lock),
+ gMut->PageAllocAttempts(lock), gMut->PageAllocHitRate(lock), lifetime);
+ break;
+ }
+
+ if (!pagePtr) {
+ // No pages are available, or VirtualAlloc/mprotect failed.
+ gMut->IncPageAllocMisses(lock);
+#if PHC_LOGGING
+ GMut::PageStats stats = gMut->GetPageStats(lock);
+#endif
+ LOG("No PageAlloc(%zu, %zu), sAllocDelay <- %zu, fullness %zu/%zu/%zu, "
+ "hits %zu/%zu (%zu%%)\n",
+ aReqSize, aAlignment, size_t(newAllocDelay), stats.mNumAlloced,
+ stats.mNumFreed, kNumAllocPages, gMut->PageAllocHits(lock),
+ gMut->PageAllocAttempts(lock), gMut->PageAllocHitRate(lock));
+ }
+
+ // Set the new alloc delay.
+ GAtomic::SetAllocDelay(newAllocDelay);
+
+ return ptr;
+}
+
+static void FreePage(GMutLock aLock, uintptr_t aIndex,
+ const Maybe<arena_id_t>& aArenaId,
+ const StackTrace& aFreeStack, Delay aReuseDelay) {
+ void* pagePtr = gConst->AllocPagePtr(aIndex);
+
+#ifdef XP_WIN
+ if (!VirtualFree(pagePtr, kPageSize, MEM_DECOMMIT)) {
+ MOZ_CRASH("VirtualFree failed");
+ }
+#else
+ if (mmap(pagePtr, kPageSize, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON,
+ -1, 0) == MAP_FAILED) {
+ MOZ_CRASH("mmap failed");
+ }
+#endif
+
+ gMut->SetPageFreed(aLock, aIndex, aArenaId, aFreeStack, aReuseDelay);
+}
+
+//---------------------------------------------------------------------------
+// replace-malloc machinery
+//---------------------------------------------------------------------------
+
+// This handles malloc, moz_arena_malloc, and realloc-with-a-nullptr.
+MOZ_ALWAYS_INLINE static void* PageMalloc(const Maybe<arena_id_t>& aArenaId,
+ size_t aReqSize) {
+ void* ptr = MaybePageAlloc(aArenaId, aReqSize, /* aAlignment */ 1,
+ /* aZero */ false);
+ return ptr ? ptr
+ : (aArenaId.isSome()
+ ? sMallocTable.moz_arena_malloc(*aArenaId, aReqSize)
+ : sMallocTable.malloc(aReqSize));
+}
+
+static void* replace_malloc(size_t aReqSize) {
+ return PageMalloc(Nothing(), aReqSize);
+}
+
+static Delay ReuseDelay(GMutLock aLock) {
+ return (kAvgPageReuseDelay / 2) +
+ Rnd64ToDelay<kAvgPageReuseDelay / 2>(gMut->Random64(aLock));
+}
+
+// This handles both calloc and moz_arena_calloc.
+MOZ_ALWAYS_INLINE static void* PageCalloc(const Maybe<arena_id_t>& aArenaId,
+ size_t aNum, size_t aReqSize) {
+ CheckedInt<size_t> checkedSize = CheckedInt<size_t>(aNum) * aReqSize;
+ if (!checkedSize.isValid()) {
+ return nullptr;
+ }
+
+ void* ptr = MaybePageAlloc(aArenaId, checkedSize.value(), /* aAlignment */ 1,
+ /* aZero */ true);
+ return ptr ? ptr
+ : (aArenaId.isSome()
+ ? sMallocTable.moz_arena_calloc(*aArenaId, aNum, aReqSize)
+ : sMallocTable.calloc(aNum, aReqSize));
+}
+
+static void* replace_calloc(size_t aNum, size_t aReqSize) {
+ return PageCalloc(Nothing(), aNum, aReqSize);
+}
+
+// This function handles both realloc and moz_arena_realloc.
+//
+// As always, realloc is complicated, and doubly so when there are two
+// different kinds of allocations in play. Here are the possible transitions,
+// and what we do in practice.
+//
+// - normal-to-normal: This is straightforward and obviously necessary.
+//
+// - normal-to-page: This is disallowed because it would require getting the
+// arenaId of the normal allocation, which isn't possible in non-DEBUG builds
+// for security reasons.
+//
+// - page-to-page: This is done whenever possible, i.e. whenever the new size
+// is less than or equal to 4 KiB. This choice counterbalances the
+// disallowing of normal-to-page allocations, in order to avoid biasing
+// towards or away from page allocations. It always occurs in-place.
+//
+// - page-to-normal: this is done only when necessary, i.e. only when the new
+// size is greater than 4 KiB. This choice naturally flows from the
+// prior choice on page-to-page transitions.
+//
+// In summary: realloc doesn't change the allocation kind unless it must.
+//
+MOZ_ALWAYS_INLINE static void* PageRealloc(const Maybe<arena_id_t>& aArenaId,
+ void* aOldPtr, size_t aNewSize) {
+ if (!aOldPtr) {
+ // Null pointer. Treat like malloc(aNewSize).
+ return PageMalloc(aArenaId, aNewSize);
+ }
+
+ PtrKind pk = gConst->PtrKind(aOldPtr);
+ if (pk.IsNothing()) {
+ // A normal-to-normal transition.
+ return aArenaId.isSome()
+ ? sMallocTable.moz_arena_realloc(*aArenaId, aOldPtr, aNewSize)
+ : sMallocTable.realloc(aOldPtr, aNewSize);
+ }
+
+ if (pk.IsGuardPage()) {
+ GMut::CrashOnGuardPage(aOldPtr);
+ }
+
+ // At this point we know we have an allocation page.
+ uintptr_t index = pk.AllocPageIndex();
+
+ // A page-to-something transition.
+
+ // Note that `disable` has no effect unless it is emplaced below.
+ Maybe<AutoDisableOnCurrentThread> disable;
+ // Get the stack trace *before* locking the mutex.
+ StackTrace stack;
+ if (GTls::IsDisabledOnCurrentThread()) {
+ // PHC is disabled on this thread. Leave the stack empty.
+ } else {
+ // Disable on this thread *before* getting the stack trace.
+ disable.emplace();
+ stack.Fill();
+ }
+
+ MutexAutoLock lock(GMut::sMutex);
+
+ // Check for realloc() of a freed block.
+ gMut->EnsureValidAndInUse(lock, aOldPtr, index);
+
+ if (aNewSize <= kPageSize) {
+ // A page-to-page transition. Just keep using the page allocation. We do
+ // this even if the thread is disabled, because it doesn't create a new
+ // page allocation. Note that ResizePageInUse() checks aArenaId.
+ //
+ // Move the bytes with memmove(), because the old allocation and the new
+ // allocation overlap. Move the usable size rather than the requested size,
+ // because the user might have used malloc_usable_size() and filled up the
+ // usable size.
+ size_t oldUsableSize = gMut->PageUsableSize(lock, index);
+ size_t newUsableSize = sMallocTable.malloc_good_size(aNewSize);
+ uint8_t* pagePtr = gConst->AllocPagePtr(index);
+ uint8_t* newPtr = pagePtr + kPageSize - newUsableSize;
+ memmove(newPtr, aOldPtr, std::min(oldUsableSize, aNewSize));
+ gMut->ResizePageInUse(lock, index, aArenaId, newPtr, stack);
+ LOG("PageRealloc-Reuse(%p, %zu) -> %p\n", aOldPtr, aNewSize, newPtr);
+ return newPtr;
+ }
+
+ // A page-to-normal transition (with the new size greater than page-sized).
+ // (Note that aArenaId is checked below.)
+ void* newPtr;
+ if (aArenaId.isSome()) {
+ newPtr = sMallocTable.moz_arena_malloc(*aArenaId, aNewSize);
+ } else {
+ Maybe<arena_id_t> oldArenaId = gMut->PageArena(lock, index);
+ newPtr = (oldArenaId.isSome()
+ ? sMallocTable.moz_arena_malloc(*oldArenaId, aNewSize)
+ : sMallocTable.malloc(aNewSize));
+ }
+ if (!newPtr) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(aNewSize > kPageSize);
+
+ Delay reuseDelay = ReuseDelay(lock);
+
+ // Copy the usable size rather than the requested size, because the user
+ // might have used malloc_usable_size() and filled up the usable size. Note
+ // that FreePage() checks aArenaId (via SetPageFreed()).
+ size_t oldUsableSize = gMut->PageUsableSize(lock, index);
+ memcpy(newPtr, aOldPtr, std::min(oldUsableSize, aNewSize));
+ FreePage(lock, index, aArenaId, stack, reuseDelay);
+ LOG("PageRealloc-Free(%p[%zu], %zu) -> %p, %zu delay, reuse at ~%zu\n",
+ aOldPtr, index, aNewSize, newPtr, size_t(reuseDelay),
+ size_t(GAtomic::Now()) + reuseDelay);
+
+ return newPtr;
+}
+
+static void* replace_realloc(void* aOldPtr, size_t aNewSize) {
+ return PageRealloc(Nothing(), aOldPtr, aNewSize);
+}
+
+// This handles both free and moz_arena_free.
+MOZ_ALWAYS_INLINE static void PageFree(const Maybe<arena_id_t>& aArenaId,
+ void* aPtr) {
+ PtrKind pk = gConst->PtrKind(aPtr);
+ if (pk.IsNothing()) {
+ // Not a page allocation.
+ return aArenaId.isSome() ? sMallocTable.moz_arena_free(*aArenaId, aPtr)
+ : sMallocTable.free(aPtr);
+ }
+
+ if (pk.IsGuardPage()) {
+ GMut::CrashOnGuardPage(aPtr);
+ }
+
+ // At this point we know we have an allocation page.
+ uintptr_t index = pk.AllocPageIndex();
+
+ // Note that `disable` has no effect unless it is emplaced below.
+ Maybe<AutoDisableOnCurrentThread> disable;
+ // Get the stack trace *before* locking the mutex.
+ StackTrace freeStack;
+ if (GTls::IsDisabledOnCurrentThread()) {
+ // PHC is disabled on this thread. Leave the stack empty.
+ } else {
+ // Disable on this thread *before* getting the stack trace.
+ disable.emplace();
+ freeStack.Fill();
+ }
+
+ MutexAutoLock lock(GMut::sMutex);
+
+ // Check for a double-free.
+ gMut->EnsureValidAndInUse(lock, aPtr, index);
+
+ // Note that FreePage() checks aArenaId (via SetPageFreed()).
+ Delay reuseDelay = ReuseDelay(lock);
+ FreePage(lock, index, aArenaId, freeStack, reuseDelay);
+
+#if PHC_LOGGING
+ GMut::PageStats stats = gMut->GetPageStats(lock);
+#endif
+ LOG("PageFree(%p[%zu]), %zu delay, reuse at ~%zu, fullness %zu/%zu/%zu\n",
+ aPtr, index, size_t(reuseDelay), size_t(GAtomic::Now()) + reuseDelay,
+ stats.mNumAlloced, stats.mNumFreed, kNumAllocPages);
+}
+
+static void replace_free(void* aPtr) { return PageFree(Nothing(), aPtr); }
+
+// This handles memalign and moz_arena_memalign.
+MOZ_ALWAYS_INLINE static void* PageMemalign(const Maybe<arena_id_t>& aArenaId,
+ size_t aAlignment,
+ size_t aReqSize) {
+ MOZ_RELEASE_ASSERT(IsPowerOfTwo(aAlignment));
+
+ // PHC can't satisfy an alignment greater than a page size, so fall back to
+ // mozjemalloc in that case.
+ void* ptr = nullptr;
+ if (aAlignment <= kPageSize) {
+ ptr = MaybePageAlloc(aArenaId, aReqSize, aAlignment, /* aZero */ false);
+ }
+ return ptr ? ptr
+ : (aArenaId.isSome()
+ ? sMallocTable.moz_arena_memalign(*aArenaId, aAlignment,
+ aReqSize)
+ : sMallocTable.memalign(aAlignment, aReqSize));
+}
+
+static void* replace_memalign(size_t aAlignment, size_t aReqSize) {
+ return PageMemalign(Nothing(), aAlignment, aReqSize);
+}
+
+static size_t replace_malloc_usable_size(usable_ptr_t aPtr) {
+ PtrKind pk = gConst->PtrKind(aPtr);
+ if (pk.IsNothing()) {
+ // Not a page allocation. Measure it normally.
+ return sMallocTable.malloc_usable_size(aPtr);
+ }
+
+ if (pk.IsGuardPage()) {
+ GMut::CrashOnGuardPage(const_cast<void*>(aPtr));
+ }
+
+ // At this point we know aPtr lands within an allocation page, due to the
+ // math done in the PtrKind constructor. But if aPtr points to memory
+ // before the base address of the allocation, we return 0.
+ uintptr_t index = pk.AllocPageIndex();
+
+ MutexAutoLock lock(GMut::sMutex);
+
+ void* pageBaseAddr = gMut->AllocPageBaseAddr(lock, index);
+
+ if (MOZ_UNLIKELY(aPtr < pageBaseAddr)) {
+ return 0;
+ }
+
+ return gMut->PageUsableSize(lock, index);
+}
+
+static size_t metadata_size() {
+ return sMallocTable.malloc_usable_size(gConst) +
+ sMallocTable.malloc_usable_size(gMut);
+}
+
+void replace_jemalloc_stats(jemalloc_stats_t* aStats,
+ jemalloc_bin_stats_t* aBinStats) {
+ sMallocTable.jemalloc_stats_internal(aStats, aBinStats);
+
+ // Add all the pages to `mapped`.
+ size_t mapped = kAllPagesSize;
+ aStats->mapped += mapped;
+
+ size_t allocated = 0;
+ {
+ MutexAutoLock lock(GMut::sMutex);
+
+ // Add usable space of in-use allocations to `allocated`.
+ for (size_t i = 0; i < kNumAllocPages; i++) {
+ if (gMut->IsPageInUse(lock, i)) {
+ allocated += gMut->PageUsableSize(lock, i);
+ }
+ }
+ }
+ aStats->allocated += allocated;
+
+ // guards is the gap between `allocated` and `mapped`. In some ways this
+ // almost fits into aStats->wasted since it feels like wasted memory. However
+ // wasted should only include committed memory and these guard pages are
+ // uncommitted. Therefore we don't include it anywhere.
+ // size_t guards = mapped - allocated;
+
+ // aStats.page_cache and aStats.bin_unused are left unchanged because PHC
+ // doesn't have anything corresponding to those.
+
+ // The metadata is stored in normal heap allocations, so they're measured by
+ // mozjemalloc as `allocated`. Move them into `bookkeeping`.
+ // They're also reported under explicit/heap-overhead/phc/fragmentation in
+ // about:memory.
+ size_t bookkeeping = metadata_size();
+ aStats->allocated -= bookkeeping;
+ aStats->bookkeeping += bookkeeping;
+}
+
+void replace_jemalloc_ptr_info(const void* aPtr, jemalloc_ptr_info_t* aInfo) {
+ // We need to implement this properly, because various code locations do
+ // things like checking that allocations are in the expected arena.
+ PtrKind pk = gConst->PtrKind(aPtr);
+ if (pk.IsNothing()) {
+ // Not a page allocation.
+ return sMallocTable.jemalloc_ptr_info(aPtr, aInfo);
+ }
+
+ if (pk.IsGuardPage()) {
+ // Treat a guard page as unknown because there's no better alternative.
+ *aInfo = {TagUnknown, nullptr, 0, 0};
+ return;
+ }
+
+ // At this point we know we have an allocation page.
+ uintptr_t index = pk.AllocPageIndex();
+
+ MutexAutoLock lock(GMut::sMutex);
+
+ gMut->FillJemallocPtrInfo(lock, aPtr, index, aInfo);
+#if DEBUG
+ LOG("JemallocPtrInfo(%p[%zu]) -> {%zu, %p, %zu, %zu}\n", aPtr, index,
+ size_t(aInfo->tag), aInfo->addr, aInfo->size, aInfo->arenaId);
+#else
+ LOG("JemallocPtrInfo(%p[%zu]) -> {%zu, %p, %zu}\n", aPtr, index,
+ size_t(aInfo->tag), aInfo->addr, aInfo->size);
+#endif
+}
+
+arena_id_t replace_moz_create_arena_with_params(arena_params_t* aParams) {
+ // No need to do anything special here.
+ return sMallocTable.moz_create_arena_with_params(aParams);
+}
+
+void replace_moz_dispose_arena(arena_id_t aArenaId) {
+ // No need to do anything special here.
+ return sMallocTable.moz_dispose_arena(aArenaId);
+}
+
+void replace_moz_set_max_dirty_page_modifier(int32_t aModifier) {
+ // No need to do anything special here.
+ return sMallocTable.moz_set_max_dirty_page_modifier(aModifier);
+}
+
+void* replace_moz_arena_malloc(arena_id_t aArenaId, size_t aReqSize) {
+ return PageMalloc(Some(aArenaId), aReqSize);
+}
+
+void* replace_moz_arena_calloc(arena_id_t aArenaId, size_t aNum,
+ size_t aReqSize) {
+ return PageCalloc(Some(aArenaId), aNum, aReqSize);
+}
+
+void* replace_moz_arena_realloc(arena_id_t aArenaId, void* aOldPtr,
+ size_t aNewSize) {
+ return PageRealloc(Some(aArenaId), aOldPtr, aNewSize);
+}
+
+void replace_moz_arena_free(arena_id_t aArenaId, void* aPtr) {
+ return PageFree(Some(aArenaId), aPtr);
+}
+
+void* replace_moz_arena_memalign(arena_id_t aArenaId, size_t aAlignment,
+ size_t aReqSize) {
+ return PageMemalign(Some(aArenaId), aAlignment, aReqSize);
+}
+
+class PHCBridge : public ReplaceMallocBridge {
+ virtual bool IsPHCAllocation(const void* aPtr, phc::AddrInfo* aOut) override {
+ PtrKind pk = gConst->PtrKind(aPtr);
+ if (pk.IsNothing()) {
+ return false;
+ }
+
+ bool isGuardPage = false;
+ if (pk.IsGuardPage()) {
+ if ((uintptr_t(aPtr) % kPageSize) < (kPageSize / 2)) {
+ // The address is in the lower half of a guard page, so it's probably an
+ // overflow. But first check that it is not on the very first guard
+ // page, in which case it cannot be an overflow, and we ignore it.
+ if (gConst->IsInFirstGuardPage(aPtr)) {
+ return false;
+ }
+
+ // Get the allocation page preceding this guard page.
+ pk = gConst->PtrKind(static_cast<const uint8_t*>(aPtr) - kPageSize);
+
+ } else {
+ // The address is in the upper half of a guard page, so it's probably an
+ // underflow. Get the allocation page following this guard page.
+ pk = gConst->PtrKind(static_cast<const uint8_t*>(aPtr) + kPageSize);
+ }
+
+ // Make a note of the fact that we hit a guard page.
+ isGuardPage = true;
+ }
+
+ // At this point we know we have an allocation page.
+ uintptr_t index = pk.AllocPageIndex();
+
+ if (aOut) {
+ MutexAutoLock lock(GMut::sMutex);
+ gMut->FillAddrInfo(lock, index, aPtr, isGuardPage, *aOut);
+ LOG("IsPHCAllocation: %zu, %p, %zu, %zu, %zu\n", size_t(aOut->mKind),
+ aOut->mBaseAddr, aOut->mUsableSize,
+ aOut->mAllocStack.isSome() ? aOut->mAllocStack->mLength : 0,
+ aOut->mFreeStack.isSome() ? aOut->mFreeStack->mLength : 0);
+ }
+ return true;
+ }
+
+ virtual void DisablePHCOnCurrentThread() override {
+ GTls::DisableOnCurrentThread();
+ LOG("DisablePHCOnCurrentThread: %zu\n", 0ul);
+ }
+
+ virtual void ReenablePHCOnCurrentThread() override {
+ GTls::EnableOnCurrentThread();
+ LOG("ReenablePHCOnCurrentThread: %zu\n", 0ul);
+ }
+
+ virtual bool IsPHCEnabledOnCurrentThread() override {
+ bool enabled = !GTls::IsDisabledOnCurrentThread();
+ LOG("IsPHCEnabledOnCurrentThread: %zu\n", size_t(enabled));
+ return enabled;
+ }
+
+ virtual void PHCMemoryUsage(
+ mozilla::phc::MemoryUsage& aMemoryUsage) override {
+ aMemoryUsage.mMetadataBytes = metadata_size();
+ if (gMut) {
+ MutexAutoLock lock(GMut::sMutex);
+ aMemoryUsage.mFragmentationBytes = gMut->FragmentationBytes();
+ } else {
+ aMemoryUsage.mFragmentationBytes = 0;
+ }
+ }
+};
+
+// WARNING: this function runs *very* early -- before all static initializers
+// have run. For this reason, non-scalar globals (gConst, gMut) are allocated
+// dynamically (so we can guarantee their construction in this function) rather
+// than statically. GAtomic and GTls contain simple static data that doesn't
+// involve static initializers so they don't need to be allocated dynamically.
+void replace_init(malloc_table_t* aMallocTable, ReplaceMallocBridge** aBridge) {
+ // Don't run PHC if the page size isn't 4 KiB.
+ jemalloc_stats_t stats;
+ aMallocTable->jemalloc_stats_internal(&stats, nullptr);
+ if (stats.page_size != kPageSize) {
+ return;
+ }
+
+ sMallocTable = *aMallocTable;
+
+ // The choices of which functions to replace are complex enough that we set
+ // them individually instead of using MALLOC_FUNCS/malloc_decls.h.
+
+ aMallocTable->malloc = replace_malloc;
+ aMallocTable->calloc = replace_calloc;
+ aMallocTable->realloc = replace_realloc;
+ aMallocTable->free = replace_free;
+ aMallocTable->memalign = replace_memalign;
+
+ // posix_memalign, aligned_alloc & valloc: unset, which means they fall back
+ // to replace_memalign.
+ aMallocTable->malloc_usable_size = replace_malloc_usable_size;
+ // default malloc_good_size: the default suffices.
+
+ aMallocTable->jemalloc_stats_internal = replace_jemalloc_stats;
+ // jemalloc_purge_freed_pages: the default suffices.
+ // jemalloc_free_dirty_pages: the default suffices.
+ // jemalloc_thread_local_arena: the default suffices.
+ aMallocTable->jemalloc_ptr_info = replace_jemalloc_ptr_info;
+
+ aMallocTable->moz_create_arena_with_params =
+ replace_moz_create_arena_with_params;
+ aMallocTable->moz_dispose_arena = replace_moz_dispose_arena;
+ aMallocTable->moz_arena_malloc = replace_moz_arena_malloc;
+ aMallocTable->moz_arena_calloc = replace_moz_arena_calloc;
+ aMallocTable->moz_arena_realloc = replace_moz_arena_realloc;
+ aMallocTable->moz_arena_free = replace_moz_arena_free;
+ aMallocTable->moz_arena_memalign = replace_moz_arena_memalign;
+
+ static PHCBridge bridge;
+ *aBridge = &bridge;
+
+#ifndef XP_WIN
+ // Avoid deadlocks when forking by acquiring our state lock prior to forking
+ // and releasing it after forking. See |LogAlloc|'s |replace_init| for
+ // in-depth details.
+ //
+ // Note: This must run after attempting an allocation so as to give the
+ // system malloc a chance to insert its own atfork handler.
+ sMallocTable.malloc(-1);
+ pthread_atfork(GMut::prefork, GMut::postfork_parent, GMut::postfork_child);
+#endif
+
+ // gConst and gMut are never freed. They live for the life of the process.
+ gConst = InfallibleAllocPolicy::new_<GConst>();
+ GTls::Init();
+ gMut = InfallibleAllocPolicy::new_<GMut>();
+ {
+ MutexAutoLock lock(GMut::sMutex);
+ Delay firstAllocDelay =
+ Rnd64ToDelay<kAvgFirstAllocDelay>(gMut->Random64(lock));
+ GAtomic::Init(firstAllocDelay);
+ }
+}
diff --git a/memory/replace/phc/PHC.h b/memory/replace/phc/PHC.h
new file mode 100644
index 0000000000..139a0e81c9
--- /dev/null
+++ b/memory/replace/phc/PHC.h
@@ -0,0 +1,102 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef PHC_h
+#define PHC_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Maybe.h"
+#include <stdint.h>
+#include <stdlib.h>
+
+namespace mozilla {
+namespace phc {
+
+// Note: a stack trace may have no frames due to a collection problem.
+//
+// Also note: a more compact stack trace representation could be achieved with
+// some effort.
+struct StackTrace {
+ public:
+ static const size_t kMaxFrames = 16;
+
+ // The number of PCs in the stack trace.
+ size_t mLength;
+
+ // The PCs in the stack trace. Only the first mLength are initialized.
+ const void* mPcs[kMaxFrames];
+
+ public:
+ StackTrace() : mLength(0) {}
+};
+
+// Info from PHC about an address in memory.
+class AddrInfo {
+ public:
+ enum class Kind {
+ // The address is not in PHC-managed memory.
+ Unknown = 0,
+
+ // The address is within a PHC page that has never been allocated. A crash
+ // involving such an address is unlikely in practice, because it would
+ // require the crash to happen quite early.
+ NeverAllocatedPage = 1,
+
+ // The address is within a PHC page that is in use.
+ InUsePage = 2,
+
+ // The address is within a PHC page that has been allocated and then freed.
+ // A crash involving such an address most likely indicates a
+ // use-after-free. (A sufficiently wild write -- e.g. a large buffer
+ // overflow -- could also trigger it, but this is less likely.)
+ FreedPage = 3,
+
+ // The address is within a PHC guard page. A crash involving such an
+ // address most likely indicates a buffer overflow. (Again, a sufficiently
+ // wild write could unluckily trigger it, but this is less likely.)
+ GuardPage = 4,
+ };
+
+ // The page kind.
+ Kind mKind;
+
+ // The starting address of the allocation.
+ // - Unknown | NeverAllocatedPage: nullptr.
+ // - InUsePage | FreedPage: the address of the allocation within the page.
+ // - GuardPage: the mBaseAddr value from the preceding allocation page.
+ const void* mBaseAddr;
+
+ // The usable size, which could be bigger than the requested size.
+ // - Unknown | NeverAllocatePage: 0.
+ // - InUsePage | FreedPage: the usable size of the allocation within the page.
+ // - GuardPage: the mUsableSize value from the preceding allocation page.
+ size_t mUsableSize;
+
+ // The allocation stack.
+ // - Unknown | NeverAllocatedPage: Nothing.
+ // - InUsePage | FreedPage: Some.
+ // - GuardPage: the mAllocStack value from the preceding allocation page.
+ mozilla::Maybe<StackTrace> mAllocStack;
+
+ // The free stack.
+ // - Unknown | NeverAllocatedPage | InUsePage: Nothing.
+ // - FreedPage: Some.
+ // - GuardPage: the mFreeStack value from the preceding allocation page.
+ mozilla::Maybe<StackTrace> mFreeStack;
+
+ // Default to no PHC info.
+ AddrInfo()
+ : mKind(Kind::Unknown),
+ mBaseAddr(nullptr),
+ mUsableSize(0),
+ mAllocStack(),
+ mFreeStack() {}
+};
+
+} // namespace phc
+} // namespace mozilla
+
+#endif /* PHC_h */
diff --git a/memory/replace/phc/moz.build b/memory/replace/phc/moz.build
new file mode 100644
index 0000000000..e2d233642c
--- /dev/null
+++ b/memory/replace/phc/moz.build
@@ -0,0 +1,50 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+ReplaceMalloc("phc")
+
+DEFINES["MOZ_NO_MOZALLOC"] = True
+DEFINES["IMPL_MFBT"] = True
+
+LOCAL_INCLUDES += [
+ "../logalloc",
+ "/memory/build",
+]
+
+EXPORTS += [
+ "PHC.h",
+]
+
+UNIFIED_SOURCES += [
+ "PHC.cpp",
+]
+
+if CONFIG["MOZ_BUILD_APP"] == "memory":
+ UNIFIED_SOURCES += [
+ "/mfbt/double-conversion/double-conversion/bignum-dtoa.cc",
+ "/mfbt/double-conversion/double-conversion/bignum.cc",
+ "/mfbt/double-conversion/double-conversion/cached-powers.cc",
+ "/mfbt/double-conversion/double-conversion/double-to-string.cc",
+ "/mfbt/double-conversion/double-conversion/fast-dtoa.cc",
+ "/mfbt/double-conversion/double-conversion/fixed-dtoa.cc",
+ "/mfbt/double-conversion/double-conversion/string-to-double.cc",
+ "/mfbt/double-conversion/double-conversion/strtod.cc",
+ "/mozglue/misc/Printf.cpp",
+ ]
+
+if not CONFIG["MOZ_REPLACE_MALLOC_STATIC"]:
+ UNIFIED_SOURCES += [
+ "../logalloc/FdPrintf.cpp",
+ "/mozglue/misc/StackWalk.cpp",
+ ]
+ if CONFIG["OS_ARCH"] == "WINNT":
+ OS_LIBS += [
+ "dbghelp",
+ ]
+
+TEST_DIRS += ["test"]
+
+DisableStlWrapping()
diff --git a/memory/replace/phc/test/gtest/TestPHC.cpp b/memory/replace/phc/test/gtest/TestPHC.cpp
new file mode 100644
index 0000000000..738a50eee2
--- /dev/null
+++ b/memory/replace/phc/test/gtest/TestPHC.cpp
@@ -0,0 +1,305 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+
+#include "mozmemory.h"
+#include "replace_malloc_bridge.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/mozalloc.h"
+#include "../../PHC.h"
+
+using namespace mozilla;
+
+bool PHCInfoEq(phc::AddrInfo& aInfo, phc::AddrInfo::Kind aKind, void* aBaseAddr,
+ size_t aUsableSize, bool aHasAllocStack, bool aHasFreeStack) {
+ return aInfo.mKind == aKind && aInfo.mBaseAddr == aBaseAddr &&
+ aInfo.mUsableSize == aUsableSize &&
+ // Proper stack traces will have at least 3 elements.
+ (aHasAllocStack ? (aInfo.mAllocStack->mLength > 2)
+ : (aInfo.mAllocStack.isNothing())) &&
+ (aHasFreeStack ? (aInfo.mFreeStack->mLength > 2)
+ : (aInfo.mFreeStack.isNothing()));
+}
+
+bool JeInfoEq(jemalloc_ptr_info_t& aInfo, PtrInfoTag aTag, void* aAddr,
+ size_t aSize, arena_id_t arenaId) {
+ return aInfo.tag == aTag && aInfo.addr == aAddr && aInfo.size == aSize
+#ifdef MOZ_DEBUG
+ && aInfo.arenaId == arenaId
+#endif
+ ;
+}
+
+uint8_t* GetPHCAllocation(size_t aSize, size_t aAlignment = 1) {
+ // A crude but effective way to get a PHC allocation.
+ for (int i = 0; i < 2000000; i++) {
+ void* p = (aAlignment == 1) ? moz_xmalloc(aSize)
+ : moz_xmemalign(aAlignment, aSize);
+ if (ReplaceMalloc::IsPHCAllocation(p, nullptr)) {
+ return (uint8_t*)p;
+ }
+ free(p);
+ }
+ return nullptr;
+}
+
+static const size_t kPageSize = 4096;
+
+TEST(PHC, TestPHCAllocations)
+{
+ // First, check that allocations of various sizes all get put at the end of
+ // their page as expected. Also, check their sizes are as expected.
+
+#define ASSERT_POS(n1, n2) \
+ p = (uint8_t*)moz_xrealloc(p, (n1)); \
+ ASSERT_EQ((reinterpret_cast<uintptr_t>(p) & (kPageSize - 1)), \
+ kPageSize - (n2)); \
+ ASSERT_EQ(moz_malloc_usable_size(p), (n2));
+
+ uint8_t* p = GetPHCAllocation(1);
+ if (!p) {
+ MOZ_CRASH("failed to get a PHC allocation");
+ }
+
+ // On Win64 the smallest possible allocation is 16 bytes. On other platforms
+ // it is 8 bytes.
+#if defined(XP_WIN) && defined(HAVE_64BIT_BUILD)
+ ASSERT_POS(8U, 16U);
+#else
+ ASSERT_POS(8U, 8U);
+#endif
+ ASSERT_POS(16U, 16U);
+ ASSERT_POS(32U, 32U);
+ ASSERT_POS(64U, 64U);
+ ASSERT_POS(128U, 128U);
+ ASSERT_POS(256U, 256U);
+ ASSERT_POS(512U, 512U);
+ ASSERT_POS(1024U, 1024U);
+ ASSERT_POS(2048U, 2048U);
+ ASSERT_POS(4096U, 4096U);
+
+ free(p);
+
+#undef ASSERT_POS
+
+ // Second, do similar checking with allocations of various alignments. Also
+ // check that their sizes (which are different to allocations with normal
+ // alignment) are the same as the sizes of equivalent non-PHC allocations.
+
+#define ASSERT_ALIGN(a1, a2) \
+ p = (uint8_t*)GetPHCAllocation(8, (a1)); \
+ ASSERT_EQ((reinterpret_cast<uintptr_t>(p) & (kPageSize - 1)), \
+ kPageSize - (a2)); \
+ ASSERT_EQ(moz_malloc_usable_size(p), (a2)); \
+ free(p); \
+ p = (uint8_t*)moz_xmemalign((a1), 8); \
+ ASSERT_EQ(moz_malloc_usable_size(p), (a2)); \
+ free(p);
+
+ // On Win64 the smallest possible allocation is 16 bytes. On other platforms
+ // it is 8 bytes.
+#if defined(XP_WIN) && defined(HAVE_64BIT_BUILD)
+ ASSERT_ALIGN(8U, 16U);
+#else
+ ASSERT_ALIGN(8U, 8U);
+#endif
+ ASSERT_ALIGN(16U, 16U);
+ ASSERT_ALIGN(32U, 32U);
+ ASSERT_ALIGN(64U, 64U);
+ ASSERT_ALIGN(128U, 128U);
+ ASSERT_ALIGN(256U, 256U);
+ ASSERT_ALIGN(512U, 512U);
+ ASSERT_ALIGN(1024U, 1024U);
+ ASSERT_ALIGN(2048U, 2048U);
+ ASSERT_ALIGN(4096U, 4096U);
+
+#undef ASSERT_ALIGN
+}
+
+TEST(PHC, TestPHCInfo)
+{
+ int stackVar;
+ phc::AddrInfo phcInfo;
+ jemalloc_ptr_info_t jeInfo;
+
+ // Test a default AddrInfo.
+ ASSERT_TRUE(PHCInfoEq(phcInfo, phc::AddrInfo::Kind::Unknown, nullptr, 0ul,
+ false, false));
+
+ // Test some non-PHC allocation addresses.
+ ASSERT_FALSE(ReplaceMalloc::IsPHCAllocation(nullptr, &phcInfo));
+ ASSERT_TRUE(PHCInfoEq(phcInfo, phc::AddrInfo::Kind::Unknown, nullptr, 0,
+ false, false));
+ ASSERT_FALSE(ReplaceMalloc::IsPHCAllocation(&stackVar, &phcInfo));
+ ASSERT_TRUE(PHCInfoEq(phcInfo, phc::AddrInfo::Kind::Unknown, nullptr, 0,
+ false, false));
+
+ uint8_t* p = GetPHCAllocation(32);
+ if (!p) {
+ MOZ_CRASH("failed to get a PHC allocation");
+ }
+
+ // Test an in-use PHC allocation: first byte within it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::InUsePage, p, 32ul, true, false));
+ ASSERT_EQ(moz_malloc_usable_size(p), 32ul);
+ jemalloc_ptr_info(p, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagLiveAlloc, p, 32, 0));
+
+ // Test an in-use PHC allocation: last byte within it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 31, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::InUsePage, p, 32ul, true, false));
+ ASSERT_EQ(moz_malloc_usable_size(p + 31), 32ul);
+ jemalloc_ptr_info(p + 31, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagLiveAlloc, p, 32, 0));
+
+ // Test an in-use PHC allocation: last byte before it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p - 1, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::InUsePage, p, 32ul, true, false));
+ ASSERT_EQ(moz_malloc_usable_size(p - 1), 0ul);
+ jemalloc_ptr_info(p - 1, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test an in-use PHC allocation: first byte on its allocation page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 32 - kPageSize, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::InUsePage, p, 32ul, true, false));
+ jemalloc_ptr_info(p + 32 - kPageSize, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test an in-use PHC allocation: first byte in the following guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 32, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, false));
+ jemalloc_ptr_info(p + 32, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test an in-use PHC allocation: last byte in the lower half of the
+ // following guard page.
+ ASSERT_TRUE(
+ ReplaceMalloc::IsPHCAllocation(p + 32 + (kPageSize / 2 - 1), &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, false));
+ jemalloc_ptr_info(p + 32 + (kPageSize / 2 - 1), &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test an in-use PHC allocation: last byte in the preceding guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 31 - kPageSize, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, false));
+ jemalloc_ptr_info(p + 31 - kPageSize, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test an in-use PHC allocation: first byte in the upper half of the
+ // preceding guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(
+ p + 31 - kPageSize - (kPageSize / 2 - 1), &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, false));
+ jemalloc_ptr_info(p + 31 - kPageSize - (kPageSize / 2 - 1), &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ free(p);
+
+ // Test a freed PHC allocation: first byte within it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::FreedPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagFreedAlloc, p, 32, 0));
+
+ // Test a freed PHC allocation: last byte within it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 31, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::FreedPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 31, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagFreedAlloc, p, 32, 0));
+
+ // Test a freed PHC allocation: last byte before it.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p - 1, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::FreedPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p - 1, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test a freed PHC allocation: first byte on its allocation page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 32 - kPageSize, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::FreedPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 32 - kPageSize, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test a freed PHC allocation: first byte in the following guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 32, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 32, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test a freed PHC allocation: last byte in the lower half of the following
+ // guard page.
+ ASSERT_TRUE(
+ ReplaceMalloc::IsPHCAllocation(p + 32 + (kPageSize / 2 - 1), &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 32 + (kPageSize / 2 - 1), &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test a freed PHC allocation: last byte in the preceding guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p + 31 - kPageSize, &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 31 - kPageSize, &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // Test a freed PHC allocation: first byte in the upper half of the preceding
+ // guard page.
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(
+ p + 31 - kPageSize - (kPageSize / 2 - 1), &phcInfo));
+ ASSERT_TRUE(
+ PHCInfoEq(phcInfo, phc::AddrInfo::Kind::GuardPage, p, 32ul, true, true));
+ jemalloc_ptr_info(p + 31 - kPageSize - (kPageSize / 2 - 1), &jeInfo);
+ ASSERT_TRUE(JeInfoEq(jeInfo, TagUnknown, nullptr, 0, 0));
+
+ // There are no tests for `mKind == NeverAllocatedPage` because it's not
+ // possible to reliably get ahold of such a page.
+}
+
+TEST(PHC, TestPHCDisabling)
+{
+ uint8_t* p = GetPHCAllocation(32);
+ uint8_t* q = GetPHCAllocation(32);
+ if (!p || !q) {
+ MOZ_CRASH("failed to get a PHC allocation");
+ }
+
+ ASSERT_TRUE(ReplaceMalloc::IsPHCEnabledOnCurrentThread());
+ ReplaceMalloc::DisablePHCOnCurrentThread();
+ ASSERT_FALSE(ReplaceMalloc::IsPHCEnabledOnCurrentThread());
+
+ // Test realloc() on a PHC allocation while PHC is disabled on the thread.
+ uint8_t* p2 = (uint8_t*)realloc(p, 128);
+ // The small realloc is fulfilled within the same page, but it does move.
+ ASSERT_TRUE(p2 == p - 96);
+ ASSERT_TRUE(ReplaceMalloc::IsPHCAllocation(p2, nullptr));
+ uint8_t* p3 = (uint8_t*)realloc(p2, 8192);
+ // The big realloc is not in-place, and the result is not a PHC allocation.
+ ASSERT_TRUE(p3 != p2);
+ ASSERT_FALSE(ReplaceMalloc::IsPHCAllocation(p3, nullptr));
+ free(p3);
+
+ // Test free() on a PHC allocation while PHC is disabled on the thread.
+ free(q);
+
+ // These must not be PHC allocations.
+ uint8_t* r = GetPHCAllocation(32); // This will fail.
+ ASSERT_FALSE(!!r);
+
+ ReplaceMalloc::ReenablePHCOnCurrentThread();
+ ASSERT_TRUE(ReplaceMalloc::IsPHCEnabledOnCurrentThread());
+}
diff --git a/memory/replace/phc/test/gtest/moz.build b/memory/replace/phc/test/gtest/moz.build
new file mode 100644
index 0000000000..82ccaaf9c6
--- /dev/null
+++ b/memory/replace/phc/test/gtest/moz.build
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+ "TestPHC.cpp",
+]
+
+LOCAL_INCLUDES += [
+ "../../",
+]
+
+FINAL_LIBRARY = "xul-gtest"
diff --git a/memory/replace/phc/test/moz.build b/memory/replace/phc/test/moz.build
new file mode 100644
index 0000000000..8208ae849d
--- /dev/null
+++ b/memory/replace/phc/test/moz.build
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# The gtests won't work in a SpiderMonkey-only build.
+if CONFIG["MOZ_WIDGET_TOOLKIT"]:
+ TEST_DIRS += ["gtest"]
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-28 20:29:39 +0000