summaryrefslogtreecommitdiffstats
path: root/lib/isc/mem.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 15:59:48 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 15:59:48 +0000
commit3b9b6d0b8e7f798023c9d109c490449d528fde80 (patch)
tree2e1c188dd7b8d7475cd163de9ae02c428343669b /lib/isc/mem.c
parentInitial commit. (diff)
downloadbind9-3b9b6d0b8e7f798023c9d109c490449d528fde80.tar.xz
bind9-3b9b6d0b8e7f798023c9d109c490449d528fde80.zip
Adding upstream version 1:9.18.19.upstream/1%9.18.19upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--lib/isc/mem.c1908
1 files changed, 1908 insertions, 0 deletions
diff --git a/lib/isc/mem.c b/lib/isc/mem.c
new file mode 100644
index 0000000..61a66f6
--- /dev/null
+++ b/lib/isc/mem.c
@@ -0,0 +1,1908 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/*! \file */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <isc/align.h>
+#include <isc/hash.h>
+#include <isc/magic.h>
+#include <isc/mem.h>
+#include <isc/mutex.h>
+#include <isc/once.h>
+#include <isc/os.h>
+#include <isc/print.h>
+#include <isc/refcount.h>
+#include <isc/string.h>
+#include <isc/types.h>
+#include <isc/util.h>
+
+#ifdef HAVE_LIBXML2
+#include <libxml/xmlwriter.h>
+#define ISC_XMLCHAR (const xmlChar *)
+#endif /* HAVE_LIBXML2 */
+
+#ifdef HAVE_JSON_C
+#include <json_object.h>
+#endif /* HAVE_JSON_C */
+
+/* On DragonFly BSD the header does not provide jemalloc API */
+#if defined(HAVE_MALLOC_NP_H) && !defined(__DragonFly__)
+#include <malloc_np.h>
+#define JEMALLOC_API_SUPPORTED 1
+#elif defined(HAVE_JEMALLOC)
+#include <jemalloc/jemalloc.h>
+#define JEMALLOC_API_SUPPORTED 1
+
+#if JEMALLOC_VERSION_MAJOR < 4
+#define sdallocx(ptr, size, flags) dallocx(ptr, flags)
+#define MALLOCX_TCACHE_NONE (0)
+#endif /* JEMALLOC_VERSION_MAJOR < 4 */
+
+#else
+#include "jemalloc_shim.h"
+#endif
+
+#include "mem_p.h"
+
+#define MCTXLOCK(m) LOCK(&m->lock)
+#define MCTXUNLOCK(m) UNLOCK(&m->lock)
+
+#ifndef ISC_MEM_DEBUGGING
+#define ISC_MEM_DEBUGGING 0
+#endif /* ifndef ISC_MEM_DEBUGGING */
+unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING;
+unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT;
+
+#define ISC_MEM_ILLEGAL_ARENA (UINT_MAX)
+
+/*
+ * Constants.
+ */
+
+#define ZERO_ALLOCATION_SIZE sizeof(void *)
+#define ALIGNMENT 8U /*%< must be a power of 2 */
+#define ALIGNMENT_SIZE sizeof(size_info)
+#define DEBUG_TABLE_COUNT 512U
+#define STATS_BUCKETS 512U
+#define STATS_BUCKET_SIZE 32U
+
+/*
+ * Types.
+ */
+#if ISC_MEM_TRACKLINES
+typedef struct debuglink debuglink_t;
+struct debuglink {
+ ISC_LINK(debuglink_t) link;
+ const void *ptr;
+ size_t size;
+ const char *file;
+ unsigned int line;
+};
+
+typedef ISC_LIST(debuglink_t) debuglist_t;
+
+#define FLARG_PASS , file, line
+#define FLARG , const char *file, unsigned int line
+#else /* if ISC_MEM_TRACKLINES */
+#define FLARG_PASS
+#define FLARG
+#endif /* if ISC_MEM_TRACKLINES */
+
+typedef struct element element;
+struct element {
+ element *next;
+};
+
+struct stats {
+ atomic_size_t gets;
+ atomic_size_t totalgets;
+};
+
+#define MEM_MAGIC ISC_MAGIC('M', 'e', 'm', 'C')
+#define VALID_CONTEXT(c) ISC_MAGIC_VALID(c, MEM_MAGIC)
+
+/* List of all active memory contexts. */
+
+static ISC_LIST(isc_mem_t) contexts;
+
+static isc_once_t init_once = ISC_ONCE_INIT;
+static isc_once_t shut_once = ISC_ONCE_INIT;
+static isc_mutex_t contextslock;
+
+/*%
+ * Total size of lost memory due to a bug of external library.
+ * Locked by the global lock.
+ */
+static uint64_t totallost;
+
+struct isc_mem {
+ unsigned int magic;
+ unsigned int flags;
+ unsigned int jemalloc_flags;
+ unsigned int jemalloc_arena;
+ isc_mutex_t lock;
+ bool checkfree;
+ struct stats stats[STATS_BUCKETS + 1];
+ isc_refcount_t references;
+ char name[16];
+ atomic_size_t total;
+ atomic_size_t inuse;
+ atomic_size_t maxinuse;
+ atomic_size_t malloced;
+ atomic_size_t maxmalloced;
+ atomic_bool hi_called;
+ atomic_bool is_overmem;
+ isc_mem_water_t water;
+ void *water_arg;
+ atomic_size_t hi_water;
+ atomic_size_t lo_water;
+ ISC_LIST(isc_mempool_t) pools;
+ unsigned int poolcnt;
+
+#if ISC_MEM_TRACKLINES
+ debuglist_t *debuglist;
+ size_t debuglistcnt;
+#endif /* if ISC_MEM_TRACKLINES */
+
+ ISC_LINK(isc_mem_t) link;
+};
+
+#define MEMPOOL_MAGIC ISC_MAGIC('M', 'E', 'M', 'p')
+#define VALID_MEMPOOL(c) ISC_MAGIC_VALID(c, MEMPOOL_MAGIC)
+
+struct isc_mempool {
+ /* always unlocked */
+ unsigned int magic;
+ isc_mem_t *mctx; /*%< our memory context */
+ ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */
+ element *items; /*%< low water item list */
+ size_t size; /*%< size of each item on this pool */
+ size_t allocated; /*%< # of items currently given out */
+ size_t freecount; /*%< # of items on reserved list */
+ size_t freemax; /*%< # of items allowed on free list */
+ size_t fillcount; /*%< # of items to fetch on each fill */
+ /*%< Stats only. */
+ size_t gets; /*%< # of requests to this pool */
+ /*%< Debugging only. */
+ char name[16]; /*%< printed name in stats reports */
+};
+
+/*
+ * Private Inline-able.
+ */
+
+#if !ISC_MEM_TRACKLINES
+#define ADD_TRACE(a, b, c, d, e)
+#define DELETE_TRACE(a, b, c, d, e)
+#define ISC_MEMFUNC_SCOPE
+#else /* if !ISC_MEM_TRACKLINES */
+#define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD)
+
+#define SHOULD_TRACE_OR_RECORD(ptr) \
+ ((isc_mem_debugging & TRACE_OR_RECORD) != 0 && ptr != NULL)
+
+#define ADD_TRACE(a, b, c, d, e) \
+ if (SHOULD_TRACE_OR_RECORD(b)) { \
+ add_trace_entry(a, b, c, d, e); \
+ }
+
+#define DELETE_TRACE(a, b, c, d, e) \
+ if (SHOULD_TRACE_OR_RECORD(b)) { \
+ delete_trace_entry(a, b, c, d, e); \
+ }
+
+static void
+print_active(isc_mem_t *ctx, FILE *out);
+#endif /* ISC_MEM_TRACKLINES */
+
+static size_t
+increment_malloced(isc_mem_t *ctx, size_t size) {
+ size_t malloced = atomic_fetch_add_relaxed(&ctx->malloced, size) + size;
+ size_t maxmalloced = atomic_load_relaxed(&ctx->maxmalloced);
+
+ if (malloced > maxmalloced) {
+ atomic_compare_exchange_strong(&ctx->maxmalloced, &maxmalloced,
+ malloced);
+ }
+
+ return (malloced);
+}
+
+static size_t
+decrement_malloced(isc_mem_t *ctx, size_t size) {
+ size_t malloced = atomic_fetch_sub_relaxed(&ctx->malloced, size) - size;
+
+ return (malloced);
+}
+
+#if ISC_MEM_TRACKLINES
+/*!
+ * mctx must not be locked.
+ */
+static void
+add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) {
+ debuglink_t *dl = NULL;
+ uint32_t hash;
+ uint32_t idx;
+
+ MCTXLOCK(mctx);
+
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n",
+ ptr, size, file, line, mctx);
+ }
+
+ if (mctx->debuglist == NULL) {
+ goto unlock;
+ }
+
+#ifdef __COVERITY__
+ /*
+ * Use simple conversion from pointer to hash to avoid
+ * tainting 'ptr' due to byte swap in isc_hash_function.
+ */
+ hash = (uintptr_t)ptr >> 3;
+#else
+ hash = isc_hash_function(&ptr, sizeof(ptr), true);
+#endif
+ idx = hash % DEBUG_TABLE_COUNT;
+
+ dl = mallocx(sizeof(*dl), mctx->jemalloc_flags);
+ INSIST(dl != NULL);
+ increment_malloced(mctx, sizeof(*dl));
+
+ ISC_LINK_INIT(dl, link);
+ dl->ptr = ptr;
+ dl->size = size;
+ dl->file = file;
+ dl->line = line;
+
+ ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link);
+ mctx->debuglistcnt++;
+unlock:
+ MCTXUNLOCK(mctx);
+}
+
+static void
+delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size,
+ const char *file, unsigned int line) {
+ debuglink_t *dl = NULL;
+ uint32_t hash;
+ uint32_t idx;
+
+ MCTXLOCK(mctx);
+
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n",
+ ptr, size, file, line, mctx);
+ }
+
+ if (mctx->debuglist == NULL) {
+ goto unlock;
+ }
+
+#ifdef __COVERITY__
+ /*
+ * Use simple conversion from pointer to hash to avoid
+ * tainting 'ptr' due to byte swap in isc_hash_function.
+ */
+ hash = (uintptr_t)ptr >> 3;
+#else
+ hash = isc_hash_function(&ptr, sizeof(ptr), true);
+#endif
+ idx = hash % DEBUG_TABLE_COUNT;
+
+ dl = ISC_LIST_HEAD(mctx->debuglist[idx]);
+ while (dl != NULL) {
+ if (dl->ptr == ptr) {
+ ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link);
+ decrement_malloced(mctx, sizeof(*dl));
+ sdallocx(dl, sizeof(*dl), mctx->jemalloc_flags);
+ goto unlock;
+ }
+ dl = ISC_LIST_NEXT(dl, link);
+ }
+
+ /*
+ * If we get here, we didn't find the item on the list. We're
+ * screwed.
+ */
+ UNREACHABLE();
+unlock:
+ MCTXUNLOCK(mctx);
+}
+#endif /* ISC_MEM_TRACKLINES */
+
+#define ADJUST_ZERO_ALLOCATION_SIZE(s) \
+ if (s == 0) { \
+ s = ZERO_ALLOCATION_SIZE; \
+ }
+
+#define MEM_ALIGN(a) ((a) ? MALLOCX_ALIGN(a) : 0)
+
+/*!
+ * Perform a malloc, doing memory filling and overrun detection as necessary.
+ */
+static void *
+mem_get(isc_mem_t *ctx, size_t size, int flags) {
+ char *ret = NULL;
+
+ ADJUST_ZERO_ALLOCATION_SIZE(size);
+
+ ret = mallocx(size, flags | ctx->jemalloc_flags);
+ INSIST(ret != NULL);
+
+ if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) {
+ memset(ret, 0xbe, size); /* Mnemonic for "beef". */
+ }
+
+ return (ret);
+}
+
+/*!
+ * Perform a free, doing memory filling and overrun detection as necessary.
+ */
+/* coverity[+free : arg-1] */
+static void
+mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) {
+ ADJUST_ZERO_ALLOCATION_SIZE(size);
+
+ if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) {
+ memset(mem, 0xde, size); /* Mnemonic for "dead". */
+ }
+ sdallocx(mem, size, flags | ctx->jemalloc_flags);
+}
+
+static void *
+mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
+ int flags) {
+ void *new_ptr = NULL;
+
+ ADJUST_ZERO_ALLOCATION_SIZE(new_size);
+
+ new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags);
+ INSIST(new_ptr != NULL);
+
+ if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) {
+ ssize_t diff_size = new_size - old_size;
+ void *diff_ptr = (uint8_t *)new_ptr + old_size;
+ if (diff_size > 0) {
+ /* Mnemonic for "beef". */
+ memset(diff_ptr, 0xbe, diff_size);
+ }
+ }
+
+ return (new_ptr);
+}
+
+#define stats_bucket(ctx, size) \
+ ((size / STATS_BUCKET_SIZE) >= STATS_BUCKETS \
+ ? &ctx->stats[STATS_BUCKETS] \
+ : &ctx->stats[size / STATS_BUCKET_SIZE])
+
+/*!
+ * Update internal counters after a memory get.
+ */
+static void
+mem_getstats(isc_mem_t *ctx, size_t size) {
+ struct stats *stats = stats_bucket(ctx, size);
+
+ atomic_fetch_add_relaxed(&ctx->total, size);
+ atomic_fetch_add_release(&ctx->inuse, size);
+
+ atomic_fetch_add_relaxed(&stats->gets, 1);
+ atomic_fetch_add_relaxed(&stats->totalgets, 1);
+
+ increment_malloced(ctx, size);
+}
+
+/*!
+ * Update internal counters after a memory put.
+ */
+static void
+mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) {
+ struct stats *stats = stats_bucket(ctx, size);
+ atomic_size_t s, g;
+
+ UNUSED(ptr);
+
+ s = atomic_fetch_sub_release(&ctx->inuse, size);
+ INSIST(s >= size);
+
+ g = atomic_fetch_sub_release(&stats->gets, 1);
+ INSIST(g >= 1);
+
+ decrement_malloced(ctx, size);
+}
+
+/*
+ * Private.
+ */
+
+static bool
+mem_jemalloc_arena_create(unsigned int *pnew_arenano) {
+ REQUIRE(pnew_arenano != NULL);
+
+#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4
+ unsigned int arenano = 0;
+ size_t len = sizeof(arenano);
+ int res = 0;
+
+ res = mallctl("arenas.create", &arenano, &len, NULL, 0);
+ if (res != 0) {
+ return (false);
+ }
+
+ *pnew_arenano = arenano;
+
+ return (true);
+#else
+ *pnew_arenano = ISC_MEM_ILLEGAL_ARENA;
+ return (true);
+#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */
+}
+
+static bool
+mem_jemalloc_arena_destroy(unsigned int arenano) {
+#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4
+ int res = 0;
+ char buf[256] = { 0 };
+
+ (void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano);
+ res = mallctl(buf, NULL, NULL, NULL, 0);
+ if (res != 0) {
+ return (false);
+ }
+
+ return (true);
+#else
+ UNUSED(arenano);
+ return (true);
+#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */
+}
+
+static void
+mem_initialize(void) {
+ isc_mutex_init(&contextslock);
+ ISC_LIST_INIT(contexts);
+ totallost = 0;
+}
+
+void
+isc__mem_initialize(void) {
+ RUNTIME_CHECK(isc_once_do(&init_once, mem_initialize) == ISC_R_SUCCESS);
+}
+
+static void
+mem_shutdown(void) {
+ isc__mem_checkdestroyed();
+
+ isc_mutex_destroy(&contextslock);
+}
+
+void
+isc__mem_shutdown(void) {
+ RUNTIME_CHECK(isc_once_do(&shut_once, mem_shutdown) == ISC_R_SUCCESS);
+}
+
+static void
+mem_create(isc_mem_t **ctxp, unsigned int flags, unsigned int jemalloc_flags) {
+ isc_mem_t *ctx = NULL;
+
+ REQUIRE(ctxp != NULL && *ctxp == NULL);
+
+ ctx = mallocx(sizeof(*ctx),
+ MALLOCX_ALIGN(isc_os_cacheline()) | jemalloc_flags);
+ INSIST(ctx != NULL);
+
+ *ctx = (isc_mem_t){
+ .magic = MEM_MAGIC,
+ .flags = flags,
+ .jemalloc_flags = jemalloc_flags,
+ .jemalloc_arena = ISC_MEM_ILLEGAL_ARENA,
+ .checkfree = true,
+ };
+
+ isc_mutex_init(&ctx->lock);
+ isc_refcount_init(&ctx->references, 1);
+
+ atomic_init(&ctx->total, 0);
+ atomic_init(&ctx->inuse, 0);
+ atomic_init(&ctx->maxinuse, 0);
+ atomic_init(&ctx->malloced, sizeof(*ctx));
+ atomic_init(&ctx->maxmalloced, sizeof(*ctx));
+ atomic_init(&ctx->hi_water, 0);
+ atomic_init(&ctx->lo_water, 0);
+ atomic_init(&ctx->hi_called, false);
+ atomic_init(&ctx->is_overmem, false);
+
+ for (size_t i = 0; i < STATS_BUCKETS + 1; i++) {
+ atomic_init(&ctx->stats[i].gets, 0);
+ atomic_init(&ctx->stats[i].totalgets, 0);
+ }
+ ISC_LIST_INIT(ctx->pools);
+
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGRECORD) != 0) {
+ unsigned int i;
+
+ ctx->debuglist =
+ mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)),
+ ctx->jemalloc_flags);
+ INSIST(ctx->debuglist != NULL);
+
+ for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
+ ISC_LIST_INIT(ctx->debuglist[i]);
+ }
+ increment_malloced(ctx,
+ DEBUG_TABLE_COUNT * sizeof(debuglist_t));
+ }
+#endif /* if ISC_MEM_TRACKLINES */
+
+ LOCK(&contextslock);
+ ISC_LIST_INITANDAPPEND(contexts, ctx, link);
+ UNLOCK(&contextslock);
+
+ *ctxp = ctx;
+}
+
+/*
+ * Public.
+ */
+
+static void
+destroy(isc_mem_t *ctx) {
+ unsigned int i;
+ size_t malloced;
+ unsigned int arena_no;
+
+ LOCK(&contextslock);
+ ISC_LIST_UNLINK(contexts, ctx, link);
+ totallost += isc_mem_inuse(ctx);
+ UNLOCK(&contextslock);
+
+ ctx->magic = 0;
+
+ arena_no = ctx->jemalloc_arena;
+
+ INSIST(ISC_LIST_EMPTY(ctx->pools));
+
+#if ISC_MEM_TRACKLINES
+ if (ctx->debuglist != NULL) {
+ debuglink_t *dl;
+ for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
+ for (dl = ISC_LIST_HEAD(ctx->debuglist[i]); dl != NULL;
+ dl = ISC_LIST_HEAD(ctx->debuglist[i]))
+ {
+ if (ctx->checkfree && dl->ptr != NULL) {
+ print_active(ctx, stderr);
+ }
+ INSIST(!ctx->checkfree || dl->ptr == NULL);
+
+ ISC_LIST_UNLINK(ctx->debuglist[i], dl, link);
+ sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags);
+ decrement_malloced(ctx, sizeof(*dl));
+ }
+ }
+
+ sdallocx(ctx->debuglist,
+ (DEBUG_TABLE_COUNT * sizeof(debuglist_t)),
+ ctx->jemalloc_flags);
+ decrement_malloced(ctx,
+ DEBUG_TABLE_COUNT * sizeof(debuglist_t));
+ }
+#endif /* if ISC_MEM_TRACKLINES */
+
+ if (ctx->checkfree) {
+ for (i = 0; i <= STATS_BUCKETS; i++) {
+ struct stats *stats = &ctx->stats[i];
+ size_t gets = atomic_load_acquire(&stats->gets);
+ if (gets != 0U) {
+ fprintf(stderr,
+ "Failing assertion due to probable "
+ "leaked memory in context %p (\"%s\") "
+ "(stats[%u].gets == %zu).\n",
+ ctx, ctx->name, i, gets);
+#if ISC_MEM_TRACKLINES
+ print_active(ctx, stderr);
+#endif /* if ISC_MEM_TRACKLINES */
+ INSIST(gets == 0U);
+ }
+ }
+ }
+
+ isc_mutex_destroy(&ctx->lock);
+
+ malloced = decrement_malloced(ctx, sizeof(*ctx));
+
+ if (ctx->checkfree) {
+ INSIST(malloced == 0);
+ }
+ sdallocx(ctx, sizeof(*ctx),
+ MALLOCX_ALIGN(isc_os_cacheline()) | ctx->jemalloc_flags);
+
+ if (arena_no != ISC_MEM_ILLEGAL_ARENA) {
+ RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true);
+ }
+}
+
+void
+isc_mem_attach(isc_mem_t *source, isc_mem_t **targetp) {
+ REQUIRE(VALID_CONTEXT(source));
+ REQUIRE(targetp != NULL && *targetp == NULL);
+
+ isc_refcount_increment(&source->references);
+
+ *targetp = source;
+}
+
+void
+isc__mem_detach(isc_mem_t **ctxp FLARG) {
+ isc_mem_t *ctx = NULL;
+
+ REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
+
+ ctx = *ctxp;
+ *ctxp = NULL;
+
+ if (isc_refcount_decrement(&ctx->references) == 1) {
+ isc_refcount_destroy(&ctx->references);
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "destroy mctx %p file %s line %u\n",
+ ctx, file, line);
+ }
+#endif
+ destroy(ctx);
+ }
+}
+
+/*
+ * isc_mem_putanddetach() is the equivalent of:
+ *
+ * mctx = NULL;
+ * isc_mem_attach(ptr->mctx, &mctx);
+ * isc_mem_detach(&ptr->mctx);
+ * isc_mem_put(mctx, ptr, sizeof(*ptr);
+ * isc_mem_detach(&mctx);
+ */
+
+void
+isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size,
+ size_t alignment FLARG) {
+ isc_mem_t *ctx = NULL;
+
+ REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
+ REQUIRE(ptr != NULL);
+ REQUIRE(size != 0);
+
+ ctx = *ctxp;
+ *ctxp = NULL;
+
+ DELETE_TRACE(ctx, ptr, size, file, line);
+
+ mem_putstats(ctx, ptr, size);
+ mem_put(ctx, ptr, size, MEM_ALIGN(alignment));
+
+ if (isc_refcount_decrement(&ctx->references) == 1) {
+ isc_refcount_destroy(&ctx->references);
+ destroy(ctx);
+ }
+}
+
+void
+isc__mem_destroy(isc_mem_t **ctxp FLARG) {
+ isc_mem_t *ctx = NULL;
+
+ /*
+ * This routine provides legacy support for callers who use mctxs
+ * without attaching/detaching.
+ */
+
+ REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
+
+ ctx = *ctxp;
+ *ctxp = NULL;
+
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "destroy mctx %p file %s line %u\n", ctx, file,
+ line);
+ }
+
+ if (isc_refcount_decrement(&ctx->references) > 1) {
+ print_active(ctx, stderr);
+ }
+#else /* if ISC_MEM_TRACKLINES */
+ isc_refcount_decrementz(&ctx->references);
+#endif /* if ISC_MEM_TRACKLINES */
+ isc_refcount_destroy(&ctx->references);
+ destroy(ctx);
+
+ *ctxp = NULL;
+}
+
+#define CALL_HI_WATER(ctx) \
+ { \
+ if (ctx->water != NULL && hi_water(ctx)) { \
+ (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); \
+ } \
+ }
+
+#define CALL_LO_WATER(ctx) \
+ { \
+ if ((ctx->water != NULL) && lo_water(ctx)) { \
+ (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); \
+ } \
+ }
+
+static bool
+hi_water(isc_mem_t *ctx) {
+ size_t inuse;
+ size_t maxinuse;
+ size_t hiwater = atomic_load_relaxed(&ctx->hi_water);
+
+ if (hiwater == 0) {
+ return (false);
+ }
+
+ inuse = atomic_load_acquire(&ctx->inuse);
+ if (inuse <= hiwater) {
+ return (false);
+ }
+
+ maxinuse = atomic_load_acquire(&ctx->maxinuse);
+ if (inuse > maxinuse) {
+ (void)atomic_compare_exchange_strong(&ctx->maxinuse, &maxinuse,
+ inuse);
+
+ if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) {
+ fprintf(stderr, "maxinuse = %lu\n",
+ (unsigned long)inuse);
+ }
+ }
+
+ if (atomic_load_acquire(&ctx->hi_called)) {
+ return (false);
+ }
+
+ /* We are over water (for the first time) */
+ atomic_store_release(&ctx->is_overmem, true);
+
+ return (true);
+}
+
+static bool
+lo_water(isc_mem_t *ctx) {
+ size_t inuse;
+ size_t lowater = atomic_load_relaxed(&ctx->lo_water);
+
+ if (lowater == 0) {
+ return (false);
+ }
+
+ inuse = atomic_load_acquire(&ctx->inuse);
+ if (inuse >= lowater) {
+ return (false);
+ }
+
+ if (!atomic_load_acquire(&ctx->hi_called)) {
+ return (false);
+ }
+
+ /* We are no longer overmem */
+ atomic_store_release(&ctx->is_overmem, false);
+
+ return (true);
+}
+
+void *
+isc__mem_get(isc_mem_t *ctx, size_t size, size_t alignment FLARG) {
+ void *ptr = NULL;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ ptr = mem_get(ctx, size, MEM_ALIGN(alignment));
+
+ mem_getstats(ctx, size);
+ ADD_TRACE(ctx, ptr, size, file, line);
+
+ CALL_HI_WATER(ctx);
+
+ return (ptr);
+}
+
+void
+isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size, size_t alignment FLARG) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ DELETE_TRACE(ctx, ptr, size, file, line);
+
+ mem_putstats(ctx, ptr, size);
+ mem_put(ctx, ptr, size, MEM_ALIGN(alignment));
+
+ CALL_LO_WATER(ctx);
+}
+
+void
+isc_mem_waterack(isc_mem_t *ctx, int flag) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ if (flag == ISC_MEM_LOWATER) {
+ atomic_store(&ctx->hi_called, false);
+ } else if (flag == ISC_MEM_HIWATER) {
+ atomic_store(&ctx->hi_called, true);
+ }
+}
+
+#if ISC_MEM_TRACKLINES
+static void
+print_active(isc_mem_t *mctx, FILE *out) {
+ if (mctx->debuglist != NULL) {
+ debuglink_t *dl;
+ unsigned int i;
+ bool found;
+
+ fprintf(out, "Dump of all outstanding memory "
+ "allocations:\n");
+ found = false;
+ for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
+ dl = ISC_LIST_HEAD(mctx->debuglist[i]);
+
+ if (dl != NULL) {
+ found = true;
+ }
+
+ while (dl != NULL) {
+ if (dl->ptr != NULL) {
+ fprintf(out,
+ "\tptr %p size %zu "
+ "file %s "
+ "line %u\n",
+ dl->ptr, dl->size, dl->file,
+ dl->line);
+ }
+ dl = ISC_LIST_NEXT(dl, link);
+ }
+ }
+
+ if (!found) {
+ fprintf(out, "\tNone.\n");
+ }
+ }
+}
+#endif /* if ISC_MEM_TRACKLINES */
+
+/*
+ * Print the stats[] on the stream "out" with suitable formatting.
+ */
+void
+isc_mem_stats(isc_mem_t *ctx, FILE *out) {
+ isc_mempool_t *pool = NULL;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ MCTXLOCK(ctx);
+
+ for (size_t i = 0; i <= STATS_BUCKETS; i++) {
+ size_t totalgets;
+ size_t gets;
+ struct stats *stats = &ctx->stats[i];
+
+ totalgets = atomic_load_acquire(&stats->totalgets);
+ gets = atomic_load_acquire(&stats->gets);
+
+ if (totalgets != 0U && gets != 0U) {
+ fprintf(out, "%s%5zu: %11zu gets, %11zu rem",
+ (i == STATS_BUCKETS) ? ">=" : " ", i,
+ totalgets, gets);
+ fputc('\n', out);
+ }
+ }
+
+ /*
+ * Note that since a pool can be locked now, these stats might
+ * be somewhat off if the pool is in active use at the time the
+ * stats are dumped. The link fields are protected by the
+ * isc_mem_t's lock, however, so walking this list and
+ * extracting integers from stats fields is always safe.
+ */
+ pool = ISC_LIST_HEAD(ctx->pools);
+ if (pool != NULL) {
+ fprintf(out, "[Pool statistics]\n");
+ fprintf(out, "%15s %10s %10s %10s %10s %10s %10s %1s\n", "name",
+ "size", "allocated", "freecount", "freemax",
+ "fillcount", "gets", "L");
+ }
+ while (pool != NULL) {
+ fprintf(out,
+ "%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n",
+ pool->name, pool->size, (size_t)0, pool->allocated,
+ pool->freecount, pool->freemax, pool->fillcount,
+ pool->gets, "N");
+ pool = ISC_LIST_NEXT(pool, link);
+ }
+
+#if ISC_MEM_TRACKLINES
+ print_active(ctx, out);
+#endif /* if ISC_MEM_TRACKLINES */
+
+ MCTXUNLOCK(ctx);
+}
+
+void *
+isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) {
+ void *ptr = NULL;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ ptr = mem_get(ctx, size, 0);
+
+ /* Recalculate the real allocated size */
+ size = sallocx(ptr, ctx->jemalloc_flags);
+
+ mem_getstats(ctx, size);
+ ADD_TRACE(ctx, ptr, size, file, line);
+
+ CALL_HI_WATER(ctx);
+
+ return (ptr);
+}
+
+void *
+isc__mem_reget(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
+ size_t alignment FLARG) {
+ void *new_ptr = NULL;
+
+ if (old_ptr == NULL) {
+ REQUIRE(old_size == 0);
+ new_ptr = isc__mem_get(ctx, new_size, alignment FLARG_PASS);
+ } else if (new_size == 0) {
+ isc__mem_put(ctx, old_ptr, old_size, alignment FLARG_PASS);
+ } else {
+ DELETE_TRACE(ctx, old_ptr, old_size, file, line);
+ mem_putstats(ctx, old_ptr, old_size);
+
+ new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size,
+ MEM_ALIGN(alignment));
+
+ mem_getstats(ctx, new_size);
+ ADD_TRACE(ctx, new_ptr, new_size, file, line);
+
+ /*
+ * We want to postpone the call to water in edge case
+ * where the realloc will exactly hit on the boundary of
+ * the water and we would call water twice.
+ */
+ CALL_LO_WATER(ctx);
+ CALL_HI_WATER(ctx);
+ }
+
+ return (new_ptr);
+}
+
+void *
+isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) {
+ void *new_ptr = NULL;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ if (old_ptr == NULL) {
+ new_ptr = isc__mem_allocate(ctx, new_size FLARG_PASS);
+ } else if (new_size == 0) {
+ isc__mem_free(ctx, old_ptr FLARG_PASS);
+ } else {
+ size_t old_size = sallocx(old_ptr, ctx->jemalloc_flags);
+
+ DELETE_TRACE(ctx, old_ptr, old_size, file, line);
+ mem_putstats(ctx, old_ptr, old_size);
+
+ new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, 0);
+
+ /* Recalculate the real allocated size */
+ new_size = sallocx(new_ptr, ctx->jemalloc_flags);
+
+ mem_getstats(ctx, new_size);
+ ADD_TRACE(ctx, new_ptr, new_size, file, line);
+
+ /*
+ * We want to postpone the call to water in edge case
+ * where the realloc will exactly hit on the boundary of
+ * the water and we would call water twice.
+ */
+ CALL_LO_WATER(ctx);
+ CALL_HI_WATER(ctx);
+ }
+
+ return (new_ptr);
+}
+
+void
+isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) {
+ size_t size = 0;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ size = sallocx(ptr, ctx->jemalloc_flags);
+
+ DELETE_TRACE(ctx, ptr, size, file, line);
+
+ mem_putstats(ctx, ptr, size);
+ mem_put(ctx, ptr, size, 0);
+
+ CALL_LO_WATER(ctx);
+}
+
+/*
+ * Other useful things.
+ */
+
+char *
+isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) {
+ size_t len;
+ char *ns = NULL;
+
+ REQUIRE(VALID_CONTEXT(mctx));
+ REQUIRE(s != NULL);
+
+ len = strlen(s) + 1;
+
+ ns = isc__mem_allocate(mctx, len FLARG_PASS);
+
+ strlcpy(ns, s, len);
+
+ return (ns);
+}
+
+char *
+isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) {
+ size_t len;
+ char *ns = NULL;
+
+ REQUIRE(VALID_CONTEXT(mctx));
+ REQUIRE(s != NULL);
+ REQUIRE(size != 0);
+
+ len = strlen(s) + 1;
+ if (len > size) {
+ len = size;
+ }
+
+ ns = isc__mem_allocate(mctx, len FLARG_PASS);
+
+ strlcpy(ns, s, len);
+
+ return (ns);
+}
+
+void
+isc_mem_setdestroycheck(isc_mem_t *ctx, bool flag) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ MCTXLOCK(ctx);
+
+ ctx->checkfree = flag;
+
+ MCTXUNLOCK(ctx);
+}
+
+size_t
+isc_mem_inuse(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_acquire(&ctx->inuse));
+}
+
+size_t
+isc_mem_maxinuse(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_acquire(&ctx->maxinuse));
+}
+
+size_t
+isc_mem_total(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_acquire(&ctx->total));
+}
+
+size_t
+isc_mem_malloced(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_acquire(&ctx->malloced));
+}
+
+size_t
+isc_mem_maxmalloced(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_acquire(&ctx->maxmalloced));
+}
+
+void
+isc_mem_clearwater(isc_mem_t *mctx) {
+ isc_mem_setwater(mctx, NULL, NULL, 0, 0);
+}
+
+void
+isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg,
+ size_t hiwater, size_t lowater) {
+ isc_mem_water_t oldwater;
+ void *oldwater_arg;
+
+ REQUIRE(VALID_CONTEXT(ctx));
+ REQUIRE(hiwater >= lowater);
+
+ oldwater = ctx->water;
+ oldwater_arg = ctx->water_arg;
+
+ /* No water was set and new water is also NULL */
+ if (oldwater == NULL && water == NULL) {
+ return;
+ }
+
+ /* The water function is being set for the first time */
+ if (oldwater == NULL) {
+ REQUIRE(water != NULL && lowater > 0);
+
+ INSIST(atomic_load(&ctx->hi_water) == 0);
+ INSIST(atomic_load(&ctx->lo_water) == 0);
+
+ ctx->water = water;
+ ctx->water_arg = water_arg;
+ atomic_store(&ctx->hi_water, hiwater);
+ atomic_store(&ctx->lo_water, lowater);
+
+ return;
+ }
+
+ REQUIRE((water == oldwater && water_arg == oldwater_arg) ||
+ (water == NULL && water_arg == NULL && hiwater == 0));
+
+ atomic_store(&ctx->hi_water, hiwater);
+ atomic_store(&ctx->lo_water, lowater);
+
+ if (atomic_load_acquire(&ctx->hi_called) &&
+ (atomic_load_acquire(&ctx->inuse) < lowater || lowater == 0U))
+ {
+ (oldwater)(oldwater_arg, ISC_MEM_LOWATER);
+ }
+}
+
+bool
+isc_mem_isovermem(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ return (atomic_load_relaxed(&ctx->is_overmem));
+}
+
+void
+isc_mem_setname(isc_mem_t *ctx, const char *name) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ LOCK(&ctx->lock);
+ strlcpy(ctx->name, name, sizeof(ctx->name));
+ UNLOCK(&ctx->lock);
+}
+
+const char *
+isc_mem_getname(isc_mem_t *ctx) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ if (ctx->name[0] == 0) {
+ return ("");
+ }
+
+ return (ctx->name);
+}
+
+/*
+ * Memory pool stuff
+ */
+
+void
+isc__mempool_create(isc_mem_t *restrict mctx, const size_t element_size,
+ isc_mempool_t **restrict mpctxp FLARG) {
+ isc_mempool_t *restrict mpctx = NULL;
+ size_t size = element_size;
+
+ REQUIRE(VALID_CONTEXT(mctx));
+ REQUIRE(size > 0U);
+ REQUIRE(mpctxp != NULL && *mpctxp == NULL);
+
+ /*
+ * Mempools are stored as a linked list of element.
+ */
+ if (size < sizeof(element)) {
+ size = sizeof(element);
+ }
+
+ /*
+ * Allocate space for this pool, initialize values, and if all
+ * works well, attach to the memory context.
+ */
+ mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t));
+
+ *mpctx = (isc_mempool_t){
+ .size = size,
+ .freemax = 1,
+ .fillcount = 1,
+ };
+
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "create pool %p file %s line %u mctx %p\n",
+ mpctx, file, line, mctx);
+ }
+#endif /* ISC_MEM_TRACKLINES */
+
+ isc_mem_attach(mctx, &mpctx->mctx);
+ mpctx->magic = MEMPOOL_MAGIC;
+
+ *mpctxp = (isc_mempool_t *)mpctx;
+
+ MCTXLOCK(mctx);
+ ISC_LIST_INITANDAPPEND(mctx->pools, mpctx, link);
+ mctx->poolcnt++;
+ MCTXUNLOCK(mctx);
+}
+
+void
+isc_mempool_setname(isc_mempool_t *restrict mpctx, const char *name) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+ REQUIRE(name != NULL);
+
+ strlcpy(mpctx->name, name, sizeof(mpctx->name));
+}
+
+void
+isc__mempool_destroy(isc_mempool_t **restrict mpctxp FLARG) {
+ isc_mempool_t *restrict mpctx = NULL;
+ isc_mem_t *mctx = NULL;
+ element *restrict item = NULL;
+
+ REQUIRE(mpctxp != NULL);
+ REQUIRE(VALID_MEMPOOL(*mpctxp));
+
+ mpctx = *mpctxp;
+ *mpctxp = NULL;
+
+ mctx = mpctx->mctx;
+
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "destroy pool %p file %s line %u mctx %p\n",
+ mpctx, file, line, mctx);
+ }
+#endif
+
+ if (mpctx->allocated > 0) {
+ UNEXPECTED_ERROR("mempool %s leaked memory", mpctx->name);
+ }
+ REQUIRE(mpctx->allocated == 0);
+
+ /*
+ * Return any items on the free list
+ */
+ while (mpctx->items != NULL) {
+ INSIST(mpctx->freecount > 0);
+ mpctx->freecount--;
+
+ item = mpctx->items;
+ mpctx->items = item->next;
+
+ mem_putstats(mctx, item, mpctx->size);
+ mem_put(mctx, item, mpctx->size, 0);
+ }
+
+ /*
+ * Remove our linked list entry from the memory context.
+ */
+ MCTXLOCK(mctx);
+ ISC_LIST_UNLINK(mctx->pools, mpctx, link);
+ mctx->poolcnt--;
+ MCTXUNLOCK(mctx);
+
+ mpctx->magic = 0;
+
+ isc_mem_putanddetach(&mpctx->mctx, mpctx, sizeof(isc_mempool_t));
+}
+
+void *
+isc__mempool_get(isc_mempool_t *restrict mpctx FLARG) {
+ element *restrict item = NULL;
+
+ REQUIRE(VALID_MEMPOOL(mpctx));
+
+ mpctx->allocated++;
+
+ if (mpctx->items == NULL) {
+ isc_mem_t *mctx = mpctx->mctx;
+#if !__SANITIZE_ADDRESS__
+ const size_t fillcount = mpctx->fillcount;
+#else
+ const size_t fillcount = 1;
+#endif
+ /*
+ * We need to dip into the well. Fill up our free list.
+ */
+ for (size_t i = 0; i < fillcount; i++) {
+ item = mem_get(mctx, mpctx->size, 0);
+ mem_getstats(mctx, mpctx->size);
+ item->next = mpctx->items;
+ mpctx->items = item;
+ mpctx->freecount++;
+ }
+ }
+
+ item = mpctx->items;
+ INSIST(item != NULL);
+
+ mpctx->items = item->next;
+
+ INSIST(mpctx->freecount > 0);
+ mpctx->freecount--;
+ mpctx->gets++;
+
+ ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line);
+
+ return (item);
+}
+
+/* coverity[+free : arg-1] */
+void
+isc__mempool_put(isc_mempool_t *restrict mpctx, void *mem FLARG) {
+ element *restrict item = NULL;
+
+ REQUIRE(VALID_MEMPOOL(mpctx));
+ REQUIRE(mem != NULL);
+
+ isc_mem_t *mctx = mpctx->mctx;
+ const size_t freecount = mpctx->freecount;
+#if !__SANITIZE_ADDRESS__
+ const size_t freemax = mpctx->freemax;
+#else
+ const size_t freemax = 0;
+#endif
+
+ INSIST(mpctx->allocated > 0);
+ mpctx->allocated--;
+
+ DELETE_TRACE(mctx, mem, mpctx->size, file, line);
+
+ /*
+ * If our free list is full, return this to the mctx directly.
+ */
+ if (freecount >= freemax) {
+ mem_putstats(mctx, mem, mpctx->size);
+ mem_put(mctx, mem, mpctx->size, 0);
+ return;
+ }
+
+ /*
+ * Otherwise, attach it to our free list and bump the counter.
+ */
+ item = (element *)mem;
+ item->next = mpctx->items;
+ mpctx->items = item;
+ mpctx->freecount++;
+}
+
+/*
+ * Quotas
+ */
+
+void
+isc_mempool_setfreemax(isc_mempool_t *restrict mpctx,
+ const unsigned int limit) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+ mpctx->freemax = limit;
+}
+
+unsigned int
+isc_mempool_getfreemax(isc_mempool_t *restrict mpctx) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+
+ return (mpctx->freemax);
+}
+
+unsigned int
+isc_mempool_getfreecount(isc_mempool_t *restrict mpctx) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+
+ return (mpctx->freecount);
+}
+
+unsigned int
+isc_mempool_getallocated(isc_mempool_t *restrict mpctx) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+
+ return (mpctx->allocated);
+}
+
+void
+isc_mempool_setfillcount(isc_mempool_t *restrict mpctx,
+ unsigned int const limit) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+ REQUIRE(limit > 0);
+
+ mpctx->fillcount = limit;
+}
+
+unsigned int
+isc_mempool_getfillcount(isc_mempool_t *restrict mpctx) {
+ REQUIRE(VALID_MEMPOOL(mpctx));
+
+ return (mpctx->fillcount);
+}
+
+/*
+ * Requires contextslock to be held by caller.
+ */
+#if ISC_MEM_TRACKLINES
+static void
+print_contexts(FILE *file) {
+ isc_mem_t *ctx;
+
+ for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
+ ctx = ISC_LIST_NEXT(ctx, link))
+ {
+ fprintf(file, "context: %p (%s): %" PRIuFAST32 " references\n",
+ ctx, ctx->name[0] == 0 ? "<unknown>" : ctx->name,
+ isc_refcount_current(&ctx->references));
+ print_active(ctx, file);
+ }
+ fflush(file);
+}
+#endif
+
+static atomic_uintptr_t checkdestroyed = 0;
+
+void
+isc_mem_checkdestroyed(FILE *file) {
+ atomic_store_release(&checkdestroyed, (uintptr_t)file);
+}
+
+void
+isc__mem_checkdestroyed(void) {
+ FILE *file = (FILE *)atomic_load_acquire(&checkdestroyed);
+
+ if (file == NULL) {
+ return;
+ }
+
+ LOCK(&contextslock);
+ if (!ISC_LIST_EMPTY(contexts)) {
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & TRACE_OR_RECORD) != 0) {
+ print_contexts(file);
+ }
+#endif /* if ISC_MEM_TRACKLINES */
+ UNREACHABLE();
+ }
+ UNLOCK(&contextslock);
+}
+
+unsigned int
+isc_mem_references(isc_mem_t *ctx) {
+ return (isc_refcount_current(&ctx->references));
+}
+
+typedef struct summarystat {
+ uint64_t total;
+ uint64_t inuse;
+ uint64_t malloced;
+ uint64_t contextsize;
+} summarystat_t;
+
+#ifdef HAVE_LIBXML2
+#define TRY0(a) \
+ do { \
+ xmlrc = (a); \
+ if (xmlrc < 0) \
+ goto error; \
+ } while (0)
+static int
+xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) {
+ REQUIRE(VALID_CONTEXT(ctx));
+
+ int xmlrc;
+
+ MCTXLOCK(ctx);
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "context"));
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "id"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%p", ctx));
+ TRY0(xmlTextWriterEndElement(writer)); /* id */
+
+ if (ctx->name[0] != 0) {
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "name"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%s", ctx->name));
+ TRY0(xmlTextWriterEndElement(writer)); /* name */
+ }
+
+ summary->contextsize += sizeof(*ctx);
+#if ISC_MEM_TRACKLINES
+ if (ctx->debuglist != NULL) {
+ summary->contextsize += DEBUG_TABLE_COUNT *
+ sizeof(debuglist_t) +
+ ctx->debuglistcnt * sizeof(debuglink_t);
+ }
+#endif /* if ISC_MEM_TRACKLINES */
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "references"));
+ TRY0(xmlTextWriterWriteFormatString(
+ writer, "%" PRIuFAST32,
+ isc_refcount_current(&ctx->references)));
+ TRY0(xmlTextWriterEndElement(writer)); /* references */
+
+ summary->total += isc_mem_total(ctx);
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "total"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ (uint64_t)isc_mem_total(ctx)));
+ TRY0(xmlTextWriterEndElement(writer)); /* total */
+
+ summary->inuse += isc_mem_inuse(ctx);
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ (uint64_t)isc_mem_inuse(ctx)));
+ TRY0(xmlTextWriterEndElement(writer)); /* inuse */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxinuse"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ (uint64_t)isc_mem_maxinuse(ctx)));
+ TRY0(xmlTextWriterEndElement(writer)); /* maxinuse */
+
+ summary->malloced += isc_mem_malloced(ctx);
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ (uint64_t)isc_mem_malloced(ctx)));
+ TRY0(xmlTextWriterEndElement(writer)); /* malloced */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxmalloced"));
+ TRY0(xmlTextWriterWriteFormatString(
+ writer, "%" PRIu64 "", (uint64_t)isc_mem_maxmalloced(ctx)));
+ TRY0(xmlTextWriterEndElement(writer)); /* maxmalloced */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%u", ctx->poolcnt));
+ TRY0(xmlTextWriterEndElement(writer)); /* pools */
+ summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t);
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater"));
+ TRY0(xmlTextWriterWriteFormatString(
+ writer, "%" PRIu64 "",
+ (uint64_t)atomic_load_relaxed(&ctx->hi_water)));
+ TRY0(xmlTextWriterEndElement(writer)); /* hiwater */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater"));
+ TRY0(xmlTextWriterWriteFormatString(
+ writer, "%" PRIu64 "",
+ (uint64_t)atomic_load_relaxed(&ctx->lo_water)));
+ TRY0(xmlTextWriterEndElement(writer)); /* lowater */
+
+ TRY0(xmlTextWriterEndElement(writer)); /* context */
+
+error:
+ MCTXUNLOCK(ctx);
+
+ return (xmlrc);
+}
+
+int
+isc_mem_renderxml(void *writer0) {
+ isc_mem_t *ctx;
+ summarystat_t summary = { 0 };
+ uint64_t lost;
+ int xmlrc;
+ xmlTextWriterPtr writer = (xmlTextWriterPtr)writer0;
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "contexts"));
+
+ LOCK(&contextslock);
+ lost = totallost;
+ for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
+ ctx = ISC_LIST_NEXT(ctx, link))
+ {
+ xmlrc = xml_renderctx(ctx, &summary, writer);
+ if (xmlrc < 0) {
+ UNLOCK(&contextslock);
+ goto error;
+ }
+ }
+ UNLOCK(&contextslock);
+
+ TRY0(xmlTextWriterEndElement(writer)); /* contexts */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "summary"));
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "TotalUse"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ summary.total));
+ TRY0(xmlTextWriterEndElement(writer)); /* TotalUse */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "InUse"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ summary.inuse));
+ TRY0(xmlTextWriterEndElement(writer)); /* InUse */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Malloced"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ summary.malloced));
+ TRY0(xmlTextWriterEndElement(writer)); /* InUse */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "ContextSize"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
+ summary.contextsize));
+ TRY0(xmlTextWriterEndElement(writer)); /* ContextSize */
+
+ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Lost"));
+ TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", lost));
+ TRY0(xmlTextWriterEndElement(writer)); /* Lost */
+
+ TRY0(xmlTextWriterEndElement(writer)); /* summary */
+error:
+ return (xmlrc);
+}
+
+#endif /* HAVE_LIBXML2 */
+
+#ifdef HAVE_JSON_C
+#define CHECKMEM(m) RUNTIME_CHECK(m != NULL)
+
+static isc_result_t
+json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) {
+ REQUIRE(VALID_CONTEXT(ctx));
+ REQUIRE(summary != NULL);
+ REQUIRE(array != NULL);
+
+ json_object *ctxobj, *obj;
+ char buf[1024];
+
+ MCTXLOCK(ctx);
+
+ summary->contextsize += sizeof(*ctx);
+ summary->total += isc_mem_total(ctx);
+ summary->inuse += isc_mem_inuse(ctx);
+ summary->malloced += isc_mem_malloced(ctx);
+#if ISC_MEM_TRACKLINES
+ if (ctx->debuglist != NULL) {
+ summary->contextsize += DEBUG_TABLE_COUNT *
+ sizeof(debuglist_t) +
+ ctx->debuglistcnt * sizeof(debuglink_t);
+ }
+#endif /* if ISC_MEM_TRACKLINES */
+
+ ctxobj = json_object_new_object();
+ CHECKMEM(ctxobj);
+
+ snprintf(buf, sizeof(buf), "%p", ctx);
+ obj = json_object_new_string(buf);
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "id", obj);
+
+ if (ctx->name[0] != 0) {
+ obj = json_object_new_string(ctx->name);
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "name", obj);
+ }
+
+ obj = json_object_new_int64(isc_refcount_current(&ctx->references));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "references", obj);
+
+ obj = json_object_new_int64(isc_mem_total(ctx));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "total", obj);
+
+ obj = json_object_new_int64(isc_mem_inuse(ctx));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "inuse", obj);
+
+ obj = json_object_new_int64(isc_mem_maxinuse(ctx));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "maxinuse", obj);
+
+ obj = json_object_new_int64(isc_mem_malloced(ctx));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "malloced", obj);
+
+ obj = json_object_new_int64(isc_mem_maxmalloced(ctx));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "maxmalloced", obj);
+
+ obj = json_object_new_int64(ctx->poolcnt);
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "pools", obj);
+
+ summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t);
+
+ obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "hiwater", obj);
+
+ obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water));
+ CHECKMEM(obj);
+ json_object_object_add(ctxobj, "lowater", obj);
+
+ MCTXUNLOCK(ctx);
+ json_object_array_add(array, ctxobj);
+ return (ISC_R_SUCCESS);
+}
+
+isc_result_t
+isc_mem_renderjson(void *memobj0) {
+ isc_result_t result = ISC_R_SUCCESS;
+ isc_mem_t *ctx;
+ summarystat_t summary = { 0 };
+ uint64_t lost;
+ json_object *ctxarray, *obj;
+ json_object *memobj = (json_object *)memobj0;
+
+ ctxarray = json_object_new_array();
+ CHECKMEM(ctxarray);
+
+ LOCK(&contextslock);
+ lost = totallost;
+ for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
+ ctx = ISC_LIST_NEXT(ctx, link))
+ {
+ result = json_renderctx(ctx, &summary, ctxarray);
+ if (result != ISC_R_SUCCESS) {
+ UNLOCK(&contextslock);
+ goto error;
+ }
+ }
+ UNLOCK(&contextslock);
+
+ obj = json_object_new_int64(summary.total);
+ CHECKMEM(obj);
+ json_object_object_add(memobj, "TotalUse", obj);
+
+ obj = json_object_new_int64(summary.inuse);
+ CHECKMEM(obj);
+ json_object_object_add(memobj, "InUse", obj);
+
+ obj = json_object_new_int64(summary.malloced);
+ CHECKMEM(obj);
+ json_object_object_add(memobj, "Malloced", obj);
+
+ obj = json_object_new_int64(summary.contextsize);
+ CHECKMEM(obj);
+ json_object_object_add(memobj, "ContextSize", obj);
+
+ obj = json_object_new_int64(lost);
+ CHECKMEM(obj);
+ json_object_object_add(memobj, "Lost", obj);
+
+ json_object_object_add(memobj, "contexts", ctxarray);
+ return (ISC_R_SUCCESS);
+
+error:
+ if (ctxarray != NULL) {
+ json_object_put(ctxarray);
+ }
+ return (result);
+}
+#endif /* HAVE_JSON_C */
+
+void
+isc__mem_create(isc_mem_t **mctxp FLARG) {
+ mem_create(mctxp, isc_mem_defaultflags, 0);
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp,
+ file, line);
+ }
+#endif /* ISC_MEM_TRACKLINES */
+}
+
+void
+isc__mem_create_arena(isc_mem_t **mctxp FLARG) {
+ unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA;
+
+ RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no));
+
+ /*
+ * We use MALLOCX_TCACHE_NONE to bypass the tcache and route
+ * allocations directly to the arena. That is a recommendation
+ * from jemalloc developers:
+ *
+ * https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849
+ */
+ mem_create(mctxp, isc_mem_defaultflags,
+ arena_no == ISC_MEM_ILLEGAL_ARENA
+ ? 0
+ : MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE);
+ (*mctxp)->jemalloc_arena = arena_no;
+#if ISC_MEM_TRACKLINES
+ if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
+ fprintf(stderr,
+ "create mctx %p file %s line %u for jemalloc arena "
+ "%u\n",
+ *mctxp, file, line, arena_no);
+ }
+#endif /* ISC_MEM_TRACKLINES */
+}
+
+#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4
+static bool
+jemalloc_set_ssize_value(const char *valname, ssize_t newval) {
+ int ret;
+
+ ret = mallctl(valname, NULL, NULL, &newval, sizeof(newval));
+ return (ret == 0);
+}
+#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */
+
+static isc_result_t
+mem_set_arena_ssize_value(isc_mem_t *mctx, const char *arena_valname,
+ const ssize_t newval) {
+ REQUIRE(VALID_CONTEXT(mctx));
+#if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4
+ bool ret;
+ char buf[256] = { 0 };
+
+ if (mctx->jemalloc_arena == ISC_MEM_ILLEGAL_ARENA) {
+ return (ISC_R_UNEXPECTED);
+ }
+
+ (void)snprintf(buf, sizeof(buf), "arena.%u.%s", mctx->jemalloc_arena,
+ arena_valname);
+
+ ret = jemalloc_set_ssize_value(buf, newval);
+
+ if (!ret) {
+ return (ISC_R_FAILURE);
+ }
+
+ return (ISC_R_SUCCESS);
+#else
+ UNUSED(arena_valname);
+ UNUSED(newval);
+ return (ISC_R_NOTIMPLEMENTED);
+#endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */
+}
+
+isc_result_t
+isc_mem_arena_set_muzzy_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
+ return (mem_set_arena_ssize_value(mctx, "muzzy_decay_ms", decay_ms));
+}
+
+isc_result_t
+isc_mem_arena_set_dirty_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
+ return (mem_set_arena_ssize_value(mctx, "dirty_decay_ms", decay_ms));
+}
+
+void
+isc__mem_printactive(isc_mem_t *ctx, FILE *file) {
+#if ISC_MEM_TRACKLINES
+ REQUIRE(VALID_CONTEXT(ctx));
+ REQUIRE(file != NULL);
+
+ print_active(ctx, file);
+#else /* if ISC_MEM_TRACKLINES */
+ UNUSED(ctx);
+ UNUSED(file);
+#endif /* if ISC_MEM_TRACKLINES */
+}