diff options
Diffstat (limited to 'src/pool.c')
| -rw-r--r-- | src/pool.c | 1108 |
1 files changed, 1108 insertions, 0 deletions
diff --git a/src/pool.c b/src/pool.c new file mode 100644 index 0000000..54ae25b --- /dev/null +++ b/src/pool.c @@ -0,0 +1,1108 @@ +/* + * Memory management functions. + * + * Copyright 2000-2007 Willy Tarreau <w@1wt.eu> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + */ + +#include <sys/mman.h> +#include <errno.h> + +#include <haproxy/activity.h> +#include <haproxy/api.h> +#include <haproxy/applet-t.h> +#include <haproxy/cfgparse.h> +#include <haproxy/channel.h> +#include <haproxy/cli.h> +#include <haproxy/errors.h> +#include <haproxy/global.h> +#include <haproxy/list.h> +#include <haproxy/pool.h> +#include <haproxy/sc_strm.h> +#include <haproxy/stats-t.h> +#include <haproxy/stconn.h> +#include <haproxy/thread.h> +#include <haproxy/tools.h> + + +/* These ones are initialized per-thread on startup by init_pools() */ +THREAD_LOCAL size_t pool_cache_bytes = 0; /* total cache size */ +THREAD_LOCAL size_t pool_cache_count = 0; /* #cache objects */ + +static struct list pools __read_mostly = LIST_HEAD_INIT(pools); +int mem_poison_byte __read_mostly = 'P'; +uint pool_debugging __read_mostly = /* set of POOL_DBG_* flags */ +#ifdef DEBUG_FAIL_ALLOC + POOL_DBG_FAIL_ALLOC | +#endif +#ifdef DEBUG_DONT_SHARE_POOLS + POOL_DBG_DONT_MERGE | +#endif +#ifdef DEBUG_POOL_INTEGRITY + POOL_DBG_COLD_FIRST | +#endif +#ifdef DEBUG_POOL_INTEGRITY + POOL_DBG_INTEGRITY | +#endif +#ifdef CONFIG_HAP_NO_GLOBAL_POOLS + POOL_DBG_NO_GLOBAL | +#endif +#ifndef CONFIG_HAP_POOLS + POOL_DBG_NO_CACHE | +#endif +#if defined(DEBUG_POOL_TRACING) + POOL_DBG_CALLER | +#endif +#if defined(DEBUG_MEMORY_POOLS) + POOL_DBG_TAG | +#endif + 0; + +static const struct { + uint flg; + const char *set; + const char *clr; + const char *hlp; +} dbg_options[] = { + /* flg, set, clr, hlp */ + { POOL_DBG_FAIL_ALLOC, "fail", "no-fail", "randomly fail allocations" }, + { POOL_DBG_DONT_MERGE, "no-merge", "merge", "disable merging of similar pools" }, + { POOL_DBG_COLD_FIRST, "cold-first", "hot-first", "pick cold objects first" }, + { POOL_DBG_INTEGRITY, "integrity", "no-integrity", "enable cache integrity checks" }, + { POOL_DBG_NO_GLOBAL, "no-global", "global", "disable global shared cache" }, + { POOL_DBG_NO_CACHE, "no-cache", "cache", "disable thread-local cache" }, + { POOL_DBG_CALLER, "caller", "no-caller", "save caller information in cache" }, + { POOL_DBG_TAG, "tag", "no-tag", "add tag at end of allocated objects" }, + { POOL_DBG_POISON, "poison", "no-poison", "poison newly allocated objects" }, + { 0 /* end */ } +}; + +static int mem_fail_rate __read_mostly = 0; +static int using_default_allocator __read_mostly = 1; +static int disable_trim __read_mostly = 0; +static int(*my_mallctl)(const char *, void *, size_t *, void *, size_t) = NULL; + +/* ask the allocator to trim memory pools. + * This must run under thread isolation so that competing threads trying to + * allocate or release memory do not prevent the allocator from completing + * its job. We just have to be careful as callers might already be isolated + * themselves. + */ +static void trim_all_pools(void) +{ + int isolated = thread_isolated(); + + if (disable_trim) + return; + + if (!isolated) + thread_isolate(); + + if (my_mallctl) { + unsigned int i, narenas = 0; + size_t len = sizeof(narenas); + + if (my_mallctl("arenas.narenas", &narenas, &len, NULL, 0) == 0) { + for (i = 0; i < narenas; i ++) { + char mib[32] = {0}; + snprintf(mib, sizeof(mib), "arena.%u.purge", i); + (void)my_mallctl(mib, NULL, NULL, NULL, 0); + } + } + } else { +#if defined(HA_HAVE_MALLOC_TRIM) + if (using_default_allocator) + malloc_trim(0); +#elif defined(HA_HAVE_MALLOC_ZONE) + if (using_default_allocator) { + vm_address_t *zones; + unsigned int i, nzones; + + if (malloc_get_all_zones(0, NULL, &zones, &nzones) == KERN_SUCCESS) { + for (i = 0; i < nzones; i ++) { + malloc_zone_t *zone = (malloc_zone_t *)zones[i]; + + /* we cannot purge anonymous zones */ + if (zone->zone_name) + malloc_zone_pressure_relief(zone, 0); + } + } + } +#endif + } + + if (!isolated) + thread_release(); +} + +/* check if we're using the same allocator as the one that provides + * malloc_trim() and mallinfo(). The principle is that on glibc, both + * malloc_trim() and mallinfo() are provided, and using mallinfo() we + * can check if malloc() is performed through glibc or any other one + * the executable was linked against (e.g. jemalloc). Prior to this we + * have to check whether we're running on jemalloc by verifying if the + * mallctl() function is provided. Its pointer will be used later. + */ +static void detect_allocator(void) +{ +#if defined(__ELF__) + extern int mallctl(const char *, void *, size_t *, void *, size_t) __attribute__((weak)); + + my_mallctl = mallctl; +#endif + + if (!my_mallctl) { + my_mallctl = get_sym_curr_addr("mallctl"); + using_default_allocator = (my_mallctl == NULL); + } + + if (!my_mallctl) { +#if defined(HA_HAVE_MALLOC_TRIM) +#ifdef HA_HAVE_MALLINFO2 + struct mallinfo2 mi1, mi2; +#else + struct mallinfo mi1, mi2; +#endif + void *ptr; + +#ifdef HA_HAVE_MALLINFO2 + mi1 = mallinfo2(); +#else + mi1 = mallinfo(); +#endif + ptr = DISGUISE(malloc(1)); +#ifdef HA_HAVE_MALLINFO2 + mi2 = mallinfo2(); +#else + mi2 = mallinfo(); +#endif + free(DISGUISE(ptr)); + + using_default_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1)); +#elif defined(HA_HAVE_MALLOC_ZONE) + using_default_allocator = (malloc_default_zone() != NULL); +#endif + } +} + +static int is_trim_enabled(void) +{ + return using_default_allocator; +} + +static int mem_should_fail(const struct pool_head *pool) +{ + int ret = 0; + + if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) { + if (mem_fail_rate > statistical_prng_range(100)) + ret = 1; + else + ret = 0; + } + return ret; +} + +/* Try to find an existing shared pool with the same characteristics and + * returns it, otherwise creates this one. NULL is returned if no memory + * is available for a new creation. Two flags are supported : + * - MEM_F_SHARED to indicate that the pool may be shared with other users + * - MEM_F_EXACT to indicate that the size must not be rounded up + */ +struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags) +{ + unsigned int extra_mark, extra_caller, extra; + struct pool_head *pool; + struct pool_head *entry; + struct list *start; + unsigned int align; + int thr __maybe_unused; + + /* We need to store a (void *) at the end of the chunks. Since we know + * that the malloc() function will never return such a small size, + * let's round the size up to something slightly bigger, in order to + * ease merging of entries. Note that the rounding is a power of two. + * This extra (void *) is not accounted for in the size computation + * so that the visible parts outside are not affected. + * + * Note: for the LRU cache, we need to store 2 doubly-linked lists. + */ + + extra_mark = (pool_debugging & POOL_DBG_TAG) ? POOL_EXTRA_MARK : 0; + extra_caller = (pool_debugging & POOL_DBG_CALLER) ? POOL_EXTRA_CALLER : 0; + extra = extra_mark + extra_caller; + + if (!(flags & MEM_F_EXACT)) { + align = 4 * sizeof(void *); // 2 lists = 4 pointers min + size = ((size + extra + align - 1) & -align) - extra; + } + + if (!(pool_debugging & POOL_DBG_NO_CACHE)) { + /* we'll store two lists there, we need the room for this. This is + * guaranteed by the test above, except if MEM_F_EXACT is set, or if + * the only EXTRA part is in fact the one that's stored in the cache + * in addition to the pci struct. + */ + if (size + extra - extra_caller < sizeof(struct pool_cache_item)) + size = sizeof(struct pool_cache_item) + extra_caller - extra; + } + + /* TODO: thread: we do not lock pool list for now because all pools are + * created during HAProxy startup (so before threads creation) */ + start = &pools; + pool = NULL; + + list_for_each_entry(entry, &pools, list) { + if (entry->size == size) { + /* either we can share this place and we take it, or + * we look for a shareable one or for the next position + * before which we will insert a new one. + */ + if ((flags & entry->flags & MEM_F_SHARED) && + (!(pool_debugging & POOL_DBG_DONT_MERGE) || + strcmp(name, entry->name) == 0)) { + /* we can share this one */ + pool = entry; + DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name); + break; + } + } + else if (entry->size > size) { + /* insert before this one */ + start = &entry->list; + break; + } + } + + if (!pool) { + void *pool_addr; + + pool_addr = calloc(1, sizeof(*pool) + __alignof__(*pool)); + if (!pool_addr) + return NULL; + + /* always provide an aligned pool */ + pool = (struct pool_head*)((((size_t)pool_addr) + __alignof__(*pool)) & -(size_t)__alignof__(*pool)); + pool->base_addr = pool_addr; // keep it, it's the address to free later + + if (name) + strlcpy2(pool->name, name, sizeof(pool->name)); + pool->alloc_sz = size + extra; + pool->size = size; + pool->flags = flags; + LIST_APPEND(start, &pool->list); + + if (!(pool_debugging & POOL_DBG_NO_CACHE)) { + /* update per-thread pool cache if necessary */ + for (thr = 0; thr < MAX_THREADS; thr++) { + LIST_INIT(&pool->cache[thr].list); + pool->cache[thr].tid = thr; + pool->cache[thr].pool = pool; + } + } + } + pool->users++; + return pool; +} + +/* Tries to allocate an object for the pool <pool> using the system's allocator + * and directly returns it. The pool's allocated counter is checked and updated, + * but no other checks are performed. + */ +void *pool_get_from_os(struct pool_head *pool) +{ + if (!pool->limit || pool->allocated < pool->limit) { + void *ptr = pool_alloc_area(pool->alloc_sz); + if (ptr) { + _HA_ATOMIC_INC(&pool->allocated); + return ptr; + } + _HA_ATOMIC_INC(&pool->failed); + } + activity[tid].pool_fail++; + return NULL; + +} + +/* Releases a pool item back to the operating system and atomically updates + * the allocation counter. + */ +void pool_put_to_os(struct pool_head *pool, void *ptr) +{ +#ifdef DEBUG_UAF + /* This object will be released for real in order to detect a use after + * free. We also force a write to the area to ensure we crash on double + * free or free of a const area. + */ + *(uint32_t *)ptr = 0xDEADADD4; +#endif /* DEBUG_UAF */ + + pool_free_area(ptr, pool->alloc_sz); + _HA_ATOMIC_DEC(&pool->allocated); +} + +/* Tries to allocate an object for the pool <pool> using the system's allocator + * and directly returns it. The pool's counters are updated but the object is + * never cached, so this is usable with and without local or shared caches. + */ +void *pool_alloc_nocache(struct pool_head *pool) +{ + void *ptr = NULL; + + ptr = pool_get_from_os(pool); + if (!ptr) + return NULL; + + swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4); + _HA_ATOMIC_INC(&pool->used); + + /* keep track of where the element was allocated from */ + POOL_DEBUG_SET_MARK(pool, ptr); + POOL_DEBUG_TRACE_CALLER(pool, (struct pool_cache_item *)ptr, NULL); + return ptr; +} + +/* Release a pool item back to the OS and keeps the pool's counters up to date. + * This is always defined even when pools are not enabled (their usage stats + * are maintained). + */ +void pool_free_nocache(struct pool_head *pool, void *ptr) +{ + _HA_ATOMIC_DEC(&pool->used); + swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used); + pool_put_to_os(pool, ptr); +} + + +/* Updates <pch>'s fill_pattern and fills the free area after <item> with it, + * up to <size> bytes. The item part is left untouched. + */ +void pool_fill_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size) +{ + ulong *ptr = (ulong *)item; + uint ofs; + ulong u; + + if (size <= sizeof(*item)) + return; + + /* Upgrade the fill_pattern to change about half of the bits + * (to be sure to catch static flag corruption), and apply it. + */ + u = pch->fill_pattern += ~0UL / 3; // 0x55...55 + ofs = sizeof(*item) / sizeof(*ptr); + while (ofs < size / sizeof(*ptr)) + ptr[ofs++] = u; +} + +/* check for a pool_cache_item integrity after extracting it from the cache. It + * must have been previously initialized using pool_fill_pattern(). If any + * corruption is detected, the function provokes an immediate crash. + */ +void pool_check_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size) +{ + const ulong *ptr = (const ulong *)item; + uint ofs; + ulong u; + + if (size <= sizeof(*item)) + return; + + /* let's check that all words past *item are equal */ + ofs = sizeof(*item) / sizeof(*ptr); + u = ptr[ofs++]; + while (ofs < size / sizeof(*ptr)) { + if (unlikely(ptr[ofs] != u)) + ABORT_NOW(); + ofs++; + } +} + +/* removes up to <count> items from the end of the local pool cache <ph> for + * pool <pool>. The shared pool is refilled with these objects in the limit + * of the number of acceptable objects, and the rest will be released to the + * OS. It is not a problem is <count> is larger than the number of objects in + * the local cache. The counters are automatically updated. Must not be used + * with pools disabled. + */ +static void pool_evict_last_items(struct pool_head *pool, struct pool_cache_head *ph, uint count) +{ + struct pool_cache_item *item; + struct pool_item *pi, *head = NULL; + uint released = 0; + uint cluster = 0; + uint to_free_max; + + BUG_ON(pool_debugging & POOL_DBG_NO_CACHE); + + /* Note: this will be zero when global pools are disabled */ + to_free_max = pool_releasable(pool); + + while (released < count && !LIST_ISEMPTY(&ph->list)) { + item = LIST_PREV(&ph->list, typeof(item), by_pool); + BUG_ON(&item->by_pool == &ph->list); + if (unlikely(pool_debugging & POOL_DBG_INTEGRITY)) + pool_check_pattern(ph, item, pool->size); + LIST_DELETE(&item->by_pool); + LIST_DELETE(&item->by_lru); + + if (to_free_max > released || cluster) { + /* will never match when global pools are disabled */ + pi = (struct pool_item *)item; + pi->next = NULL; + pi->down = head; + head = pi; + cluster++; + if (cluster >= CONFIG_HAP_POOL_CLUSTER_SIZE) { + /* enough to make a cluster */ + pool_put_to_shared_cache(pool, head, cluster); + cluster = 0; + head = NULL; + } + } else + pool_free_nocache(pool, item); + + released++; + } + + /* incomplete cluster left */ + if (cluster) + pool_put_to_shared_cache(pool, head, cluster); + + ph->count -= released; + pool_cache_count -= released; + pool_cache_bytes -= released * pool->size; +} + +/* Evicts some of the oldest objects from one local cache, until its number of + * objects is no more than 16+1/8 of the total number of locally cached objects + * or the total size of the local cache is no more than 75% of its maximum (i.e. + * we don't want a single cache to use all the cache for itself). For this, the + * list is scanned in reverse. If <full> is non-null, all objects are evicted. + * Must not be used when pools are disabled. + */ +void pool_evict_from_local_cache(struct pool_head *pool, int full) +{ + struct pool_cache_head *ph = &pool->cache[tid]; + + BUG_ON(pool_debugging & POOL_DBG_NO_CACHE); + + while ((ph->count && full) || + (ph->count >= CONFIG_HAP_POOL_CLUSTER_SIZE && + ph->count >= 16 + pool_cache_count / 8 && + pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) { + pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE); + } +} + +/* Evicts some of the oldest objects from the local cache, pushing them to the + * global pool. Must not be used when pools are disabled. + */ +void pool_evict_from_local_caches() +{ + struct pool_cache_item *item; + struct pool_cache_head *ph; + struct pool_head *pool; + + BUG_ON(pool_debugging & POOL_DBG_NO_CACHE); + + do { + item = LIST_PREV(&th_ctx->pool_lru_head, struct pool_cache_item *, by_lru); + BUG_ON(&item->by_lru == &th_ctx->pool_lru_head); + /* note: by definition we remove oldest objects so they also are the + * oldest in their own pools, thus their next is the pool's head. + */ + ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list); + BUG_ON(ph->tid != tid); + + pool = container_of(ph - tid, struct pool_head, cache); + BUG_ON(pool != ph->pool); + + pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE); + } while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8); +} + +/* Frees an object to the local cache, possibly pushing oldest objects to the + * shared cache, which itself may decide to release some of them to the OS. + * While it is unspecified what the object becomes past this point, it is + * guaranteed to be released from the users' perpective. A caller address may + * be passed and stored into the area when DEBUG_POOL_TRACING is set. Must not + * be used with pools disabled. + */ +void pool_put_to_cache(struct pool_head *pool, void *ptr, const void *caller) +{ + struct pool_cache_item *item = (struct pool_cache_item *)ptr; + struct pool_cache_head *ph = &pool->cache[tid]; + + BUG_ON(pool_debugging & POOL_DBG_NO_CACHE); + + LIST_INSERT(&ph->list, &item->by_pool); + LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru); + POOL_DEBUG_TRACE_CALLER(pool, item, caller); + ph->count++; + if (unlikely(pool_debugging & POOL_DBG_INTEGRITY)) + pool_fill_pattern(ph, item, pool->size); + pool_cache_count++; + pool_cache_bytes += pool->size; + + if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) { + if (ph->count >= 16 + pool_cache_count / 8 + CONFIG_HAP_POOL_CLUSTER_SIZE) + pool_evict_from_local_cache(pool, 0); + if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE) + pool_evict_from_local_caches(); + } +} + +/* Tries to refill the local cache <pch> from the shared one for pool <pool>. + * This is only used when pools are in use and shared pools are enabled. No + * malloc() is attempted, and poisonning is never performed. The purpose is to + * get the fastest possible refilling so that the caller can easily check if + * the cache has enough objects for its use. Must not be used when pools are + * disabled. + */ +void pool_refill_local_from_shared(struct pool_head *pool, struct pool_cache_head *pch) +{ + struct pool_cache_item *item; + struct pool_item *ret, *down; + uint count; + + BUG_ON(pool_debugging & POOL_DBG_NO_CACHE); + + /* we'll need to reference the first element to figure the next one. We + * must temporarily lock it so that nobody allocates then releases it, + * or the dereference could fail. + */ + ret = _HA_ATOMIC_LOAD(&pool->free_list); + do { + while (unlikely(ret == POOL_BUSY)) { + __ha_cpu_relax(); + ret = _HA_ATOMIC_LOAD(&pool->free_list); + } + if (ret == NULL) + return; + } while (unlikely((ret = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY)); + + if (unlikely(ret == NULL)) { + HA_ATOMIC_STORE(&pool->free_list, NULL); + return; + } + + /* this releases the lock */ + HA_ATOMIC_STORE(&pool->free_list, ret->next); + + /* now store the retrieved object(s) into the local cache */ + count = 0; + for (; ret; ret = down) { + down = ret->down; + item = (struct pool_cache_item *)ret; + POOL_DEBUG_TRACE_CALLER(pool, item, NULL); + LIST_INSERT(&pch->list, &item->by_pool); + LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru); + count++; + if (unlikely(pool_debugging & POOL_DBG_INTEGRITY)) + pool_fill_pattern(pch, item, pool->size); + } + HA_ATOMIC_ADD(&pool->used, count); + pch->count += count; + pool_cache_count += count; + pool_cache_bytes += count * pool->size; +} + +/* Adds pool item cluster <item> to the shared cache, which contains <count> + * elements. The caller is advised to first check using pool_releasable() if + * it's wise to add this series of objects there. Both the pool and the item's + * head must be valid. + */ +void pool_put_to_shared_cache(struct pool_head *pool, struct pool_item *item, uint count) +{ + struct pool_item *free_list; + + _HA_ATOMIC_SUB(&pool->used, count); + free_list = _HA_ATOMIC_LOAD(&pool->free_list); + do { + while (unlikely(free_list == POOL_BUSY)) { + __ha_cpu_relax(); + free_list = _HA_ATOMIC_LOAD(&pool->free_list); + } + _HA_ATOMIC_STORE(&item->next, free_list); + __ha_barrier_atomic_store(); + } while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, item)); + __ha_barrier_atomic_store(); + swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used); +} + +/* + * This function frees whatever can be freed in pool <pool>. + */ +void pool_flush(struct pool_head *pool) +{ + struct pool_item *next, *temp, *down; + + if (!pool || (pool_debugging & (POOL_DBG_NO_CACHE|POOL_DBG_NO_GLOBAL))) + return; + + /* The loop below atomically detaches the head of the free list and + * replaces it with a NULL. Then the list can be released. + */ + next = pool->free_list; + do { + while (unlikely(next == POOL_BUSY)) { + __ha_cpu_relax(); + next = _HA_ATOMIC_LOAD(&pool->free_list); + } + if (next == NULL) + return; + } while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY)); + _HA_ATOMIC_STORE(&pool->free_list, NULL); + __ha_barrier_atomic_store(); + + while (next) { + temp = next; + next = temp->next; + for (; temp; temp = down) { + down = temp->down; + pool_put_to_os(pool, temp); + } + } + /* here, we should have pool->allocated == pool->used */ +} + +/* + * This function frees whatever can be freed in all pools, but respecting + * the minimum thresholds imposed by owners. It makes sure to be alone to + * run by using thread_isolate(). <pool_ctx> is unused. + */ +void pool_gc(struct pool_head *pool_ctx) +{ + struct pool_head *entry; + int isolated = thread_isolated(); + + if (!isolated) + thread_isolate(); + + list_for_each_entry(entry, &pools, list) { + struct pool_item *temp, *down; + + while (entry->free_list && + (int)(entry->allocated - entry->used) > (int)entry->minavail) { + temp = entry->free_list; + entry->free_list = temp->next; + for (; temp; temp = down) { + down = temp->down; + pool_put_to_os(entry, temp); + } + } + } + + trim_all_pools(); + + if (!isolated) + thread_release(); +} + +/* + * Returns a pointer to type <type> taken from the pool <pool_type> or + * dynamically allocated. In the first case, <pool_type> is updated to point to + * the next element in the list. <flags> is a binary-OR of POOL_F_* flags. + * Prefer using pool_alloc() which does the right thing without flags. + */ +void *__pool_alloc(struct pool_head *pool, unsigned int flags) +{ + void *p = NULL; + void *caller = __builtin_return_address(0); + + if (unlikely(pool_debugging & POOL_DBG_FAIL_ALLOC)) + if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool)) + return NULL; + + if (likely(!(pool_debugging & POOL_DBG_NO_CACHE)) && !p) + p = pool_get_from_cache(pool, caller); + + if (unlikely(!p)) + p = pool_alloc_nocache(pool); + + if (likely(p)) { + if (unlikely(flags & POOL_F_MUST_ZERO)) + memset(p, 0, pool->size); + else if (unlikely(!(flags & POOL_F_NO_POISON) && (pool_debugging & POOL_DBG_POISON))) + memset(p, mem_poison_byte, pool->size); + } + return p; +} + +/* + * Puts a memory area back to the corresponding pool. <ptr> be valid. Using + * pool_free() is preferred. + */ +void __pool_free(struct pool_head *pool, void *ptr) +{ + const void *caller = __builtin_return_address(0); + + /* we'll get late corruption if we refill to the wrong pool or double-free */ + POOL_DEBUG_CHECK_MARK(pool, ptr); + POOL_DEBUG_RESET_MARK(pool, ptr); + + if (unlikely(pool_debugging & POOL_DBG_NO_CACHE)) { + pool_free_nocache(pool, ptr); + return; + } + + pool_put_to_cache(pool, ptr, caller); +} + + +#ifdef DEBUG_UAF + +/************* use-after-free allocator *************/ + +/* allocates an area of size <size> and returns it. The semantics are similar + * to those of malloc(). However the allocation is rounded up to 4kB so that a + * full page is allocated. This ensures the object can be freed alone so that + * future dereferences are easily detected. The returned object is always + * 16-bytes aligned to avoid issues with unaligned structure objects. In case + * some padding is added, the area's start address is copied at the end of the + * padding to help detect underflows. + */ +void *pool_alloc_area_uaf(size_t size) +{ + size_t pad = (4096 - size) & 0xFF0; + void *ret; + + ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (ret != MAP_FAILED) { + /* let's dereference the page before returning so that the real + * allocation in the system is performed without holding the lock. + */ + *(int *)ret = 0; + if (pad >= sizeof(void *)) + *(void **)(ret + pad - sizeof(void *)) = ret + pad; + ret += pad; + } else { + ret = NULL; + } + return ret; +} + +/* frees an area <area> of size <size> allocated by pool_alloc_area(). The + * semantics are identical to free() except that the size must absolutely match + * the one passed to pool_alloc_area(). In case some padding is added, the + * area's start address is compared to the one at the end of the padding, and + * a segfault is triggered if they don't match, indicating an underflow. + */ +void pool_free_area_uaf(void *area, size_t size) +{ + size_t pad = (4096 - size) & 0xFF0; + + if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area) + ABORT_NOW(); + + munmap(area - pad, (size + 4095) & -4096); +} + +#endif /* DEBUG_UAF */ + +/* + * This function destroys a pool by freeing it completely, unless it's still + * in use. This should be called only under extreme circumstances. It always + * returns NULL if the resulting pool is empty, easing the clearing of the old + * pointer, otherwise it returns the pool. + * . + */ +void *pool_destroy(struct pool_head *pool) +{ + if (pool) { + if (!(pool_debugging & POOL_DBG_NO_CACHE)) + pool_evict_from_local_cache(pool, 1); + + pool_flush(pool); + if (pool->used) + return pool; + pool->users--; + if (!pool->users) { + LIST_DELETE(&pool->list); + /* note that if used == 0, the cache is empty */ + free(pool->base_addr); + } + } + return NULL; +} + +/* This destroys all pools on exit. It is *not* thread safe. */ +void pool_destroy_all() +{ + struct pool_head *entry, *back; + + list_for_each_entry_safe(entry, back, &pools, list) { + /* there's only one occurrence of each pool in the list, + * and we're existing instead of looping on the whole + * list just to decrement users, force it to 1 here. + */ + entry->users = 1; + pool_destroy(entry); + } +} + +/* This function dumps memory usage information into the trash buffer. */ +void dump_pools_to_trash() +{ + struct pool_head *entry; + unsigned long long allocated, used; + int nbpools; + unsigned long long cached_bytes = 0; + uint cached = 0; + + allocated = used = nbpools = 0; + chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n"); + list_for_each_entry(entry, &pools, list) { + if (!(pool_debugging & POOL_DBG_NO_CACHE)) { + int i; + for (cached = i = 0; i < global.nbthread; i++) + cached += entry->cache[i].count; + cached_bytes += cached * (ullong)entry->size; + } + chunk_appendf(&trash, " - Pool %s (%u bytes) : %u allocated (%llu bytes), %u used" + " (~%u by thread caches)" + ", needed_avg %u, %u failures, %u users, @%p%s\n", + entry->name, entry->size, entry->allocated, + (ullong)entry->size * entry->allocated, entry->used, + cached, + swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed, + entry->users, entry, + (entry->flags & MEM_F_SHARED) ? " [SHARED]" : ""); + + allocated += entry->allocated * (ullong)entry->size; + used += entry->used * (ullong)entry->size; + nbpools++; + } + chunk_appendf(&trash, "Total: %d pools, %llu bytes allocated, %llu used" + " (~%llu by thread caches)" + ".\n", + nbpools, allocated, used, cached_bytes + ); +} + +/* Dump statistics on pools usage. */ +void dump_pools(void) +{ + dump_pools_to_trash(); + qfprintf(stderr, "%s", trash.area); +} + +/* This function returns the total number of failed pool allocations */ +int pool_total_failures() +{ + struct pool_head *entry; + int failed = 0; + + list_for_each_entry(entry, &pools, list) + failed += entry->failed; + return failed; +} + +/* This function returns the total amount of memory allocated in pools (in bytes) */ +unsigned long long pool_total_allocated() +{ + struct pool_head *entry; + unsigned long long allocated = 0; + + list_for_each_entry(entry, &pools, list) + allocated += entry->allocated * (ullong)entry->size; + return allocated; +} + +/* This function returns the total amount of memory used in pools (in bytes) */ +unsigned long long pool_total_used() +{ + struct pool_head *entry; + unsigned long long used = 0; + + list_for_each_entry(entry, &pools, list) + used += entry->used * (ullong)entry->size; + return used; +} + +/* This function parses a string made of a set of debugging features as + * specified after -dM on the command line, and will set pool_debugging + * accordingly. On success it returns a strictly positive value. It may zero + * with the first warning in <err>, -1 with a help message in <err>, or -2 with + * the first error in <err> return the first error in <err>. <err> is undefined + * on success, and will be non-null and locally allocated on help/error/warning. + * The caller must free it. Warnings are used to report features that were not + * enabled at build time, and errors are used to report unknown features. + */ +int pool_parse_debugging(const char *str, char **err) +{ + struct ist args; + char *end; + uint new_dbg; + int v; + + + /* if it's empty or starts with a number, it's the mem poisonning byte */ + v = strtol(str, &end, 0); + if (!*end || *end == ',') { + mem_poison_byte = *str ? v : 'P'; + if (mem_poison_byte >= 0) + pool_debugging |= POOL_DBG_POISON; + else + pool_debugging &= ~POOL_DBG_POISON; + str = end; + } + + new_dbg = pool_debugging; + + for (args = ist(str); istlen(args); args = istadv(istfind(args, ','), 1)) { + struct ist feat = iststop(args, ','); + + if (!istlen(feat)) + continue; + + if (isteq(feat, ist("help"))) { + ha_free(err); + memprintf(err, + "-dM alone enables memory poisonning with byte 0x50 on allocation. A numeric\n" + "value may be appended immediately after -dM to use another value (0 supported).\n" + "Then an optional list of comma-delimited keywords may be appended to set or\n" + "clear some debugging options ('*' marks the current setting):\n\n" + " set clear description\n" + " -----------------+-----------------+-----------------------------------------\n"); + + for (v = 0; dbg_options[v].flg; v++) { + memprintf(err, "%s %c %-15s|%c %-15s| %s\n", + *err, + (pool_debugging & dbg_options[v].flg) ? '*' : ' ', + dbg_options[v].set, + (pool_debugging & dbg_options[v].flg) ? ' ' : '*', + dbg_options[v].clr, + dbg_options[v].hlp); + } + return -1; + } + + for (v = 0; dbg_options[v].flg; v++) { + if (isteq(feat, ist(dbg_options[v].set))) { + new_dbg |= dbg_options[v].flg; + break; + } + else if (isteq(feat, ist(dbg_options[v].clr))) { + new_dbg &= ~dbg_options[v].flg; + break; + } + } + + if (!dbg_options[v].flg) { + memprintf(err, "unknown pool debugging feature <%.*s>", (int)istlen(feat), istptr(feat)); + return -2; + } + } + + pool_debugging = new_dbg; + return 1; +} + +/* This function dumps memory usage information onto the stream connector's + * read buffer. It returns 0 as long as it does not complete, non-zero upon + * completion. No state is used. + */ +static int cli_io_handler_dump_pools(struct appctx *appctx) +{ + dump_pools_to_trash(); + if (applet_putchk(appctx, &trash) == -1) + return 0; + return 1; +} + +/* callback used to create early pool <name> of size <size> and store the + * resulting pointer into <ptr>. If the allocation fails, it quits with after + * emitting an error message. + */ +void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size) +{ + *ptr = create_pool(name, size, MEM_F_SHARED); + if (!*ptr) { + ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n", + name, size, strerror(errno)); + exit(1); + } +} + +/* Initializes all per-thread arrays on startup */ +static void init_pools() +{ + int thr; + + for (thr = 0; thr < MAX_THREADS; thr++) { + LIST_INIT(&ha_thread_ctx[thr].pool_lru_head); + } + + detect_allocator(); +} + +INITCALL0(STG_PREPARE, init_pools); + +/* Report in build options if trim is supported */ +static void pools_register_build_options(void) +{ + if (is_trim_enabled()) { + char *ptr = NULL; + memprintf(&ptr, "Support for malloc_trim() is enabled."); + hap_register_build_opts(ptr, 1); + } +} +INITCALL0(STG_REGISTER, pools_register_build_options); + +/* register cli keywords */ +static struct cli_kw_list cli_kws = {{ },{ + { { "show", "pools", NULL }, "show pools : report information about the memory pools usage", NULL, cli_io_handler_dump_pools }, + {{},} +}}; + +INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws); + + +/* config parser for global "tune.fail-alloc" */ +static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx, + const struct proxy *defpx, const char *file, int line, + char **err) +{ + if (too_many_args(1, args, err, NULL)) + return -1; + mem_fail_rate = atoi(args[1]); + if (mem_fail_rate < 0 || mem_fail_rate > 100) { + memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]); + return -1; + } + return 0; +} + +/* config parser for global "no-memory-trimming" */ +static int mem_parse_global_no_mem_trim(char **args, int section_type, struct proxy *curpx, + const struct proxy *defpx, const char *file, int line, + char **err) +{ + if (too_many_args(0, args, err, NULL)) + return -1; + disable_trim = 1; + return 0; +} + +/* register global config keywords */ +static struct cfg_kw_list mem_cfg_kws = {ILH, { + { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc }, + { CFG_GLOBAL, "no-memory-trimming", mem_parse_global_no_mem_trim }, + { 0, NULL, NULL } +}}; + +INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws); + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */ |