/* * Copyright © 2020, VideoLAN and dav1d authors * Copyright © 2020, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" #include #include "src/internal.h" #if TRACK_HEAP_ALLOCATIONS #include #include "src/log.h" #define DEFAULT_ALIGN 16 typedef struct { size_t sz; unsigned align; enum AllocationType type; } Dav1dAllocationData; typedef struct { size_t curr_sz; size_t peak_sz; unsigned num_allocs; unsigned num_reuses; } AllocStats; static AllocStats tracked_allocs[N_ALLOC_TYPES]; static size_t curr_total_sz; static size_t peak_total_sz; static pthread_mutex_t track_alloc_mutex = PTHREAD_MUTEX_INITIALIZER; static void *track_alloc(const enum AllocationType type, char *ptr, const size_t sz, const size_t align) { assert(align >= sizeof(Dav1dAllocationData)); if (ptr) { ptr += align; Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1]; AllocStats *const s = &tracked_allocs[type]; d->sz = sz; d->align = (unsigned)align; d->type = type; pthread_mutex_lock(&track_alloc_mutex); s->num_allocs++; s->curr_sz += sz; if (s->curr_sz > s->peak_sz) s->peak_sz = s->curr_sz; curr_total_sz += sz; if (curr_total_sz > peak_total_sz) peak_total_sz = curr_total_sz; pthread_mutex_unlock(&track_alloc_mutex); } return ptr; } static void *track_free(char *const ptr) { const Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1]; const size_t sz = d->sz; pthread_mutex_lock(&track_alloc_mutex); tracked_allocs[d->type].curr_sz -= sz; curr_total_sz -= sz; pthread_mutex_unlock(&track_alloc_mutex); return ptr - d->align; } static void dav1d_track_reuse(const enum AllocationType type) { pthread_mutex_lock(&track_alloc_mutex); tracked_allocs[type].num_reuses++; pthread_mutex_unlock(&track_alloc_mutex); } void *dav1d_malloc(const enum AllocationType type, const size_t sz) { void *const ptr = malloc(sz + DEFAULT_ALIGN); return track_alloc(type, ptr, sz, DEFAULT_ALIGN); } void *dav1d_alloc_aligned(const enum AllocationType type, const size_t sz, const size_t align) { assert(!(align & (align - 1))); void *ptr; #ifdef _WIN32 ptr = _aligned_malloc(sz + align, align); #elif defined(HAVE_POSIX_MEMALIGN) if (posix_memalign(&ptr, align, sz + align)) return NULL; #else ptr = memalign(align, sz + align); #endif return track_alloc(type, ptr, sz, align); } void *dav1d_realloc(const enum AllocationType type, void *ptr, const size_t sz) { if (!ptr) return dav1d_malloc(type, sz); ptr = realloc((char*)ptr - DEFAULT_ALIGN, sz + DEFAULT_ALIGN); if (ptr) ptr = track_free((char*)ptr + DEFAULT_ALIGN); return track_alloc(type, ptr, sz, DEFAULT_ALIGN); } void dav1d_free(void *ptr) { if (ptr) free(track_free(ptr)); } void dav1d_free_aligned(void *ptr) { if (ptr) { ptr = track_free(ptr); #ifdef _WIN32 _aligned_free(ptr); #else free(ptr); #endif } } static COLD int cmp_stats(const void *const a, const void *const b) { const size_t a_sz = ((const AllocStats*)a)->peak_sz; const size_t b_sz = ((const AllocStats*)b)->peak_sz; return a_sz < b_sz ? -1 : a_sz > b_sz; } /* Insert spaces as thousands separators for better readability */ static COLD int format_tsep(char *const s, const size_t n, const size_t value) { if (value < 1000) return snprintf(s, n, "%u", (unsigned)value); const int len = format_tsep(s, n, value / 1000); assert((size_t)len < n); return len + snprintf(s + len, n - len, " %03u", (unsigned)(value % 1000)); } COLD void dav1d_log_alloc_stats(Dav1dContext *const c) { static const char *const type_names[N_ALLOC_TYPES] = { [ALLOC_BLOCK ] = "Block data", [ALLOC_CDEF ] = "CDEF line buffers", [ALLOC_CDF ] = "CDF contexts", [ALLOC_COEF ] = "Coefficient data", [ALLOC_COMMON_CTX] = "Common context data", [ALLOC_DAV1DDATA ] = "Dav1dData", [ALLOC_IPRED ] = "Intra pred edges", [ALLOC_LF ] = "Loopfilter data", [ALLOC_LR ] = "Looprestoration data", [ALLOC_OBU_HDR ] = "OBU headers", [ALLOC_OBU_META ] = "OBU metadata", [ALLOC_PAL ] = "Palette data", [ALLOC_PIC ] = "Picture buffers", [ALLOC_PIC_CTX ] = "Picture context data", [ALLOC_REFMVS ] = "Reference mv data", [ALLOC_SEGMAP ] = "Segmentation maps", [ALLOC_THREAD_CTX] = "Thread context data", [ALLOC_TILE ] = "Tile data", }; struct { AllocStats stats; enum AllocationType type; } data[N_ALLOC_TYPES]; unsigned total_allocs = 0; unsigned total_reuses = 0; pthread_mutex_lock(&track_alloc_mutex); for (int i = 0; i < N_ALLOC_TYPES; i++) { AllocStats *const s = &data[i].stats; *s = tracked_allocs[i]; data[i].type = i; total_allocs += s->num_allocs; total_reuses += s->num_reuses; } size_t total_sz = peak_total_sz; pthread_mutex_unlock(&track_alloc_mutex); /* Sort types by memory usage */ qsort(&data, N_ALLOC_TYPES, sizeof(*data), cmp_stats); const double inv_total_share = 100.0 / total_sz; char total_sz_buf[32]; const int sz_len = 4 + format_tsep(total_sz_buf, sizeof(total_sz_buf), total_sz); dav1d_log(c, "\n Type Allocs Reuses Share Peak size\n" "---------------------------------------------------------------------\n"); for (int i = N_ALLOC_TYPES - 1; i >= 0; i--) { const AllocStats *const s = &data[i].stats; if (s->num_allocs) { const double share = s->peak_sz * inv_total_share; char sz_buf[32]; format_tsep(sz_buf, sizeof(sz_buf), s->peak_sz); dav1d_log(c, " %-20s%10u%10u%8.1f%%%*s\n", type_names[data[i].type], s->num_allocs, s->num_reuses, share, sz_len, sz_buf); } } dav1d_log(c, "---------------------------------------------------------------------\n" "%31u%10u %s\n", total_allocs, total_reuses, total_sz_buf); } #endif /* TRACK_HEAP_ALLOCATIONS */ static COLD void mem_pool_destroy(Dav1dMemPool *const pool) { pthread_mutex_destroy(&pool->lock); dav1d_free(pool); } void dav1d_mem_pool_push(Dav1dMemPool *const pool, Dav1dMemPoolBuffer *const buf) { pthread_mutex_lock(&pool->lock); const int ref_cnt = --pool->ref_cnt; if (!pool->end) { buf->next = pool->buf; pool->buf = buf; pthread_mutex_unlock(&pool->lock); assert(ref_cnt > 0); } else { pthread_mutex_unlock(&pool->lock); dav1d_free_aligned(buf->data); if (!ref_cnt) mem_pool_destroy(pool); } } Dav1dMemPoolBuffer *dav1d_mem_pool_pop(Dav1dMemPool *const pool, const size_t size) { assert(!(size & (sizeof(void*) - 1))); pthread_mutex_lock(&pool->lock); Dav1dMemPoolBuffer *buf = pool->buf; pool->ref_cnt++; uint8_t *data; if (buf) { pool->buf = buf->next; pthread_mutex_unlock(&pool->lock); data = buf->data; if ((uintptr_t)buf - (uintptr_t)data != size) { /* Reallocate if the size has changed */ dav1d_free_aligned(data); goto alloc; } #if TRACK_HEAP_ALLOCATIONS dav1d_track_reuse(pool->type); #endif } else { pthread_mutex_unlock(&pool->lock); alloc: data = dav1d_alloc_aligned(pool->type, size + sizeof(Dav1dMemPoolBuffer), 64); if (!data) { pthread_mutex_lock(&pool->lock); const int ref_cnt = --pool->ref_cnt; pthread_mutex_unlock(&pool->lock); if (!ref_cnt) mem_pool_destroy(pool); return NULL; } buf = (Dav1dMemPoolBuffer*)(data + size); buf->data = data; } return buf; } COLD int dav1d_mem_pool_init(const enum AllocationType type, Dav1dMemPool **const ppool) { Dav1dMemPool *const pool = dav1d_malloc(ALLOC_COMMON_CTX, sizeof(Dav1dMemPool)); if (pool) { if (!pthread_mutex_init(&pool->lock, NULL)) { pool->buf = NULL; pool->ref_cnt = 1; pool->end = 0; #if TRACK_HEAP_ALLOCATIONS pool->type = type; #endif *ppool = pool; return 0; } dav1d_free(pool); } *ppool = NULL; return DAV1D_ERR(ENOMEM); } COLD void dav1d_mem_pool_end(Dav1dMemPool *const pool) { if (pool) { pthread_mutex_lock(&pool->lock); Dav1dMemPoolBuffer *buf = pool->buf; const int ref_cnt = --pool->ref_cnt; pool->buf = NULL; pool->end = 1; pthread_mutex_unlock(&pool->lock); while (buf) { void *const data = buf->data; buf = buf->next; dav1d_free_aligned(data); } if (!ref_cnt) mem_pool_destroy(pool); } }