diff options
Diffstat (limited to 'src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp')
| -rw-r--r-- | src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp | 1019 |
1 files changed, 1019 insertions, 0 deletions
diff --git a/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp b/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp new file mode 100644 index 00000000..6969e0f6 --- /dev/null +++ b/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp @@ -0,0 +1,1019 @@ +/* $Id: alloc-ef-r0drv.cpp $ */ +/** @file + * IPRT - Memory Allocation, electric fence for ring-0 drivers. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <https://www.gnu.org/licenses>. + * + * The contents of this file may alternatively be used under the terms + * of the Common Development and Distribution License Version 1.0 + * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included + * in the VirtualBox distribution, in which case the provisions of the + * CDDL are applicable instead of those of the GPL. + * + * You may elect to license modified versions of this file under the + * terms and conditions of either the GPL or the CDDL or both. + * + * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define RTMEM_NO_WRAP_TO_EF_APIS +#include "internal/iprt.h" +#include <iprt/mem.h> + +#include <iprt/alloc.h> +#include <iprt/asm.h> +#include <iprt/asm-amd64-x86.h> +#include <iprt/assert.h> +#include <iprt/errcore.h> +#include <iprt/log.h> +#include <iprt/memobj.h> +#include <iprt/param.h> +#include <iprt/string.h> +#include <iprt/thread.h> + +#include "internal/mem.h" + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#if defined(DOXYGEN_RUNNING) +# define RTR0MEM_EF_IN_FRONT +#endif + +/** @def RTR0MEM_EF_SIZE + * The size of the fence. This must be page aligned. + */ +#define RTR0MEM_EF_SIZE PAGE_SIZE + +/** @def RTR0MEM_EF_ALIGNMENT + * The allocation alignment, power of two of course. + * + * Use this for working around misaligned sizes, usually stemming from + * allocating a string or something after the main structure. When you + * encounter this, please fix the allocation to RTMemAllocVar or RTMemAllocZVar. + */ +#if 0 +# define RTR0MEM_EF_ALIGNMENT (ARCH_BITS / 8) +#else +# define RTR0MEM_EF_ALIGNMENT 1 +#endif + +/** @def RTR0MEM_EF_IN_FRONT + * Define this to put the fence up in front of the block. + * The default (when this isn't defined) is to up it up after the block. + */ +//# define RTR0MEM_EF_IN_FRONT + +/** @def RTR0MEM_EF_FREE_DELAYED + * This define will enable free() delay and protection of the freed data + * while it's being delayed. The value of RTR0MEM_EF_FREE_DELAYED defines + * the threshold of the delayed blocks. + * Delayed blocks does not consume any physical memory, only virtual address space. + */ +#define RTR0MEM_EF_FREE_DELAYED (20 * _1M) + +/** @def RTR0MEM_EF_FREE_FILL + * This define will enable memset(,RTR0MEM_EF_FREE_FILL,)'ing the user memory + * in the block before freeing/decommitting it. This is useful in GDB since GDB + * appears to be able to read the content of the page even after it's been + * decommitted. + */ +#define RTR0MEM_EF_FREE_FILL 'f' + +/** @def RTR0MEM_EF_FILLER + * This define will enable memset(,RTR0MEM_EF_FILLER,)'ing the allocated + * memory when the API doesn't require it to be zero'd. + */ +#define RTR0MEM_EF_FILLER 0xef + +/** @def RTR0MEM_EF_NOMAN_FILLER + * This define will enable memset(,RTR0MEM_EF_NOMAN_FILLER,)'ing the + * unprotected but not allocated area of memory, the so called no man's land. + */ +#define RTR0MEM_EF_NOMAN_FILLER 0xaa + +/** @def RTR0MEM_EF_FENCE_FILLER + * This define will enable memset(,RTR0MEM_EF_FENCE_FILLER,)'ing the + * fence itself, as debuggers can usually read them. + */ +#define RTR0MEM_EF_FENCE_FILLER 0xcc + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#ifdef RT_OS_WINDOWS +# include <iprt/win/windows.h> +#elif !defined(RT_OS_FREEBSD) +# include <sys/mman.h> +#endif +#include <iprt/avl.h> +#include <iprt/thread.h> + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +/** + * Allocation types. + */ +typedef enum RTMEMTYPE +{ + RTMEMTYPE_RTMEMALLOC, + RTMEMTYPE_RTMEMALLOCZ, + RTMEMTYPE_RTMEMREALLOC, + RTMEMTYPE_RTMEMFREE, + RTMEMTYPE_RTMEMFREEZ, + + RTMEMTYPE_NEW, + RTMEMTYPE_NEW_ARRAY, + RTMEMTYPE_DELETE, + RTMEMTYPE_DELETE_ARRAY +} RTMEMTYPE; + +/** + * Node tracking a memory allocation. + */ +typedef struct RTR0MEMEFBLOCK +{ + /** Avl node code, key is the user block pointer. */ + AVLPVNODECORE Core; + /** Allocation type. */ + RTMEMTYPE enmType; + /** The memory object. */ + RTR0MEMOBJ hMemObj; + /** The unaligned size of the block. */ + size_t cbUnaligned; + /** The aligned size of the block. */ + size_t cbAligned; + /** The allocation tag (read-only string). */ + const char *pszTag; + /** The return address of the allocator function. */ + void *pvCaller; + /** Line number of the alloc call. */ + unsigned iLine; + /** File from within the allocation was made. */ + const char *pszFile; + /** Function from within the allocation was made. */ + const char *pszFunction; +} RTR0MEMEFBLOCK, *PRTR0MEMEFBLOCK; + + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** Spinlock protecting the all the block's globals. */ +static volatile uint32_t g_BlocksLock; +/** Tree tracking the allocations. */ +static AVLPVTREE g_BlocksTree; + +#ifdef RTR0MEM_EF_FREE_DELAYED +/** Tail of the delayed blocks. */ +static volatile PRTR0MEMEFBLOCK g_pBlocksDelayHead; +/** Tail of the delayed blocks. */ +static volatile PRTR0MEMEFBLOCK g_pBlocksDelayTail; +/** Number of bytes in the delay list (includes fences). */ +static volatile size_t g_cbBlocksDelay; +#endif /* RTR0MEM_EF_FREE_DELAYED */ + +/** Array of pointers free watches for. */ +void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL}; +/** Enable logging of all freed memory. */ +bool gfRTMemFreeLog = false; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ + + +/** + * @callback_method_impl{FNRTSTROUTPUT} + */ +static DECLCALLBACK(size_t) rtR0MemEfWrite(void *pvArg, const char *pachChars, size_t cbChars) +{ + RT_NOREF1(pvArg); + if (cbChars) + { + RTLogWriteDebugger(pachChars, cbChars); + RTLogWriteStdOut(pachChars, cbChars); + RTLogWriteUser(pachChars, cbChars); + } + return cbChars; +} + + +/** + * Complains about something. + */ +static void rtR0MemComplain(const char *pszOp, const char *pszFormat, ...) +{ + va_list args; + RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem error: %s: ", pszOp); + va_start(args, pszFormat); + RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args); + va_end(args); + RTAssertDoPanic(); +} + +/** + * Log an event. + */ +DECLINLINE(void) rtR0MemLog(const char *pszOp, const char *pszFormat, ...) +{ +#if 0 + va_list args; + RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem info: %s: ", pszOp); + va_start(args, pszFormat); + RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args); + va_end(args); +#else + NOREF(pszOp); NOREF(pszFormat); +#endif +} + + + +/** + * Acquires the lock. + */ +DECLINLINE(RTCCUINTREG) rtR0MemBlockLock(void) +{ + RTCCUINTREG uRet; + unsigned c = 0; + if (RTThreadPreemptIsEnabled(NIL_RTTHREAD)) + { + for (;;) + { + uRet = ASMIntDisableFlags(); + if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0)) + break; + ASMSetFlags(uRet); + RTThreadSleepNoLog(((++c) >> 2) & 31); + } + } + else + { + for (;;) + { + uRet = ASMIntDisableFlags(); + if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0)) + break; + ASMSetFlags(uRet); + ASMNopPause(); + if (++c & 3) + ASMNopPause(); + } + } + return uRet; +} + + +/** + * Releases the lock. + */ +DECLINLINE(void) rtR0MemBlockUnlock(RTCCUINTREG fSavedIntFlags) +{ + Assert(g_BlocksLock == 1); + ASMAtomicXchgU32(&g_BlocksLock, 0); + ASMSetFlags(fSavedIntFlags); +} + + +/** + * Creates a block. + */ +DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned, + const char *pszTag, void *pvCaller, RT_SRC_POS_DECL) +{ + PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTMemAlloc(sizeof(*pBlock)); + if (pBlock) + { + pBlock->enmType = enmType; + pBlock->cbUnaligned = cbUnaligned; + pBlock->cbAligned = cbAligned; + pBlock->pszTag = pszTag; + pBlock->pvCaller = pvCaller; + pBlock->iLine = iLine; + pBlock->pszFile = pszFile; + pBlock->pszFunction = pszFunction; + } + return pBlock; +} + + +/** + * Frees a block. + */ +DECLINLINE(void) rtR0MemBlockFree(PRTR0MEMEFBLOCK pBlock) +{ + RTMemFree(pBlock); +} + + +/** + * Insert a block from the tree. + */ +DECLINLINE(void) rtR0MemBlockInsert(PRTR0MEMEFBLOCK pBlock, void *pv, RTR0MEMOBJ hMemObj) +{ + pBlock->Core.Key = pv; + pBlock->hMemObj = hMemObj; + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core); + rtR0MemBlockUnlock(fSavedIntFlags); + AssertRelease(fRc); +} + + +/** + * Remove a block from the tree and returns it to the caller. + */ +DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockRemove(void *pv) +{ + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVRemove(&g_BlocksTree, pv); + rtR0MemBlockUnlock(fSavedIntFlags); + return pBlock; +} + + +/** + * Gets a block. + */ +DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockGet(void *pv) +{ + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVGet(&g_BlocksTree, pv); + rtR0MemBlockUnlock(fSavedIntFlags); + return pBlock; +} + + +/** + * Dumps one allocation. + */ +static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser) +{ + PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode; + RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "%p %08lx(+%02lx) %p\n", + pBlock->Core.Key, + (unsigned long)pBlock->cbUnaligned, + (unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned), + pBlock->pvCaller); + NOREF(pvUser); + return 0; +} + + +/** + * Dumps the allocated blocks. + * This is something which you should call from gdb. + */ +RT_C_DECLS_BEGIN +void RTMemDump(void); +RT_C_DECLS_END + +void RTMemDump(void) +{ + RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "address size(alg) caller\n"); + RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL); +} + +#ifdef RTR0MEM_EF_FREE_DELAYED + +/** + * Insert a delayed block. + */ +DECLINLINE(void) rtR0MemBlockDelayInsert(PRTR0MEMEFBLOCK pBlock) +{ + size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE; + pBlock->Core.pRight = NULL; + pBlock->Core.pLeft = NULL; + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + if (g_pBlocksDelayHead) + { + g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock; + pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead; + g_pBlocksDelayHead = pBlock; + } + else + { + g_pBlocksDelayTail = pBlock; + g_pBlocksDelayHead = pBlock; + } + g_cbBlocksDelay += cbBlock; + rtR0MemBlockUnlock(fSavedIntFlags); +} + +/** + * Removes a delayed block. + */ +DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockDelayRemove(void) +{ + PRTR0MEMEFBLOCK pBlock = NULL; + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + if (g_cbBlocksDelay > RTR0MEM_EF_FREE_DELAYED) + { + pBlock = g_pBlocksDelayTail; + if (pBlock) + { + g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft; + if (pBlock->Core.pLeft) + pBlock->Core.pLeft->pRight = NULL; + else + g_pBlocksDelayHead = NULL; + g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE; + } + } + rtR0MemBlockUnlock(fSavedIntFlags); + return pBlock; +} + +#endif /* RTR0MEM_EF_FREE_DELAYED */ + + +static void rtR0MemFreeBlock(PRTR0MEMEFBLOCK pBlock, const char *pszOp) +{ + void *pv = pBlock->Core.Key; +# ifdef RTR0MEM_EF_IN_FRONT + void *pvBlock = (char *)pv - RTR0MEM_EF_SIZE; +# else + void *pvBlock = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK); +# endif + size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE; + + int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /*offSub*/, RT_ALIGN_Z(cbBlock, PAGE_SIZE), RTMEM_PROT_READ | RTMEM_PROT_WRITE); + if (RT_FAILURE(rc)) + rtR0MemComplain(pszOp, "RTR0MemObjProtect([%p], 0, %#x, RTMEM_PROT_READ | RTMEM_PROT_WRITE) -> %Rrc\n", + pvBlock, cbBlock, rc); + + rc = RTR0MemObjFree(pBlock->hMemObj, true /*fFreeMappings*/); + if (RT_FAILURE(rc)) + rtR0MemComplain(pszOp, "RTR0MemObjFree([%p LB %#x]) -> %Rrc\n", pvBlock, cbBlock, rc); + pBlock->hMemObj = NIL_RTR0MEMOBJ; + + rtR0MemBlockFree(pBlock); +} + + +/** + * Initialize call, we shouldn't fail here. + */ +void rtR0MemEfInit(void) +{ + +} + +/** + * @callback_method_impl{AVLPVCALLBACK} + */ +static DECLCALLBACK(int) rtR0MemEfDestroyBlock(PAVLPVNODECORE pNode, void *pvUser) +{ + PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode; + + /* Note! pszFile and pszFunction may be invalid at this point. */ + rtR0MemComplain("rtR0MemEfDestroyBlock", "Leaking %zu bytes at %p (iLine=%u pvCaller=%p)\n", + pBlock->cbAligned, pBlock->Core.Key, pBlock->iLine, pBlock->pvCaller); + + rtR0MemFreeBlock(pBlock, "rtR0MemEfDestroyBlock"); + + NOREF(pvUser); + return VINF_SUCCESS; +} + + +/** + * Termination call. + * + * Will check and free memory. + */ +void rtR0MemEfTerm(void) +{ +#ifdef RTR0MEM_EF_FREE_DELAYED + /* + * Release delayed frees. + */ + RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock(); + for (;;) + { + PRTR0MEMEFBLOCK pBlock = g_pBlocksDelayTail; + if (pBlock) + { + g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft; + if (pBlock->Core.pLeft) + pBlock->Core.pLeft->pRight = NULL; + else + g_pBlocksDelayHead = NULL; + rtR0MemBlockUnlock(fSavedIntFlags); + + rtR0MemFreeBlock(pBlock, "rtR0MemEfTerm"); + + rtR0MemBlockLock(); + } + else + break; + } + g_cbBlocksDelay = 0; + rtR0MemBlockUnlock(fSavedIntFlags); +#endif + + /* + * Complain about leaks. Then release them. + */ + RTAvlPVDestroy(&g_BlocksTree, rtR0MemEfDestroyBlock, NULL); +} + + +/** + * Internal allocator. + */ +static void * rtR0MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned, + const char *pszTag, void *pvCaller, RT_SRC_POS_DECL) +{ + /* + * Sanity. + */ + if ( RT_ALIGN_Z(RTR0MEM_EF_SIZE, PAGE_SIZE) != RTR0MEM_EF_SIZE + && RTR0MEM_EF_SIZE <= 0) + { + rtR0MemComplain(pszOp, "Invalid E-fence size! %#x\n", RTR0MEM_EF_SIZE); + return NULL; + } + if (!cbUnaligned) + { +#if 1 + rtR0MemComplain(pszOp, "Request of ZERO bytes allocation!\n"); + return NULL; +#else + cbAligned = cbUnaligned = 1; +#endif + } + +#ifndef RTR0MEM_EF_IN_FRONT + /* Alignment decreases fence accuracy, but this is at least partially + * counteracted by filling and checking the alignment padding. When the + * fence is in front then then no extra alignment is needed. */ + cbAligned = RT_ALIGN_Z(cbAligned, RTR0MEM_EF_ALIGNMENT); +#endif + + /* + * Allocate the trace block. + */ + PRTR0MEMEFBLOCK pBlock = rtR0MemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS); + if (!pBlock) + { + rtR0MemComplain(pszOp, "Failed to allocate trace block!\n"); + return NULL; + } + + /* + * Allocate a block with page alignment space + the size of the E-fence. + */ + void *pvBlock = NULL; + RTR0MEMOBJ hMemObj; + size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE; + int rc = RTR0MemObjAllocPage(&hMemObj, cbBlock, false /*fExecutable*/); + if (RT_SUCCESS(rc)) + pvBlock = RTR0MemObjAddress(hMemObj); + if (pvBlock) + { + /* + * Calc the start of the fence and the user block + * and then change the page protection of the fence. + */ +#ifdef RTR0MEM_EF_IN_FRONT + void *pvEFence = pvBlock; + void *pv = (char *)pvEFence + RTR0MEM_EF_SIZE; +# ifdef RTR0MEM_EF_NOMAN_FILLER + memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbUnaligned); +# endif +#else + void *pvEFence = (char *)pvBlock + (cbBlock - RTR0MEM_EF_SIZE); + void *pv = (char *)pvEFence - cbAligned; +# ifdef RTR0MEM_EF_NOMAN_FILLER + memset(pvBlock, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbAligned); + memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbAligned - cbUnaligned); +# endif +#endif + +#ifdef RTR0MEM_EF_FENCE_FILLER + memset(pvEFence, RTR0MEM_EF_FENCE_FILLER, RTR0MEM_EF_SIZE); +#endif + rc = RTR0MemObjProtect(hMemObj, (uint8_t *)pvEFence - (uint8_t *)pvBlock, RTR0MEM_EF_SIZE, RTMEM_PROT_NONE); + if (!rc) + { + rtR0MemBlockInsert(pBlock, pv, hMemObj); + if (enmType == RTMEMTYPE_RTMEMALLOCZ) + memset(pv, 0, cbUnaligned); +#ifdef RTR0MEM_EF_FILLER + else + memset(pv, RTR0MEM_EF_FILLER, cbUnaligned); +#endif + + rtR0MemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned); + return pv; + } + rtR0MemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTR0MEM_EF_SIZE, rc); + RTR0MemObjFree(hMemObj, true /*fFreeMappings*/); + } + else + { + rtR0MemComplain(pszOp, "Failed to allocated %zu (%zu) bytes (rc=%Rrc).\n", cbBlock, cbUnaligned, rc); + if (RT_SUCCESS(rc)) + RTR0MemObjFree(hMemObj, true /*fFreeMappings*/); + } + + rtR0MemBlockFree(pBlock); + return NULL; +} + + +/** + * Internal free. + */ +static void rtR0MemFree(const char *pszOp, RTMEMTYPE enmType, void *pv, size_t cbUser, void *pvCaller, RT_SRC_POS_DECL) +{ + NOREF(enmType); RT_SRC_POS_NOREF(); + + /* + * Simple case. + */ + if (!pv) + return; + + /* + * Check watch points. + */ + for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++) + if (gapvRTMemFreeWatch[i] == pv) + RTAssertDoPanic(); + + /* + * Find the block. + */ + PRTR0MEMEFBLOCK pBlock = rtR0MemBlockRemove(pv); + if (pBlock) + { + if (gfRTMemFreeLog) + RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned); + +#ifdef RTR0MEM_EF_NOMAN_FILLER + /* + * Check whether the no man's land is untouched. + */ +# ifdef RTR0MEM_EF_IN_FRONT + void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned, + RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned, + RTR0MEM_EF_NOMAN_FILLER); +# else + /* Alignment must match allocation alignment in rtMemAlloc(). */ + void *pvWrong = ASMMemFirstMismatchingU8((char *)pv + pBlock->cbUnaligned, + pBlock->cbAligned - pBlock->cbUnaligned, + RTR0MEM_EF_NOMAN_FILLER); + if (pvWrong) + RTAssertDoPanic(); + pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK), + RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned, + RTR0MEM_EF_NOMAN_FILLER); +# endif + if (pvWrong) + RTAssertDoPanic(); +#endif + + /* + * Fill the user part of the block. + */ + AssertMsg(enmType != RTMEMTYPE_RTMEMFREEZ || cbUser == pBlock->cbUnaligned, + ("cbUser=%#zx cbUnaligned=%#zx\n", cbUser, pBlock->cbUnaligned)); + RT_NOREF(cbUser); + if (enmType == RTMEMTYPE_RTMEMFREEZ) + RT_BZERO(pv, pBlock->cbUnaligned); +#ifdef RTR0MEM_EF_FREE_FILL + else + memset(pv, RTR0MEM_EF_FREE_FILL, pBlock->cbUnaligned); +#endif + +#if defined(RTR0MEM_EF_FREE_DELAYED) && RTR0MEM_EF_FREE_DELAYED > 0 + /* + * We're doing delayed freeing. + * That means we'll expand the E-fence to cover the entire block. + */ + int rc = RTR0MemObjProtect(pBlock->hMemObj, +# ifdef RTR0MEM_EF_IN_FRONT + RTR0MEM_EF_SIZE, +# else + 0 /*offSub*/, +# endif + RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE), + RTMEM_PROT_NONE); + if (RT_SUCCESS(rc)) + { + /* + * Insert it into the free list and process pending frees. + */ + rtR0MemBlockDelayInsert(pBlock); + while ((pBlock = rtR0MemBlockDelayRemove()) != NULL) + rtR0MemFreeBlock(pBlock, pszOp); + } + else + rtR0MemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc); + +#else /* !RTR0MEM_EF_FREE_DELAYED */ + rtR0MemFreeBlock(pBlock, pszOp); +#endif /* !RTR0MEM_EF_FREE_DELAYED */ + } + else + rtR0MemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv); +} + + +/** + * Internal realloc. + */ +static void *rtR0MemRealloc(const char *pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew, + const char *pszTag, void *pvCaller, RT_SRC_POS_DECL) +{ + /* + * Allocate new and copy. + */ + if (!pvOld) + return rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS); + if (!cbNew) + { + rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS); + return NULL; + } + + /* + * Get the block, allocate the new, copy the data, free the old one. + */ + PRTR0MEMEFBLOCK pBlock = rtR0MemBlockGet(pvOld); + if (pBlock) + { + void *pvRet = rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS); + if (pvRet) + { + memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned)); + rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS); + } + return pvRet; + } + rtR0MemComplain(pszOp, "pvOld=%p was not found!\n", pvOld); + return NULL; +} + + + + +RTDECL(void *) RTMemEfTmpAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfTmpAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void) RTMemEfTmpFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfAlloc(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfAllocZ(size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfAllocVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + size_t cbAligned; + if (cbUnaligned >= 16) + cbAligned = RT_ALIGN_Z(cbUnaligned, 16); + else + cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *)); + return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfAllocZVar(size_t cbUnaligned, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + size_t cbAligned; + if (cbUnaligned >= 16) + cbAligned = RT_ALIGN_Z(cbUnaligned, 16); + else + cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *)); + return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfRealloc(void *pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + +RTDECL(void *) RTMemEfReallocZ(void *pvOld, size_t cbOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + void *pvDst = rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS); + if (pvDst && cbNew > cbOld) + memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld); + return pvDst; +} + + +RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void) RTMemEfFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("Free", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS); +} + + +RTDECL(void *) RTMemEfDup(const void *pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS); + if (pvDst) + memcpy(pvDst, pvSrc, cb); + return pvDst; +} + + +RTDECL(void *) RTMemEfDupEx(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF +{ + void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS); + if (pvDst) + { + memcpy(pvDst, pvSrc, cbSrc); + memset((uint8_t *)pvDst + cbSrc, 0, cbExtra); + } + return pvDst; +} + + + + +/* + * + * The NP (no position) versions. + * + */ + + + +RTDECL(void *) RTMemEfTmpAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfTmpAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void) RTMemEfTmpFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfAllocNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfAllocZNP(size_t cb, const char *pszTag) RT_NO_THROW_DEF +{ + return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfAllocVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF +{ + size_t cbAligned; + if (cbUnaligned >= 16) + cbAligned = RT_ALIGN_Z(cbUnaligned, 16); + else + cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *)); + return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfAllocZVarNP(size_t cbUnaligned, const char *pszTag) RT_NO_THROW_DEF +{ + size_t cbAligned; + if (cbUnaligned >= 16) + cbAligned = RT_ALIGN_Z(cbUnaligned, 16); + else + cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *)); + return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfReallocNP(void *pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF +{ + return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfReallocZNP(void *pvOld, size_t cbOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF +{ + void *pvDst = rtR0MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL); + if (pvDst && cbNew > cbOld) + memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld); + return pvDst; +} + + +RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void) RTMemEfFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF +{ + if (pv) + rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL); +} + + +RTDECL(void *) RTMemEfDupNP(const void *pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF +{ + void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL); + if (pvDst) + memcpy(pvDst, pvSrc, cb); + return pvDst; +} + + +RTDECL(void *) RTMemEfDupExNP(const void *pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF +{ + void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL); + if (pvDst) + { + memcpy(pvDst, pvSrc, cbSrc); + memset((uint8_t *)pvDst + cbSrc, 0, cbExtra); + } + return pvDst; +} + |