diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-06 03:01:46 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-06 03:01:46 +0000 |
commit | f8fe689a81f906d1b91bb3220acde2a4ecb14c5b (patch) | |
tree | 26484e9d7e2c67806c2d1760196ff01aaa858e8c /src/VBox/VMM/VMMR3/MMPagePool.cpp | |
parent | Initial commit. (diff) | |
download | virtualbox-f8fe689a81f906d1b91bb3220acde2a4ecb14c5b.tar.xz virtualbox-f8fe689a81f906d1b91bb3220acde2a4ecb14c5b.zip |
Adding upstream version 6.0.4-dfsg.upstream/6.0.4-dfsgupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/VMM/VMMR3/MMPagePool.cpp')
-rw-r--r-- | src/VBox/VMM/VMMR3/MMPagePool.cpp | 527 |
1 files changed, 527 insertions, 0 deletions
diff --git a/src/VBox/VMM/VMMR3/MMPagePool.cpp b/src/VBox/VMM/VMMR3/MMPagePool.cpp new file mode 100644 index 00000000..5364a3cd --- /dev/null +++ b/src/VBox/VMM/VMMR3/MMPagePool.cpp @@ -0,0 +1,527 @@ +/* $Id: MMPagePool.cpp $ */ +/** @file + * MM - Memory Manager - Page Pool. + */ + +/* + * Copyright (C) 2006-2019 Oracle Corporation + * + * This file is part of VirtualBox Open Source Edition (OSE), as + * available from http://www.virtualbox.org. This file is free software; + * you can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) as published by the Free Software + * Foundation, in version 2 as it comes in the "COPYING" file of the + * VirtualBox OSE distribution. VirtualBox OSE is distributed in the + * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_MM_POOL +#include <VBox/vmm/mm.h> +#include <VBox/vmm/pgm.h> +#include <VBox/vmm/stam.h> +#include "MMInternal.h" +#include <VBox/vmm/vm.h> +#include <VBox/param.h> +#include <iprt/errcore.h> +#include <VBox/log.h> +#include <iprt/alloc.h> +#include <iprt/assert.h> +#define USE_INLINE_ASM_BIT_OPS +#ifdef USE_INLINE_ASM_BIT_OPS +# include <iprt/asm.h> +#endif +#include <iprt/string.h> + + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +#ifdef IN_RING3 +static void * mmR3PagePoolAlloc(PMMPAGEPOOL pPool); +static void mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv); +#endif + + +/** + * Initializes the page pool + * + * @return VBox status code. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +int mmR3PagePoolInit(PVM pVM) +{ + AssertMsg(!pVM->mm.s.pPagePoolR3, ("Already initialized!\n")); + + /* + * Allocate the pool structures. + */ + /** @todo @bugref{1865},@bugref{3202}: mapping the page pool page into ring-0. + * Need to change the ways we allocate it... */ + AssertReleaseReturn(sizeof(*pVM->mm.s.pPagePoolR3) + sizeof(*pVM->mm.s.pPagePoolLowR3) < PAGE_SIZE, VERR_INTERNAL_ERROR); + int rc = SUPR3PageAllocEx(1, 0 /*fFlags*/, (void **)&pVM->mm.s.pPagePoolR3, NULL /*pR0Ptr*/, NULL /*paPages*/); + if (RT_FAILURE(rc)) + return rc; + memset(pVM->mm.s.pPagePoolR3, 0, PAGE_SIZE); + pVM->mm.s.pPagePoolR3->pVM = pVM; + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cPages, STAMTYPE_U32, "/MM/Page/Def/cPages", STAMUNIT_PAGES, "Number of pages in the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreePages, STAMTYPE_U32, "/MM/Page/Def/cFreePages", STAMUNIT_PAGES, "Number of free pages in the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cSubPools, STAMTYPE_U32, "/MM/Page/Def/cSubPools", STAMUNIT_COUNT, "Number of sub pools in the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cAllocCalls", STAMUNIT_CALLS, "Number of MMR3PageAlloc() calls for the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreeCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cFreeCalls", STAMUNIT_CALLS, "Number of MMR3PageFree()+MMR3PageFreeByPhys() calls for the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToPhysCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cToPhysCalls", STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for this pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToVirtCalls, STAMTYPE_COUNTER, "/MM/Page/Def/cToVirtCalls", STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the default pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cErrors, STAMTYPE_COUNTER, "/MM/Page/Def/cErrors", STAMUNIT_ERRORS,"Number of errors for the default pool."); + + pVM->mm.s.pPagePoolLowR3 = pVM->mm.s.pPagePoolR3 + 1; + pVM->mm.s.pPagePoolLowR3->pVM = pVM; + pVM->mm.s.pPagePoolLowR3->fLow = true; + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cPages, STAMTYPE_U32, "/MM/Page/Low/cPages", STAMUNIT_PAGES, "Number of pages in the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreePages, STAMTYPE_U32, "/MM/Page/Low/cFreePages", STAMUNIT_PAGES, "Number of free pages in the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cSubPools, STAMTYPE_U32, "/MM/Page/Low/cSubPools", STAMUNIT_COUNT, "Number of sub pools in the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cAllocCalls", STAMUNIT_CALLS, "Number of MMR3PageAllocLow() calls for the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreeCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cFreeCalls", STAMUNIT_CALLS, "Number of MMR3PageFreeLow()+MMR3PageFreeByPhys() calls for the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToPhysCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToPhysCalls", STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToVirtCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToVirtCalls", STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the <4GB pool."); + STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cErrors, STAMTYPE_COUNTER, "/MM/Page/Low/cErrors", STAMUNIT_ERRORS,"Number of errors for the <4GB pool."); + +#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE + pVM->mm.s.pPagePoolR0 = (uintptr_t)pVM->mm.s.pPagePoolR3; + pVM->mm.s.pPagePoolLowR0 = (uintptr_t)pVM->mm.s.pPagePoolLowR3; +#endif + + /** @todo init a mutex? */ + return VINF_SUCCESS; +} + + +/** + * Release all locks and free the allocated memory. + * + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +void mmR3PagePoolTerm(PVM pVM) +{ + if (pVM->mm.s.pPagePoolR3) + { + /* + * Unlock all memory held by subpools and free the memory. + * (The MM Heap will free the memory used for internal stuff.) + */ + Assert(!pVM->mm.s.pPagePoolR3->fLow); + PMMPAGESUBPOOL pSubPool = pVM->mm.s.pPagePoolR3->pHead; + while (pSubPool) + { + int rc = SUPR3PageFreeEx(pSubPool->pvPages, pSubPool->cPages); + AssertMsgRC(rc, ("SUPR3PageFreeEx(%p) failed with rc=%Rrc\n", pSubPool->pvPages, rc)); + pSubPool->pvPages = NULL; + + /* next */ + pSubPool = pSubPool->pNext; + } + pVM->mm.s.pPagePoolR3 = NULL; +#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE + pVM->mm.s.pPagePoolR0 = NIL_RTR0PTR; +#endif + } + + if (pVM->mm.s.pPagePoolLowR3) + { + /* + * Free the memory. + */ + Assert(pVM->mm.s.pPagePoolLowR3->fLow); + PMMPAGESUBPOOL pSubPool = pVM->mm.s.pPagePoolLowR3->pHead; + while (pSubPool) + { + int rc = SUPR3LowFree(pSubPool->pvPages, pSubPool->cPages); + AssertMsgRC(rc, ("SUPR3LowFree(%p) failed with rc=%d\n", pSubPool->pvPages, rc)); + pSubPool->pvPages = NULL; + + /* next */ + pSubPool = pSubPool->pNext; + } + pVM->mm.s.pPagePoolLowR3 = NULL; +#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE + pVM->mm.s.pPagePoolLowR0 = NIL_RTR0PTR; +#endif + } +} + + +/** + * Allocates a page from the page pool. + * + * @returns Pointer to allocated page(s). + * @returns NULL on failure. + * @param pPool Pointer to the page pool. + * @thread The Emulation Thread. + */ +DECLINLINE(void *) mmR3PagePoolAlloc(PMMPAGEPOOL pPool) +{ + VM_ASSERT_EMT(pPool->pVM); + STAM_COUNTER_INC(&pPool->cAllocCalls); + + /* + * Walk free list. + */ + if (pPool->pHeadFree) + { + PMMPAGESUBPOOL pSub = pPool->pHeadFree; + /* decrement free count and unlink if no more free entries. */ + if (!--pSub->cPagesFree) + pPool->pHeadFree = pSub->pNextFree; +#ifdef VBOX_WITH_STATISTICS + pPool->cFreePages--; +#endif + + /* find free spot in bitmap. */ +#ifdef USE_INLINE_ASM_BIT_OPS + const int iPage = ASMBitFirstClear(pSub->auBitmap, pSub->cPages); + if (iPage >= 0) + { + Assert(!ASMBitTest(pSub->auBitmap, iPage)); + ASMBitSet(pSub->auBitmap, iPage); + return (uint8_t *)pSub->pvPages + PAGE_SIZE * iPage; + } +#else + unsigned *pu = &pSub->auBitmap[0]; + unsigned *puEnd = &pSub->auBitmap[pSub->cPages / (sizeof(pSub->auBitmap) * 8)]; + while (pu < puEnd) + { + unsigned u; + if ((u = *pu) != ~0U) + { + unsigned iBit = 0; + unsigned uMask = 1; + while (iBit < sizeof(pSub->auBitmap[0]) * 8) + { + if (!(u & uMask)) + { + *pu |= uMask; + return (uint8_t *)pSub->pvPages + + PAGE_SIZE * (iBit + ((uint8_t *)pu - (uint8_t *)&pSub->auBitmap[0]) * 8); + } + iBit++; + uMask <<= 1; + } + STAM_COUNTER_INC(&pPool->cErrors); + AssertMsgFailed(("how odd, expected to find a free bit in %#x, but didn't\n", u)); + } + /* next */ + pu++; + } +#endif + STAM_COUNTER_INC(&pPool->cErrors); +#ifdef VBOX_WITH_STATISTICS + pPool->cFreePages++; +#endif + AssertMsgFailed(("how strange, expected to find a free bit in %p, but didn't (%d pages supposed to be free!)\n", pSub, pSub->cPagesFree + 1)); + } + + /* + * Allocate new subpool. + */ + unsigned cPages = !pPool->fLow ? 128 : 32; + PMMPAGESUBPOOL pSub; + int rc = MMHyperAlloc(pPool->pVM, + RT_UOFFSETOF_DYN(MMPAGESUBPOOL, auBitmap[cPages / (sizeof(pSub->auBitmap[0]) * 8)]) + + (sizeof(SUPPAGE) + sizeof(MMPPLOOKUPHCPHYS)) * cPages + + sizeof(MMPPLOOKUPHCPTR), + 0, + MM_TAG_MM_PAGE, + (void **)&pSub); + if (RT_FAILURE(rc)) + return NULL; + + PSUPPAGE paPhysPages = (PSUPPAGE)&pSub->auBitmap[cPages / (sizeof(pSub->auBitmap[0]) * 8)]; + Assert((uintptr_t)paPhysPages >= (uintptr_t)&pSub->auBitmap[1]); + if (!pPool->fLow) + { + rc = SUPR3PageAllocEx(cPages, + 0 /* fFlags */, + &pSub->pvPages, + NULL, + paPhysPages); + if (RT_FAILURE(rc)) + rc = VMSetError(pPool->pVM, rc, RT_SRC_POS, + N_("Failed to lock host %zd bytes of memory (out of memory)"), (size_t)cPages << PAGE_SHIFT); + } + else + rc = SUPR3LowAlloc(cPages, &pSub->pvPages, NULL, paPhysPages); + if (RT_SUCCESS(rc)) + { + /* + * Setup the sub structure and allocate the requested page. + */ + pSub->cPages = cPages; + pSub->cPagesFree= cPages - 1; + pSub->paPhysPages = paPhysPages; + memset(pSub->auBitmap, 0, cPages / 8); + /* allocate first page. */ + pSub->auBitmap[0] |= 1; + /* link into free chain. */ + pSub->pNextFree = pPool->pHeadFree; + pPool->pHeadFree= pSub; + /* link into main chain. */ + pSub->pNext = pPool->pHead; + pPool->pHead = pSub; + /* update pool statistics. */ + pPool->cSubPools++; + pPool->cPages += cPages; +#ifdef VBOX_WITH_STATISTICS + pPool->cFreePages += cPages - 1; +#endif + + /* + * Initialize the physical pages with backpointer to subpool. + */ + unsigned i = cPages; + while (i-- > 0) + { + AssertMsg(paPhysPages[i].Phys && !(paPhysPages[i].Phys & PAGE_OFFSET_MASK), + ("i=%d Phys=%d\n", i, paPhysPages[i].Phys)); + paPhysPages[i].uReserved = (RTHCUINTPTR)pSub; + } + + /* + * Initialize the physical lookup record with backpointers to the physical pages. + */ + PMMPPLOOKUPHCPHYS paLookupPhys = (PMMPPLOOKUPHCPHYS)&paPhysPages[cPages]; + i = cPages; + while (i-- > 0) + { + paLookupPhys[i].pPhysPage = &paPhysPages[i]; + paLookupPhys[i].Core.Key = paPhysPages[i].Phys; + RTAvlHCPhysInsert(&pPool->pLookupPhys, &paLookupPhys[i].Core); + } + + /* + * And the one record for virtual memory lookup. + */ + PMMPPLOOKUPHCPTR pLookupVirt = (PMMPPLOOKUPHCPTR)&paLookupPhys[cPages]; + pLookupVirt->pSubPool = pSub; + pLookupVirt->Core.Key = pSub->pvPages; + RTAvlPVInsert(&pPool->pLookupVirt, &pLookupVirt->Core); + + /* return allocated page (first). */ + return pSub->pvPages; + } + + MMHyperFree(pPool->pVM, pSub); + STAM_COUNTER_INC(&pPool->cErrors); + if (pPool->fLow) + VMSetError(pPool->pVM, rc, RT_SRC_POS, + N_("Failed to expand page pool for memory below 4GB. Current size: %d pages"), + pPool->cPages); + AssertMsgFailed(("Failed to expand pool%s. rc=%Rrc poolsize=%d\n", + pPool->fLow ? " (<4GB)" : "", rc, pPool->cPages)); + return NULL; +} + + +/** + * Frees a page from the page pool. + * + * @param pPool Pointer to the page pool. + * @param pv Pointer to the page to free. + * I.e. pointer returned by mmR3PagePoolAlloc(). + * @thread The Emulation Thread. + */ +DECLINLINE(void) mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv) +{ + VM_ASSERT_EMT(pPool->pVM); + STAM_COUNTER_INC(&pPool->cFreeCalls); + + /* + * Lookup the virtual address. + */ + PMMPPLOOKUPHCPTR pLookup = (PMMPPLOOKUPHCPTR)RTAvlPVGetBestFit(&pPool->pLookupVirt, pv, false); + if ( !pLookup + || (uint8_t *)pv >= (uint8_t *)pLookup->pSubPool->pvPages + (pLookup->pSubPool->cPages << PAGE_SHIFT) + ) + { + STAM_COUNTER_INC(&pPool->cErrors); + AssertMsgFailed(("invalid pointer %p\n", pv)); + return; + } + + /* + * Free the page. + */ + PMMPAGESUBPOOL pSubPool = pLookup->pSubPool; + /* clear bitmap bit */ + const unsigned iPage = ((uint8_t *)pv - (uint8_t *)pSubPool->pvPages) >> PAGE_SHIFT; +#ifdef USE_INLINE_ASM_BIT_OPS + Assert(ASMBitTest(pSubPool->auBitmap, iPage)); + ASMBitClear(pSubPool->auBitmap, iPage); +#else + unsigned iBit = iPage % (sizeof(pSubPool->auBitmap[0]) * 8); + unsigned iIndex = iPage / (sizeof(pSubPool->auBitmap[0]) * 8); + pSubPool->auBitmap[iIndex] &= ~(1 << iBit); +#endif + /* update stats. */ + pSubPool->cPagesFree++; +#ifdef VBOX_WITH_STATISTICS + pPool->cFreePages++; +#endif + if (pSubPool->cPagesFree == 1) + { + pSubPool->pNextFree = pPool->pHeadFree; + pPool->pHeadFree = pSubPool; + } +} + + +/** + * Allocates a page from the page pool. + * + * This function may returns pages which has physical addresses any + * where. If you require a page to be within the first 4GB of physical + * memory, use MMR3PageAllocLow(). + * + * @returns Pointer to the allocated page page. + * @returns NULL on failure. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +VMMR3DECL(void *) MMR3PageAlloc(PVM pVM) +{ + /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */ + return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3); +} + + +/** + * Allocates a page from the page pool and return its physical address. + * + * This function may returns pages which has physical addresses any + * where. If you require a page to be within the first 4GB of physical + * memory, use MMR3PageAllocLow(). + * + * @returns Pointer to the allocated page page. + * @returns NIL_RTHCPHYS on failure. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +VMMR3DECL(RTHCPHYS) MMR3PageAllocPhys(PVM pVM) +{ + /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */ + /** @todo optimize this, it's the most common case now. */ + void *pv = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3); + if (pv) + return mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pv); + return NIL_RTHCPHYS; +} + + +/** + * Frees a page allocated from the page pool by MMR3PageAlloc() or + * MMR3PageAllocPhys(). + * + * @param pVM The cross context VM structure. + * @param pvPage Pointer to the page. + * @thread The Emulation Thread. + */ +VMMR3DECL(void) MMR3PageFree(PVM pVM, void *pvPage) +{ + mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage); +} + + +/** + * Allocates a page from the low page pool. + * + * @returns Pointer to the allocated page. + * @returns NULL on failure. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +VMMR3DECL(void *) MMR3PageAllocLow(PVM pVM) +{ + return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolLowR3); +} + + +/** + * Frees a page allocated from the page pool by MMR3PageAllocLow(). + * + * @param pVM The cross context VM structure. + * @param pvPage Pointer to the page. + * @thread The Emulation Thread. + */ +VMMR3DECL(void) MMR3PageFreeLow(PVM pVM, void *pvPage) +{ + mmR3PagePoolFree(pVM->mm.s.pPagePoolLowR3, pvPage); +} + + +/** + * Free a page allocated from the page pool by physical address. + * This works for pages allocated by MMR3PageAlloc(), MMR3PageAllocPhys() + * and MMR3PageAllocLow(). + * + * @param pVM The cross context VM structure. + * @param HCPhysPage The physical address of the page to be freed. + * @thread The Emulation Thread. + */ +VMMR3DECL(void) MMR3PageFreeByPhys(PVM pVM, RTHCPHYS HCPhysPage) +{ + void *pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolR3, HCPhysPage); + if (!pvPage) + pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolLowR3, HCPhysPage); + if (pvPage) + mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage); + else + AssertMsgFailed(("Invalid address HCPhysPT=%#x\n", HCPhysPage)); +} + + +/** + * Gets the HC pointer to the dummy page. + * + * The dummy page is used as a place holder to prevent potential bugs + * from doing really bad things to the system. + * + * @returns Pointer to the dummy page. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +VMMR3DECL(void *) MMR3PageDummyHCPtr(PVM pVM) +{ + VM_ASSERT_EMT(pVM); + if (!pVM->mm.s.pvDummyPage) + { + pVM->mm.s.pvDummyPage = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3); + AssertRelease(pVM->mm.s.pvDummyPage); + pVM->mm.s.HCPhysDummyPage = mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pVM->mm.s.pvDummyPage); + AssertRelease(!(pVM->mm.s.HCPhysDummyPage & ~X86_PTE_PAE_PG_MASK)); + } + return pVM->mm.s.pvDummyPage; +} + + +/** + * Gets the HC Phys to the dummy page. + * + * The dummy page is used as a place holder to prevent potential bugs + * from doing really bad things to the system. + * + * @returns Pointer to the dummy page. + * @param pVM The cross context VM structure. + * @thread The Emulation Thread. + */ +VMMR3DECL(RTHCPHYS) MMR3PageDummyHCPhys(PVM pVM) +{ + VM_ASSERT_EMT(pVM); + if (!pVM->mm.s.pvDummyPage) + MMR3PageDummyHCPtr(pVM); + return pVM->mm.s.HCPhysDummyPage; +} + |