diff options
Diffstat (limited to 'src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp')
| -rw-r--r-- | src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp | 2692 |
1 files changed, 2692 insertions, 0 deletions
diff --git a/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp b/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp new file mode 100644 index 00000000..0f4313bb --- /dev/null +++ b/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp @@ -0,0 +1,2692 @@ +/* $Id: PGMRZDynMap.cpp $ */ +/** @file + * PGM - Page Manager and Monitor, dynamic mapping cache. + */ + +/* + * Copyright (C) 2008-2020 Oracle Corporation + * + * This file is part of VirtualBox Open Source Edition (OSE), as + * available from http://www.virtualbox.org. This file is free software; + * you can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) as published by the Free Software + * Foundation, in version 2 as it comes in the "COPYING" file of the + * VirtualBox OSE distribution. VirtualBox OSE is distributed in the + * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. + */ + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PGM_DYNMAP +#include <VBox/vmm/pgm.h> +#include "PGMInternal.h" +#include <VBox/vmm/vm.h> +#include "PGMInline.h" +#include <VBox/err.h> +#include <VBox/param.h> +#include <VBox/sup.h> +#include <iprt/asm.h> +#include <iprt/asm-amd64-x86.h> +#include <iprt/assert.h> +#ifndef IN_RC +# include <iprt/cpuset.h> +# include <iprt/mem.h> +# include <iprt/memobj.h> +# include <iprt/mp.h> +# include <iprt/semaphore.h> +# include <iprt/spinlock.h> +#endif +#include <iprt/string.h> + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#ifdef IN_RING0 +/** The max size of the mapping cache (in pages). */ +# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT) +/** The small segment size that is adopted on out-of-memory conditions with a + * single big segment. */ +# define PGMR0DYNMAP_SMALL_SEG_PAGES 128 +/** The number of pages we reserve per CPU. */ +# define PGMR0DYNMAP_PAGES_PER_CPU 256 +/** The minimum number of pages we reserve per CPU. + * This must be equal or larger than the autoset size. */ +# define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64 +/** Calcs the overload threshold (safety margin). Current set at 50%. */ +# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2) +/** The number of guard pages. + * @remarks Never do tuning of the hashing or whatnot with a strict build! */ +# if defined(VBOX_STRICT) +# define PGMR0DYNMAP_GUARD_PAGES 1 +# else +# define PGMR0DYNMAP_GUARD_PAGES 0 +# endif +#endif /* IN_RING0 */ +/** The dummy physical address of guard pages. */ +#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed) +/** The dummy reference count of guard pages. (Must be non-zero.) */ +#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed) +#if 0 +/** Define this to just clear the present bit on guard pages. + * The alternative is to replace the entire PTE with an bad not-present + * PTE. Either way, XNU will screw us. :-/ */ +# define PGMR0DYNMAP_GUARD_NP +#endif +/** The dummy PTE value for a page. */ +#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK +/** The dummy PTE value for a page. */ +#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /*X86_PTE_PAE_PG_MASK*/ + +#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled, + * disable this to work around that. */ +/** + * Acquire the spinlock. + * This will declare a temporary variable and expands to two statements! + */ +# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \ + RTSpinlockAcquire((pThis)->hSpinlock) + +/** + * Releases the spinlock. + */ +# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \ + RTSpinlockRelease((pThis)->hSpinlock) + +/** + * Re-acquires the spinlock. + */ +# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \ + RTSpinlockAcquire((pThis)->hSpinlock) +#else +# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0) +# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0) +# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0) +#endif + + +/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */ +#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet)) + +/** Converts a PGMCPUM::AutoSet pointer into a PVM. */ +#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM)) + +/** Converts a PGMCPUM::AutoSet pointer into a PVM. */ +#ifdef IN_RC +# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap) +#else +# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap) +#endif + +/** + * Gets the set index of the current CPU. + * + * This always returns 0 when in raw-mode context because there is only ever + * one EMT in that context (at least presently). + */ +#ifdef IN_RC +# define PGMRZDYNMAP_CUR_CPU() (0) +#else +# define PGMRZDYNMAP_CUR_CPU() RTMpCurSetIndex() +#endif + +/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */ +#define PGMRZDYNMAP_MAGIC UINT32_C(0x19640201) + + +/** Zaps an set entry. */ +#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \ + do \ + { \ + (pEntry)->iPage = UINT16_MAX; \ + (pEntry)->cRefs = 0; \ + (pEntry)->cInlinedRefs = 0; \ + (pEntry)->cUnrefs = 0; \ + } while (0) + + +/** @def PGMRZDYNMAP_STRICT_RELEASE + * Define this to force pages to be released and make non-present ASAP after + * use. This should not normally be enabled as it is a bit expensive. */ +#if 0 || defined(DOXYGEN_RUNNING) +# define PGMRZDYNMAP_STRICT_RELEASE +#endif + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +#ifdef IN_RING0 +/** + * Ring-0 dynamic mapping cache segment. + * + * The dynamic mapping cache can be extended with additional segments if the + * load is found to be too high. This done the next time a VM is created, under + * the protection of the init mutex. The arrays is reallocated and the new + * segment is added to the end of these. Nothing is rehashed of course, as the + * indexes / addresses must remain unchanged. + * + * This structure is only modified while owning the init mutex or during module + * init / term. + */ +typedef struct PGMR0DYNMAPSEG +{ + /** Pointer to the next segment. */ + struct PGMR0DYNMAPSEG *pNext; + /** The memory object for the virtual address range that we're abusing. */ + RTR0MEMOBJ hMemObj; + /** The start page in the cache. (I.e. index into the arrays.) */ + uint16_t iPage; + /** The number of pages this segment contributes. */ + uint16_t cPages; + /** The number of page tables. */ + uint16_t cPTs; + /** The memory objects for the page tables. */ + RTR0MEMOBJ ahMemObjPTs[1]; +} PGMR0DYNMAPSEG; +/** Pointer to a ring-0 dynamic mapping cache segment. */ +typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG; + + +/** + * Ring-0 dynamic mapping cache entry. + * + * @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY. + */ +typedef struct PGMR0DYNMAPENTRY +{ + /** The physical address of the currently mapped page. + * This is duplicate for three reasons: cache locality, cache policy of the PT + * mappings and sanity checks. */ + RTHCPHYS HCPhys; + /** Pointer to the page. */ + void *pvPage; + /** The number of references. */ + int32_t volatile cRefs; + /** PTE pointer union. */ + union PGMR0DYNMAPENTRY_PPTE + { + /** PTE pointer, 32-bit legacy version. */ + PX86PTE pLegacy; + /** PTE pointer, PAE version. */ + PX86PTEPAE pPae; + /** PTE pointer, the void version. */ + void *pv; + } uPte; + /** CPUs that haven't invalidated this entry after it's last update. */ + RTCPUSET PendingSet; +} PGMR0DYNMAPENTRY; +/** Pointer a mapping cache entry for the ring-0. + * @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */ +typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY; + + +/** + * Dynamic mapping cache for ring-0. + * + * This is initialized during VMMR0 module init but no segments are allocated + * at that time. Segments will be added when the first VM is started and + * removed again when the last VM shuts down, thus avoid consuming memory while + * dormant. At module termination, the remaining bits will be freed up. + * + * @sa PPGMRZDYNMAP, PGMRCDYNMAP. + */ +typedef struct PGMR0DYNMAP +{ + /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */ + uint32_t u32Magic; + /** Spinlock serializing the normal operation of the cache. */ + RTSPINLOCK hSpinlock; + /** Array for tracking and managing the pages. */ + PPGMR0DYNMAPENTRY paPages; + /** The cache size given as a number of pages. */ + uint32_t cPages; + /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */ + bool fLegacyMode; + /** The current load. + * This does not include guard pages. */ + uint32_t cLoad; + /** The max load ever. + * This is maintained to trigger the adding of more mapping space. */ + uint32_t cMaxLoad; + /** Initialization / termination lock. */ + RTSEMFASTMUTEX hInitLock; + /** The number of guard pages. */ + uint32_t cGuardPages; + /** The number of users (protected by hInitLock). */ + uint32_t cUsers; + /** Array containing a copy of the original page tables. + * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */ + void *pvSavedPTEs; + /** List of segments. */ + PPGMR0DYNMAPSEG pSegHead; + /** The paging mode. */ + SUPPAGINGMODE enmPgMode; +} PGMR0DYNMAP; + + +/** + * Paging level data. + */ +typedef struct PGMR0DYNMAPPGLVL +{ + uint32_t cLevels; /**< The number of levels. */ + struct + { + RTHCPHYS HCPhys; /**< The address of the page for the current level, + * i.e. what hMemObj/hMapObj is currently mapping. */ + RTHCPHYS fPhysMask; /**< Mask for extracting HCPhys from uEntry. */ + RTR0MEMOBJ hMemObj; /**< Memory object for HCPhys, PAGE_SIZE. */ + RTR0MEMOBJ hMapObj; /**< Mapping object for hMemObj. */ + uint32_t fPtrShift; /**< The pointer shift count. */ + uint64_t fPtrMask; /**< The mask to apply to the shifted pointer to get the table index. */ + uint64_t fAndMask; /**< And mask to check entry flags. */ + uint64_t fResMask; /**< The result from applying fAndMask. */ + union + { + void *pv; /**< hMapObj address. */ + PX86PGUINT paLegacy; /**< Legacy table view. */ + PX86PGPAEUINT paPae; /**< PAE/AMD64 table view. */ + } u; + } a[4]; +} PGMR0DYNMAPPGLVL; +/** Pointer to paging level data. */ +typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL; +#endif + +/** Mapping cache entry for the current context. + * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */ +typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY; +/** Pointer a mapping cache entry for the current context. + * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */ +typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY; + +/** Pointer to the mapping cache instance for the current context. + * @sa PGMR0DYNMAP, PGMRCDYNMAP */ +typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP; + + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#ifdef IN_RING0 +/** Pointer to the ring-0 dynamic mapping cache. */ +static PGMR0DYNMAP *g_pPGMR0DynMap; +#endif +/** For overflow testing. */ +static bool g_fPGMR0DynMapTestRunning = false; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs); +#ifdef IN_RING0 +static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis); +static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis); +static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis); +#endif +#if 0 /*def DEBUG*/ +static int pgmR0DynMapTest(PVM pVM); +#endif + + +/** + * Initializes the auto mapping sets for a VM. + * + * @returns VINF_SUCCESS on success, VERR_PGM_DYNMAP_IPE on failure. + * @param pVM The cross context VM structure. + */ +static int pgmRZDynMapInitAutoSetsForVM(PVM pVM) +{ + VMCPUID idCpu = pVM->cCpus; + AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_PGM_DYNMAP_IPE); + while (idCpu-- > 0) + { + PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet; + uint32_t j = RT_ELEMENTS(pSet->aEntries); + while (j-- > 0) + { + pSet->aEntries[j].pvPage = NULL; + pSet->aEntries[j].HCPhys = NIL_RTHCPHYS; + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]); + } + pSet->cEntries = PGMMAPSET_CLOSED; + pSet->iSubset = UINT32_MAX; + pSet->iCpu = -1; + memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable)); + } + + return VINF_SUCCESS; +} + + +#ifdef IN_RING0 + +/** + * Initializes the ring-0 dynamic mapping cache. + * + * @returns VBox status code. + */ +VMMR0DECL(int) PGMR0DynMapInit(void) +{ + Assert(!g_pPGMR0DynMap); + + /* + * Create and initialize the cache instance. + */ + PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis)); + AssertLogRelReturn(pThis, VERR_NO_MEMORY); + int rc = VINF_SUCCESS; + pThis->enmPgMode = SUPR0GetPagingMode(); + switch (pThis->enmPgMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + pThis->fLegacyMode = false; + break; + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + pThis->fLegacyMode = false; + break; + default: + rc = VERR_PGM_DYNMAP_IPE; + break; + } + if (RT_SUCCESS(rc)) + { + rc = RTSemFastMutexCreate(&pThis->hInitLock); + if (RT_SUCCESS(rc)) + { + rc = RTSpinlockCreate(&pThis->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "PGMR0DynMap"); + if (RT_SUCCESS(rc)) + { + pThis->u32Magic = PGMRZDYNMAP_MAGIC; + g_pPGMR0DynMap = pThis; + return VINF_SUCCESS; + } + RTSemFastMutexDestroy(pThis->hInitLock); + } + } + RTMemFree(pThis); + return rc; +} + + +/** + * Terminates the ring-0 dynamic mapping cache. + */ +VMMR0DECL(void) PGMR0DynMapTerm(void) +{ + /* + * Destroy the cache. + * + * There is not supposed to be any races here, the loader should + * make sure about that. So, don't bother locking anything. + * + * The VM objects should all be destroyed by now, so there is no + * dangling users or anything like that to clean up. This routine + * is just a mirror image of PGMR0DynMapInit. + */ + PPGMRZDYNMAP pThis = g_pPGMR0DynMap; + if (pThis) + { + AssertPtr(pThis); + g_pPGMR0DynMap = NULL; + + /* This should *never* happen, but in case it does try not to leak memory. */ + AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages, + ("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n", + pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages)); + if (pThis->paPages) + pgmR0DynMapTearDown(pThis); + + /* Free the associated resources. */ + RTSemFastMutexDestroy(pThis->hInitLock); + pThis->hInitLock = NIL_RTSEMFASTMUTEX; + RTSpinlockDestroy(pThis->hSpinlock); + pThis->hSpinlock = NIL_RTSPINLOCK; + pThis->u32Magic = UINT32_MAX; + RTMemFree(pThis); + } +} + + +/** + * Initializes the dynamic mapping cache for a new VM. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + */ +VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM) +{ + AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER); + + /* + * Initialize the auto sets. + */ + int rc = pgmRZDynMapInitAutoSetsForVM(pVM); + if (RT_FAILURE(rc)) + return rc; + + /* + * Do we need the cache? Skip the last bit if we don't. + */ + if (VM_IS_RAW_MODE_ENABLED(pVM)) + return VINF_SUCCESS; + + /* + * Reference and if necessary setup or expand the cache. + */ + PPGMRZDYNMAP pThis = g_pPGMR0DynMap; + AssertPtrReturn(pThis, VERR_PGM_DYNMAP_IPE); + rc = RTSemFastMutexRequest(pThis->hInitLock); + AssertLogRelRCReturn(rc, rc); + + pThis->cUsers++; + if (pThis->cUsers == 1) + { + rc = pgmR0DynMapSetup(pThis); +#if 0 /*def DEBUG*/ + if (RT_SUCCESS(rc)) + { + rc = pgmR0DynMapTest(pVM); + if (RT_FAILURE(rc)) + pgmR0DynMapTearDown(pThis); + } +#endif + } + else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages)) + rc = pgmR0DynMapExpand(pThis); + if (RT_SUCCESS(rc)) + pVM->pgm.s.pvR0DynMapUsed = pThis; + else + pThis->cUsers--; + + RTSemFastMutexRelease(pThis->hInitLock); + return rc; +} + + +/** + * Terminates the dynamic mapping cache usage for a VM. + * + * @param pVM The cross context VM structure. + */ +VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM) +{ + /* + * Return immediately if we're not using the cache. + */ + if (!pVM->pgm.s.pvR0DynMapUsed) + return; + + PPGMRZDYNMAP pThis = g_pPGMR0DynMap; + AssertPtrReturnVoid(pThis); + + int rc = RTSemFastMutexRequest(pThis->hInitLock); + AssertLogRelRCReturnVoid(rc); + + if (pVM->pgm.s.pvR0DynMapUsed == pThis) + { + pVM->pgm.s.pvR0DynMapUsed = NULL; + +#ifdef VBOX_STRICT + PGMR0DynMapAssertIntegrity(); +#endif + + /* + * Clean up and check the auto sets. + */ + VMCPUID idCpu = pVM->cCpus; + while (idCpu-- > 0) + { + PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet; + uint32_t j = pSet->cEntries; + if (j <= RT_ELEMENTS(pSet->aEntries)) + { + /* + * The set is open, close it. + */ + while (j-- > 0) + { + int32_t cRefs = pSet->aEntries[j].cRefs; + uint32_t iPage = pSet->aEntries[j].iPage; + LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage)); + if (iPage < pThis->cPages && cRefs > 0) + pgmRZDynMapReleasePage(pThis, iPage, cRefs); + else + AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages)); + + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]); + } + pSet->cEntries = PGMMAPSET_CLOSED; + pSet->iSubset = UINT32_MAX; + pSet->iCpu = -1; + } + else + AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j)); + + j = RT_ELEMENTS(pSet->aEntries); + while (j-- > 0) + { + Assert(pSet->aEntries[j].iPage == UINT16_MAX); + Assert(!pSet->aEntries[j].cRefs); + } + } + + /* + * Release our reference to the mapping cache. + */ + Assert(pThis->cUsers > 0); + pThis->cUsers--; + if (!pThis->cUsers) + pgmR0DynMapTearDown(pThis); + } + else + AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis)); + + RTSemFastMutexRelease(pThis->hInitLock); +} + + +/** + * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper. + * + * @param idCpu The current CPU. + * @param pvUser1 The dynamic mapping cache instance. + * @param pvUser2 Unused, NULL. + */ +static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2) +{ + Assert(!pvUser2); + PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)pvUser1; + Assert(pThis == g_pPGMR0DynMap); + PPGMRZDYNMAPENTRY paPages = pThis->paPages; + uint32_t iPage = pThis->cPages; + while (iPage-- > 0) + ASMInvalidatePage((uintptr_t)paPages[iPage].pvPage); +} + + +/** + * Shoot down the TLBs for every single cache entry on all CPUs. + * + * @returns IPRT status code (RTMpOnAll). + * @param pThis The dynamic mapping cache instance. + */ +static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis) +{ + int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL); + AssertRC(rc); + if (RT_FAILURE(rc)) + { + uint32_t iPage = pThis->cPages; + while (iPage-- > 0) + ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage); + } + return rc; +} + + +/** + * Calculate the new cache size based on cMaxLoad statistics. + * + * @returns Number of pages. + * @param pThis The dynamic mapping cache instance. + * @param pcMinPages The minimal size in pages. + */ +static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages) +{ + Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES); + + /* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */ + RTCPUID cCpus = RTMpGetCount(); + AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0); + uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU; + uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN; + + /* adjust against cMaxLoad. */ + AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad)); + if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES) + pThis->cMaxLoad = 0; + + while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages)) + cPages += PGMR0DYNMAP_PAGES_PER_CPU; + + if (pThis->cMaxLoad > cMinPages) + cMinPages = pThis->cMaxLoad; + + /* adjust against max and current size. */ + if (cPages < pThis->cPages) + cPages = pThis->cPages; + cPages *= PGMR0DYNMAP_GUARD_PAGES + 1; + if (cPages > PGMR0DYNMAP_MAX_PAGES) + cPages = PGMR0DYNMAP_MAX_PAGES; + + if (cMinPages < pThis->cPages) + cMinPages = pThis->cPages; + cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1; + if (cMinPages > PGMR0DYNMAP_MAX_PAGES) + cMinPages = PGMR0DYNMAP_MAX_PAGES; + + Assert(cMinPages); + *pcMinPages = cMinPages; + return cPages; +} + + +/** + * Initializes the paging level data. + * + * @param pThis The dynamic mapping cache instance. + * @param pPgLvl The paging level data. + */ +void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl) +{ + RTCCUINTREG cr4 = ASMGetCR4(); + switch (pThis->enmPgMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + pPgLvl->cLevels = 2; + pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK; + pPgLvl->a[0].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0); + pPgLvl->a[0].fResMask = X86_PDE_P | X86_PDE_RW; + pPgLvl->a[0].fPtrMask = X86_PD_MASK; + pPgLvl->a[0].fPtrShift = X86_PD_SHIFT; + + pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK; + pPgLvl->a[1].fAndMask = X86_PTE_P | X86_PTE_RW; + pPgLvl->a[1].fResMask = X86_PTE_P | X86_PTE_RW; + pPgLvl->a[1].fPtrMask = X86_PT_MASK; + pPgLvl->a[1].fPtrShift = X86_PT_SHIFT; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + pPgLvl->cLevels = 3; + pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK; + pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE; + pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT; + pPgLvl->a[0].fAndMask = X86_PDPE_P; + pPgLvl->a[0].fResMask = X86_PDPE_P; + + pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK; + pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK; + pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT; + pPgLvl->a[1].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0); + pPgLvl->a[1].fResMask = X86_PDE_P | X86_PDE_RW; + + pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK; + pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK; + pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT; + pPgLvl->a[2].fAndMask = X86_PTE_P | X86_PTE_RW; + pPgLvl->a[2].fResMask = X86_PTE_P | X86_PTE_RW; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + pPgLvl->cLevels = 4; + pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK; + pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT; + pPgLvl->a[0].fPtrMask = X86_PML4_MASK; + pPgLvl->a[0].fAndMask = X86_PML4E_P | X86_PML4E_RW; + pPgLvl->a[0].fResMask = X86_PML4E_P | X86_PML4E_RW; + + pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK; + pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT; + pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64; + pPgLvl->a[1].fAndMask = X86_PDPE_P | X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */; + pPgLvl->a[1].fResMask = X86_PDPE_P | X86_PDPE_RW; + + pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK; + pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT; + pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK; + pPgLvl->a[2].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0); + pPgLvl->a[2].fResMask = X86_PDE_P | X86_PDE_RW; + + pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK; + pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT; + pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK; + pPgLvl->a[3].fAndMask = X86_PTE_P | X86_PTE_RW; + pPgLvl->a[3].fResMask = X86_PTE_P | X86_PTE_RW; + break; + + default: + AssertFailed(); + pPgLvl->cLevels = 0; + break; + } + + for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */ + { + pPgLvl->a[i].HCPhys = NIL_RTHCPHYS; + pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ; + pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ; + pPgLvl->a[i].u.pv = NULL; + } +} + + +/** + * Maps a PTE. + * + * This will update the segment structure when new PTs are mapped. + * + * It also assumes that we (for paranoid reasons) wish to establish a mapping + * chain from CR3 to the PT that all corresponds to the processor we're + * currently running on, and go about this by running with interrupts disabled + * and restarting from CR3 for every change. + * + * @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had + * to re-enable interrupts. + * @param pThis The dynamic mapping cache instance. + * @param pPgLvl The paging level structure. + * @param pvPage The page. + * @param pSeg The segment. + * @param cMaxPTs The max number of PTs expected in the segment. + * @param ppvPTE Where to store the PTE address. + */ +static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage, + PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE) +{ + Assert(!(ASMGetFlags() & X86_EFL_IF)); + void *pvEntry = NULL; + X86PGPAEUINT uEntry = ASMGetCR3(); + for (uint32_t i = 0; i < pPgLvl->cLevels; i++) + { + RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask; + if (pPgLvl->a[i].HCPhys != HCPhys) + { + /* + * Need to remap this level. + * The final level, the PT, will not be freed since that is what it's all about. + */ + ASMIntEnable(); + if (i + 1 == pPgLvl->cLevels) + AssertReturn(pSeg->cPTs < cMaxPTs, VERR_PGM_DYNMAP_IPE); + else + { + int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2); + pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ; + } + + int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE); + if (RT_SUCCESS(rc)) + { + rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj, + (void *)-1 /* pvFixed */, 0 /* cbAlignment */, + RTMEM_PROT_WRITE | RTMEM_PROT_READ); + if (RT_SUCCESS(rc)) + { + pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj); + AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv)); + pPgLvl->a[i].HCPhys = HCPhys; + if (i + 1 == pPgLvl->cLevels) + pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj; + ASMIntDisable(); + return VINF_TRY_AGAIN; + } + + pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ; + } + else + pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ; + pPgLvl->a[i].HCPhys = NIL_RTHCPHYS; + return rc; + } + + /* + * The next level. + */ + uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask; + if (pThis->fLegacyMode) + { + pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry]; + uEntry = pPgLvl->a[i].u.paLegacy[iEntry]; + } + else + { + pvEntry = &pPgLvl->a[i].u.paPae[iEntry]; + uEntry = pPgLvl->a[i].u.paPae[iEntry]; + } + + if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask) + { + LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n" + "PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n", + i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask, + pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode)); + return VERR_PGM_DYNMAP_IPE; + } + /*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/ + } + + /* made it thru without needing to remap anything. */ + *ppvPTE = pvEntry; + return VINF_SUCCESS; +} + + +/** + * Sets up a guard page. + * + * @param pThis The dynamic mapping cache instance. + * @param pPage The page. + */ +DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage) +{ + memset(pPage->pvPage, 0xfd, PAGE_SIZE); + pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT; + pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS; +#ifdef PGMR0DYNMAP_GUARD_NP + ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P); +#else + if (pThis->fLegacyMode) + ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE); + else + ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE); +#endif + pThis->cGuardPages++; +} + + +/** + * Adds a new segment of the specified size. + * + * @returns VBox status code. + * @param pThis The dynamic mapping cache instance. + * @param cPages The size of the new segment, give as a page count. + */ +static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages) +{ + int rc2; + AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED); + + /* + * Do the array reallocations first. + * (The pages array has to be replaced behind the spinlock of course.) + */ + void *pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * (pThis->cPages + cPages)); + if (!pvSavedPTEs) + return VERR_NO_MEMORY; + pThis->pvSavedPTEs = pvSavedPTEs; + + void *pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) * (pThis->cPages + cPages)); + if (!pvPages) + { + pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages); + if (pvSavedPTEs) + pThis->pvSavedPTEs = pvSavedPTEs; + return VERR_NO_MEMORY; + } + + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + + memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages); + void *pvToFree = pThis->paPages; + pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages; + + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + RTMemFree(pvToFree); + + /* + * Allocate the segment structure and pages of memory, then touch all the pages (paranoia). + */ + uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2; + PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF_DYN(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs])); + if (!pSeg) + return VERR_NO_MEMORY; + pSeg->pNext = NULL; + pSeg->cPages = cPages; + pSeg->iPage = pThis->cPages; + pSeg->cPTs = 0; + int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false); + if (RT_SUCCESS(rc)) + { + uint8_t *pbPage = (uint8_t *)RTR0MemObjAddress(pSeg->hMemObj); + AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage)); + memset(pbPage, 0xfe, cPages << PAGE_SHIFT); + + /* + * Walk thru the pages and set them up with a mapping of their PTE and everything. + */ + ASMIntDisable(); + PGMR0DYNMAPPGLVL PgLvl; + pgmR0DynMapPagingArrayInit(pThis, &PgLvl); + uint32_t const iEndPage = pSeg->iPage + cPages; + for (uint32_t iPage = pSeg->iPage; + iPage < iEndPage; + iPage++, pbPage += PAGE_SIZE) + { + /* Initialize the page data. */ + pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS; + pThis->paPages[iPage].pvPage = pbPage; + pThis->paPages[iPage].cRefs = 0; + pThis->paPages[iPage].uPte.pPae = 0; +#ifndef IN_RC + RTCpuSetFill(&pThis->paPages[iPage].PendingSet); +#endif + + /* Map its page table, retry until we've got a clean run (paranoia). */ + do + rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs, + &pThis->paPages[iPage].uPte.pv); + while (rc == VINF_TRY_AGAIN); + if (RT_FAILURE(rc)) + break; + + /* Save the PTE. */ + if (pThis->fLegacyMode) + ((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u; + else + ((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u; + +#ifdef VBOX_STRICT + /* Check that we've got the right entry. */ + RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage); + RTHCPHYS HCPhysPte = pThis->fLegacyMode + ? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK + : pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK; + if (HCPhysPage != HCPhysPte) + { + LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n", + iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv)); + rc = VERR_PGM_DYNMAP_IPE; + break; + } +#endif + } /* for each page */ + ASMIntEnable(); + + /* cleanup non-PT mappings */ + for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++) + RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */); + + if (RT_SUCCESS(rc)) + { +#if PGMR0DYNMAP_GUARD_PAGES > 0 + /* + * Setup guard pages. + * (Note: TLBs will be shot down later on.) + */ + uint32_t iPage = pSeg->iPage; + while (iPage < iEndPage) + { + for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++) + pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]); + iPage++; /* the guarded page */ + } + + /* Make sure the very last page is a guard page too. */ + iPage = iEndPage - 1; + if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT) + pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]); +#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */ + + /* + * Commit it by adding the segment to the list and updating the page count. + */ + pSeg->pNext = pThis->pSegHead; + pThis->pSegHead = pSeg; + pThis->cPages += cPages; + return VINF_SUCCESS; + } + + /* + * Bail out. + */ + while (pSeg->cPTs-- > 0) + { + rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */); + AssertRC(rc2); + pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ; + } + + rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); + AssertRC(rc2); + pSeg->hMemObj = NIL_RTR0MEMOBJ; + } + else if (rc == VERR_NO_PAGE_MEMORY || rc == VERR_NO_PHYS_MEMORY) + rc = VERR_NO_MEMORY; + RTMemFree(pSeg); + + /* Don't bother resizing the arrays, but free them if we're the only user. */ + if (!pThis->cPages) + { + RTMemFree(pThis->paPages); + pThis->paPages = NULL; + RTMemFree(pThis->pvSavedPTEs); + pThis->pvSavedPTEs = NULL; + } + return rc; +} + + +/** + * Called by PGMR0DynMapInitVM under the init lock. + * + * @returns VBox status code. + * @param pThis The dynamic mapping cache instance. + */ +static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis) +{ + /* + * Calc the size and add a segment of that size. + */ + uint32_t cMinPages; + uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages); + AssertReturn(cPages, VERR_PGM_DYNMAP_IPE); + int rc = pgmR0DynMapAddSeg(pThis, cPages); + if (rc == VERR_NO_MEMORY) + { + /* + * Try adding smaller segments. + */ + do + rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES); + while (RT_SUCCESS(rc) && pThis->cPages < cPages); + if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages) + rc = VINF_SUCCESS; + if (rc == VERR_NO_MEMORY) + { + if (pThis->cPages) + pgmR0DynMapTearDown(pThis); + rc = VERR_PGM_DYNMAP_SETUP_ERROR; + } + } + Assert(ASMGetFlags() & X86_EFL_IF); + +#if PGMR0DYNMAP_GUARD_PAGES > 0 + /* paranoia */ + if (RT_SUCCESS(rc)) + pgmR0DynMapTlbShootDown(pThis); +#endif + return rc; +} + + +/** + * Called by PGMR0DynMapInitVM under the init lock. + * + * @returns VBox status code. + * @param pThis The dynamic mapping cache instance. + */ +static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis) +{ + /* + * Calc the new target size and add a segment of the appropriate size. + */ + uint32_t cMinPages; + uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages); + AssertReturn(cPages, VERR_PGM_DYNMAP_IPE); + if (pThis->cPages >= cPages) + return VINF_SUCCESS; + + uint32_t cAdd = cPages - pThis->cPages; + int rc = pgmR0DynMapAddSeg(pThis, cAdd); + if (rc == VERR_NO_MEMORY) + { + /* + * Try adding smaller segments. + */ + do + rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES); + while (RT_SUCCESS(rc) && pThis->cPages < cPages); + if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages) + rc = VINF_SUCCESS; + if (rc == VERR_NO_MEMORY) + rc = VERR_PGM_DYNMAP_EXPAND_ERROR; + } + Assert(ASMGetFlags() & X86_EFL_IF); + +#if PGMR0DYNMAP_GUARD_PAGES > 0 + /* paranoia */ + if (RT_SUCCESS(rc)) + pgmR0DynMapTlbShootDown(pThis); +#endif + return rc; +} + + +/** + * Called by PGMR0DynMapTermVM under the init lock. + * + * @returns VBox status code. + * @param pThis The dynamic mapping cache instance. + */ +static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis) +{ + /* + * Restore the original page table entries + */ + PPGMRZDYNMAPENTRY paPages = pThis->paPages; + uint32_t iPage = pThis->cPages; + if (pThis->fLegacyMode) + { + X86PGUINT const *paSavedPTEs = (X86PGUINT const *)pThis->pvSavedPTEs; + while (iPage-- > 0) + { + X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u; + X86PGUINT uOld2 = uOld; NOREF(uOld2); + X86PGUINT uNew = paSavedPTEs[iPage]; + while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld)) + AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew)); + Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]); + } + } + else + { + X86PGPAEUINT const *paSavedPTEs = (X86PGPAEUINT const *)pThis->pvSavedPTEs; + while (iPage-- > 0) + { + X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u; + X86PGPAEUINT uOld2 = uOld; NOREF(uOld2); + X86PGPAEUINT uNew = paSavedPTEs[iPage]; + while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld)) + AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew)); + Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]); + } + } + + /* + * Shoot down the TLBs on all CPUs before freeing them. + */ + pgmR0DynMapTlbShootDown(pThis); + + /* + * Free the segments. + */ + while (pThis->pSegHead) + { + int rc; + PPGMR0DYNMAPSEG pSeg = pThis->pSegHead; + pThis->pSegHead = pSeg->pNext; + + uint32_t iPT = pSeg->cPTs; + while (iPT-- > 0) + { + rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc); + pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ; + } + rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc); + pSeg->hMemObj = NIL_RTR0MEMOBJ; + pSeg->pNext = NULL; + pSeg->iPage = UINT16_MAX; + pSeg->cPages = 0; + pSeg->cPTs = 0; + RTMemFree(pSeg); + } + + /* + * Free the arrays and restore the initial state. + * The cLoadMax value is left behind for the next setup. + */ + RTMemFree(pThis->paPages); + pThis->paPages = NULL; + RTMemFree(pThis->pvSavedPTEs); + pThis->pvSavedPTEs = NULL; + pThis->cPages = 0; + pThis->cLoad = 0; + pThis->cGuardPages = 0; +} + +#endif /* IN_RING0 */ +#ifdef IN_RC + +/** + * Initializes the dynamic mapping cache in raw-mode context. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + */ +VMMRCDECL(int) PGMRCDynMapInit(PVM pVM) +{ + /* + * Allocate and initialize the instance data and page array. + */ + PPGMRZDYNMAP pThis; + size_t const cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE; + size_t const cb = RT_ALIGN_Z(sizeof(*pThis), 32) + + sizeof(PGMRZDYNMAPENTRY) * cPages; + int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis); + if (RT_FAILURE(rc)) + return rc; + + pThis->u32Magic = PGMRZDYNMAP_MAGIC; + pThis->paPages = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY); + pThis->cPages = cPages; + pThis->cLoad = 0; + pThis->cMaxLoad = 0; + pThis->cGuardPages = 0; + pThis->cUsers = 1; + + for (size_t iPage = 0; iPage < cPages; iPage++) + { + pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS; + pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE; + pThis->paPages[iPage].cRefs = 0; + pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage]; + pThis->paPages[iPage].uPte.pPae = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage]; + } + + pVM->pgm.s.pRCDynMap = pThis; + + /* + * Initialize the autosets the VM. + */ + rc = pgmRZDynMapInitAutoSetsForVM(pVM); + if (RT_FAILURE(rc)) + return rc; + + return VINF_SUCCESS; +} + +#endif /* IN_RC */ + +/** + * Release references to a page, caller owns the spin lock. + * + * @param pThis The dynamic mapping cache instance. + * @param iPage The page. + * @param cRefs The number of references to release. + */ +DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs) +{ + cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs; + AssertMsg(cRefs >= 0, ("%d\n", cRefs)); + if (!cRefs) + { + pThis->cLoad--; +#ifdef PGMRZDYNMAP_STRICT_RELEASE + pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS; + ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P); + ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage); +#endif + } +} + + +/** + * Release references to a page, caller does not own the spin lock. + * + * @param pThis The dynamic mapping cache instance. + * @param iPage The page. + * @param cRefs The number of references to release. + */ +static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs) +{ + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs); + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); +} + + +/** + * pgmR0DynMapPage worker that deals with the tedious bits. + * + * @returns The page index on success, UINT32_MAX on failure. + * @param pThis The dynamic mapping cache instance. + * @param HCPhys The address of the page to be mapped. + * @param iPage The page index pgmR0DynMapPage hashed HCPhys to. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * For statistics. + * @param pfNew Set to @c true if a new entry was made and @c false if + * an old entry was found and reused. + */ +static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew) +{ + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow); RT_NOREF_PV(pVCpu); + + /* + * Check if any of the first 3 pages are unreferenced since the caller + * already has made sure they aren't matching. + */ +#ifdef VBOX_WITH_STATISTICS + bool fLooped = false; +#endif + uint32_t const cPages = pThis->cPages; + PPGMRZDYNMAPENTRY paPages = pThis->paPages; + uint32_t iFreePage; + if (!paPages[iPage].cRefs) + iFreePage = iPage; + else if (!paPages[(iPage + 1) % cPages].cRefs) + iFreePage = (iPage + 1) % cPages; + else if (!paPages[(iPage + 2) % cPages].cRefs) + iFreePage = (iPage + 2) % cPages; + else + { + /* + * Search for an unused or matching entry. + */ + iFreePage = (iPage + 3) % cPages; + for (;;) + { + if (paPages[iFreePage].HCPhys == HCPhys) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits); + *pfNew = false; + return iFreePage; + } + if (!paPages[iFreePage].cRefs) + break; + + /* advance */ + iFreePage = (iFreePage + 1) % cPages; + if (RT_UNLIKELY(iFreePage == iPage)) + return UINT32_MAX; + } + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses); +#ifdef VBOX_WITH_STATISTICS + fLooped = true; +#endif + } + Assert(iFreePage < cPages); + +#if 0 //def VBOX_WITH_STATISTICS + /* Check for lost hits. */ + if (!fLooped) + for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages) + if (paPages[iPage2].HCPhys == HCPhys) + STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits); +#endif + + /* + * Setup the new entry. + */ + *pfNew = true; + /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/ + paPages[iFreePage].HCPhys = HCPhys; +#ifndef IN_RC + RTCpuSetFill(&paPages[iFreePage].PendingSet); + + if (pThis->fLegacyMode) +#endif + { + X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u; + X86PGUINT uOld2 = uOld; NOREF(uOld2); + X86PGUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)) + | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D + | (HCPhys & X86_PTE_PG_MASK); + while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld)) + AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew)); + Assert(paPages[iFreePage].uPte.pLegacy->u == uNew); + } +#ifndef IN_RC + else +#endif + { + X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u; + X86PGPAEUINT uOld2 = uOld; NOREF(uOld2); + X86PGPAEUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)) + | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D + | (HCPhys & X86_PTE_PAE_PG_MASK); + while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld)) + AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew)); + Assert(paPages[iFreePage].uPte.pPae->u == uNew); + /*Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));*/ + } + return iFreePage; +} + + +/** + * Maps a page into the pool. + * + * @returns Page index on success, UINT32_MAX on failure. + * @param pThis The dynamic mapping cache instance. + * @param HCPhys The address of the page to be mapped. + * @param iRealCpu The real cpu set index. (optimization) + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. For statistics. + * @param ppvPage Where to the page address. + */ +DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage) +{ + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys)); + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage); + + /* + * Find an entry, if possible a matching one. The HCPhys address is hashed + * down to a page index, collisions are handled by linear searching. + * Optimized for a hit in the first 3 pages. + * + * Field easy hits here and defer the tedious searching and inserting + * to pgmR0DynMapPageSlow(). + */ + bool fNew = false; + uint32_t const cPages = pThis->cPages; + uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages; + PPGMRZDYNMAPENTRY paPages = pThis->paPages; + if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys)) + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0); + else + { + uint32_t iPage2 = (iPage + 1) % cPages; + if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys)) + { + iPage = iPage2; + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1); + } + else + { + iPage2 = (iPage + 2) % cPages; + if (paPages[iPage2].HCPhys == HCPhys) + { + iPage = iPage2; + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2); + } + else + { + iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew); + if (RT_UNLIKELY(iPage == UINT32_MAX)) + { + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + *ppvPage = NULL; + return iPage; + } + } + } + } + + /* + * Reference it, update statistics and get the return address. + */ + int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs); + if (cRefs == 1) + { + pThis->cLoad++; + if (pThis->cLoad > pThis->cMaxLoad) + pThis->cMaxLoad = pThis->cLoad; + AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages)); + } + else if (RT_UNLIKELY(cRefs <= 0)) + { + ASMAtomicDecS32(&paPages[iPage].cRefs); + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + *ppvPage = NULL; + AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%u HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX); + } + void *pvPage = paPages[iPage].pvPage; + +#ifndef IN_RC + /* + * Invalidate the entry? + */ + bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu); + if (RT_UNLIKELY(fInvalidateIt)) + RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu); +#else + NOREF(iRealCpu); +#endif + + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + + /* + * Do the actual invalidation outside the spinlock. + */ +#ifdef IN_RC + if (RT_UNLIKELY(fNew)) +#else + if (RT_UNLIKELY(fInvalidateIt)) +#endif + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg); + ASMInvalidatePage((uintptr_t)pvPage); + } + + *ppvPage = pvPage; + return iPage; +} + + +/** + * Assert the integrity of the pool. + * + * @returns VBox status code. + */ +static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis) +{ + /* + * Basic pool stuff that doesn't require any lock, just assumes we're a user. + */ + if (!pThis) + return VINF_SUCCESS; + AssertPtrReturn(pThis, VERR_INVALID_POINTER); + AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC); + if (!pThis->cUsers) + return VERR_INVALID_PARAMETER; + + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + +#define CHECK_RET(expr, a) \ + do { \ + if (RT_UNLIKELY(!(expr))) \ + { \ + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \ + RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \ + RTAssertMsg2Weak a; \ + return VERR_PGM_DYNMAP_IPE; \ + } \ + } while (0) + + /* + * Check that the PTEs are correct. + */ + uint32_t cGuard = 0; + uint32_t cLoad = 0; + PPGMRZDYNMAPENTRY paPages = pThis->paPages; + +#ifndef IN_RC + if (pThis->fLegacyMode) +#endif + { +#ifdef IN_RING0 + PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs); +#endif + uint32_t iPage = pThis->cPages; + while (iPage-- > 0) + { + CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage)); + if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT + && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS) + { +#ifdef PGMR0DYNMAP_GUARD_NP + CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P), + ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage])); +#else + CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE, + ("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u)); +#endif + cGuard++; + } + else if (paPages[iPage].HCPhys != NIL_RTHCPHYS) + { + CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys)); + X86PGUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D +#ifdef IN_RING0 + | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)) +#endif + | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK); + CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte, + ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte)); + if (paPages[iPage].cRefs) + cLoad++; + } +#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE) + else + CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage], + ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage])); +#endif + } + } +#ifndef IN_RC + else +#endif + { +#ifdef IN_RING0 + PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs); +#endif + uint32_t iPage = pThis->cPages; + while (iPage-- > 0) + { + CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage)); + if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT + && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS) + { +#ifdef PGMR0DYNMAP_GUARD_NP + CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P), + ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage])); +#else + CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE, + ("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u)); +#endif + cGuard++; + } + else if (paPages[iPage].HCPhys != NIL_RTHCPHYS) + { + CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys)); + X86PGPAEUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D +#ifdef IN_RING0 + | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)) +#endif + | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK); + CHECK_RET(paPages[iPage].uPte.pPae->u == uPte, + ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte)); + if (paPages[iPage].cRefs) + cLoad++; + } +#ifdef IN_RING0 + else + CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage], + ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage])); +#endif + } + } + + CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad)); + CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages)); + +#undef CHECK_RET + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + return VINF_SUCCESS; +} + +#ifdef IN_RING0 +/** + * Assert the integrity of the pool. + * + * @returns VBox status code. + */ +VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void) +{ + return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap); +} +#endif /* IN_RING0 */ + +#ifdef IN_RC +/** + * Assert the integrity of the pool. + * + * @returns VBox status code. + */ +VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM) +{ + return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap); +} +#endif /* IN_RC */ + + +/** + * As a final resort for a (somewhat) full auto set or full cache, try merge + * duplicate entries and flush the ones we can. + * + * @param pSet The set. + */ +static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet) +{ + LogFlow(("pgmDynMapOptimizeAutoSet\n")); + + for (uint32_t i = 0 ; i < pSet->cEntries; i++) + { + /* + * Try merge entries. + */ + uint16_t const iPage = pSet->aEntries[i].iPage; + uint32_t j = i + 1; + while ( j < pSet->cEntries + && ( pSet->iSubset == UINT32_MAX + || pSet->iSubset < pSet->cEntries) ) + { + if (pSet->aEntries[j].iPage != iPage) + j++; + else + { + uint32_t const cHardRefs = (uint32_t)pSet->aEntries[i].cRefs + + (uint32_t)pSet->aEntries[j].cRefs; + uint32_t cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs + + (uint32_t)pSet->aEntries[j].cInlinedRefs; + uint32_t cUnrefs = (uint32_t)pSet->aEntries[i].cUnrefs + + (uint32_t)pSet->aEntries[j].cUnrefs; + uint32_t cSub = RT_MIN(cUnrefs, cInlinedRefs); + cInlinedRefs -= cSub; + cUnrefs -= cSub; + + if ( cHardRefs < UINT16_MAX + && cInlinedRefs < UINT16_MAX + && cUnrefs < UINT16_MAX) + { + /* merge j into i removing j. */ + Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i)); + pSet->aEntries[i].cRefs = cHardRefs; + pSet->aEntries[i].cInlinedRefs = cInlinedRefs; + pSet->aEntries[i].cUnrefs = cUnrefs; + pSet->cEntries--; + if (j < pSet->cEntries) + { + pSet->aEntries[j] = pSet->aEntries[pSet->cEntries]; + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]); + } + else + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]); + } +#if 0 /* too complicated, skip it. */ + else + { + /* migrate the max number of refs from j into i and quit the inner loop. */ + uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs; + Assert(pSet->aEntries[j].cRefs > cMigrate); + pSet->aEntries[j].cRefs -= cMigrate; + pSet->aEntries[i].cRefs = UINT16_MAX - 1; + break; + } +#endif + } + } + + /* + * Try make use of the unused hinting (cUnrefs) to evict entries + * from both the set as well as the mapping cache. + */ + + uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs; + Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n", + i, + pSet->iSubset, + pSet->cEntries, + pSet->aEntries[i].pvPage, + pSet->aEntries[i].iPage, + pSet->aEntries[i].cRefs, + pSet->aEntries[i].cInlinedRefs, + pSet->aEntries[i].cUnrefs, + cTotalRefs)); + Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs); + + if ( cTotalRefs == pSet->aEntries[i].cUnrefs + && ( pSet->iSubset == UINT32_MAX + || pSet->iSubset < pSet->cEntries) + ) + { + Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage)); + //LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage)); + pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet), + pSet->aEntries[i].iPage, + pSet->aEntries[i].cRefs); + pSet->cEntries--; + if (i < pSet->cEntries) + { + pSet->aEntries[i] = pSet->aEntries[pSet->cEntries]; + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]); + } + + i--; + } + } +} + + + + +/** + * Signals the start of a new set of mappings. + * + * Mostly for strictness. PGMDynMapHCPage won't work unless this + * API is called. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu) +{ + LogFlow(("PGMRZDynMapStartAutoSet:\n")); + Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED); + Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX); + pVCpu->pgm.s.AutoSet.cEntries = 0; + pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU(); +} + + +#ifdef IN_RING0 +/** + * Starts or migrates the autoset of a virtual CPU. + * + * This is used by HMR0Enter. When we've longjumped out of the HM + * execution loop with the set open, we'll migrate it when re-entering. While + * under normal circumstances, we'll start it so VMXR0LoadGuestState can access + * guest memory. + * + * @returns @c true if started, @c false if migrated. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT + */ +VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu) +{ + bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED; + if (fStartIt) + PGMRZDynMapStartAutoSet(pVCpu); + else + PGMR0DynMapMigrateAutoSet(pVCpu); + return fStartIt; +} +#endif /* IN_RING0 */ + + +/** + * Checks if the set has high load. + * + * @returns true on high load, otherwise false. + * @param pSet The set. + */ +DECLINLINE(bool) pgmRZDynMapHasHighLoad(PPGMMAPSET pSet) +{ +#ifdef IN_RC + if (pSet->cEntries < MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2) + return false; +#endif + + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + uint32_t cUnusedPages = pThis->cPages - pThis->cLoad; +#ifdef IN_RC + return cUnusedPages <= MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE * 36 / 100; +#else + return cUnusedPages <= PGMR0DYNMAP_PAGES_PER_CPU_MIN; +#endif +} + + +/** + * Worker that performs the actual flushing of the set. + * + * @param pSet The set to flush. + * @param cEntries The number of entries. + */ +DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries) +{ + /* + * Release any pages it's referencing. + */ + if ( cEntries != 0 + && RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries))) + { + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + + uint32_t i = cEntries; + while (i-- > 0) + { + uint32_t iPage = pSet->aEntries[i].iPage; + Assert(iPage < pThis->cPages); + int32_t cRefs = pSet->aEntries[i].cRefs; + Assert(cRefs > 0); + pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs); + + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]); + } + + Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages); + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + } +} + + +/** + * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates + * since the PGMDynMapStartAutoSet call. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu) +{ + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + + /* + * Close and flush the set. + */ + uint32_t cEntries = pSet->cEntries; + AssertReturnVoid(cEntries != PGMMAPSET_CLOSED); + pSet->cEntries = PGMMAPSET_CLOSED; + pSet->iSubset = UINT32_MAX; + pSet->iCpu = -1; + +#ifdef IN_RC + if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE) + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]); + else +#endif + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]); + if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100) + Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries)); + else + LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries)); + + pgmDynMapFlushAutoSetWorker(pSet, cEntries); +} + + +/** + * Flushes the set if it's above a certain threshold. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu) +{ + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags())); + + /* + * Only flush it if it's 45% full. + */ + uint32_t cEntries = pSet->cEntries; + AssertReturnVoid(cEntries != PGMMAPSET_CLOSED); + Assert(pSet->iSubset == UINT32_MAX); +#ifdef IN_RC + if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE) + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]); + else +#endif + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]); + if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100 + || pgmRZDynMapHasHighLoad(pSet)) + { + pSet->cEntries = 0; + Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries)); + + pgmDynMapFlushAutoSetWorker(pSet, cEntries); + AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags())); + } +} + + +#ifndef IN_RC +/** + * Migrates the automatic mapping set of the current vCPU if it's active and + * necessary. + * + * This is called when re-entering the hardware assisted execution mode after a + * nip down to ring-3. We run the risk that the CPU might have change and we + * will therefore make sure all the cache entries currently in the auto set will + * be valid on the new CPU. If the cpu didn't change nothing will happen as all + * the entries will have been flagged as invalidated. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT + */ +VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu) +{ + LogFlow(("PGMR0DynMapMigrateAutoSet\n")); + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + int32_t iRealCpu = PGMRZDYNMAP_CUR_CPU(); + if (pSet->iCpu != iRealCpu) + { + uint32_t i = pSet->cEntries; + if (i != PGMMAPSET_CLOSED) + { + AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i)); + if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries))) + { + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + + while (i-- > 0) + { + Assert(pSet->aEntries[i].cRefs > 0); + uint32_t iPage = pSet->aEntries[i].iPage; + Assert(iPage < pThis->cPages); + if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu)) + { + RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu); + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + + ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage); + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg); + + PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis); + } + } + + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + } + } + pSet->iCpu = iRealCpu; + } +} +#endif /* !IN_RC */ + + +/** + * Worker function that flushes the current subset. + * + * This is called when the set is popped or when the set + * hash a too high load. As also pointed out elsewhere, the + * whole subset thing is a hack for working around code that + * accesses too many pages. Like PGMPool. + * + * @param pSet The set which subset to flush. + */ +static void pgmDynMapFlushSubset(PPGMMAPSET pSet) +{ + uint32_t iSubset = pSet->iSubset; + uint32_t i = pSet->cEntries; + Assert(i <= RT_ELEMENTS(pSet->aEntries)); + if ( i > iSubset + && i <= RT_ELEMENTS(pSet->aEntries)) + { + Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset)); + pSet->cEntries = iSubset; + + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis); + + while (i-- > iSubset) + { + uint32_t iPage = pSet->aEntries[i].iPage; + Assert(iPage < pThis->cPages); + int32_t cRefs = pSet->aEntries[i].cRefs; + Assert(cRefs > 0); + pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs); + + PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]); + } + + PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); + } +} + + +/** + * Creates a subset. + * + * A subset is a hack to avoid having to rewrite code that touches a lot of + * pages. It prevents the mapping set from being overflowed by automatically + * flushing previous mappings when a certain threshold is reached. + * + * Pages mapped after calling this function are only valid until the next page + * is mapped. + * + * @returns The index of the previous subset. Pass this to + * PGMDynMapPopAutoSubset when popping it. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu) +{ + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX); + uint32_t iPrevSubset = pSet->iSubset; + LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset)); + + /* + * If it looks like we're approaching the max set size or mapping space + * optimize the set to drop off unused pages. + */ + if ( pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100 + || pgmRZDynMapHasHighLoad(pSet)) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize); + pgmDynMapOptimizeAutoSet(pSet); + } + + pSet->iSubset = pSet->cEntries; + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets); + + AssertMsg(iPrevSubset <= pSet->iSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset)); + return iPrevSubset; +} + + +/** + * Pops a subset created by a previous call to PGMDynMapPushAutoSubset. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param iPrevSubset What PGMDynMapPushAutoSubset returned. + */ +VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset) +{ + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + uint32_t cEntries = pSet->cEntries; + LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries)); + AssertReturnVoid(cEntries != PGMMAPSET_CLOSED); + AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset || iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries)); +#ifdef IN_RC + if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE) + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]); + else +#endif + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]); + if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100 + && cEntries != pSet->iSubset) + { + pgmDynMapFlushSubset(pSet); + Assert(pSet->cEntries >= iPrevSubset || iPrevSubset == UINT32_MAX); + } + pSet->iSubset = iPrevSubset; +} + + +/** + * Indicates that the given page is unused and its mapping can be re-used. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pvHint The page that is now unused. This does not have to + * point at the start of the page. NULL is ignored. + */ +#ifdef LOG_ENABLED +void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL) +#else +void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint) +#endif +{ + /* + * Ignore NULL pointers and mask off the page offset bits. + */ + if (pvHint == NULL) + return; + pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK); + + PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet; + uint32_t iEntry = pSet->cEntries; + AssertReturnVoid(iEntry > 0); + + /* + * Find the entry in the usual unrolled fashion. + */ + /** @todo add a hint to the set which entry was used last since it's not + * always the last entry? */ +#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \ + ( (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \ + && (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \ + > (pSet)->aEntries[(iEntry)].cUnrefs ) + if ( iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint)) + iEntry = iEntry - 1; + else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint)) + iEntry = iEntry - 2; + else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint)) + iEntry = iEntry - 3; + else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint)) + iEntry = iEntry - 4; + else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint)) + iEntry = iEntry - 5; + else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint)) + iEntry = iEntry - 6; + else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint)) + iEntry = iEntry - 7; + else + { + /* + * Loop till we find it. + */ + bool fFound = false; + if (iEntry > 7) + { + iEntry -= 7; + while (iEntry-- > 0) + if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint)) + { + fFound = true; + break; + } + } + AssertMsgReturnVoid(fFound, + ("pvHint=%p cEntries=%#x iSubset=%#x\n" + "aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n" + "aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n" + "aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n" + "aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n" + "aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n" + "aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n" + , + pvHint, pSet->cEntries, pSet->iSubset, + pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage, + pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage, + pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage, + pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage, + pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage, + pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage)); + } +#undef IS_MATCHING_ENTRY + + /* + * Update it. + */ + uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs; + uint32_t const cUnrefs = pSet->aEntries[iEntry].cUnrefs; + LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n", + pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction)); + AssertReturnVoid(cTotalRefs > cUnrefs); + + if (RT_LIKELY(cUnrefs < UINT16_MAX - 1)) + pSet->aEntries[iEntry].cUnrefs++; + else if (pSet->aEntries[iEntry].cInlinedRefs) + { + uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs); + pSet->aEntries[iEntry].cInlinedRefs -= cSub; + pSet->aEntries[iEntry].cUnrefs -= cSub; + pSet->aEntries[iEntry].cUnrefs++; + } + else + Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction)); + +#ifdef PGMRZDYNMAP_STRICT_RELEASE + /* + * Optimize the set to trigger the unmapping and invalidation of the page. + */ + if (cUnrefs + 1 == cTotalRefs) + pgmDynMapOptimizeAutoSet(pSet); +#endif +} + + +/** + * Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined + * and pgmR0DynMapGCPageOffInlined. + * + * @returns VINF_SUCCESS, bails out to ring-3 on failure. + * @param pSet The set. + * @param HCPhys The physical address of the page. + * @param ppv Where to store the address of the mapping on success. + * + * @remarks This is a very hot path. + */ +int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL) +{ + AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags())); + PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet); + STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a); + + /* + * Map it. + */ + void *pvPage; + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + uint32_t iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage); + if (RT_UNLIKELY(iPage == UINT32_MAX)) + { + /* + * We're out of mapping space, optimize our set to try remedy the + * situation. (Only works if there are unreference hints.) + */ + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize); + pgmDynMapOptimizeAutoSet(pSet); + + iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage); + if (RT_UNLIKELY(iPage == UINT32_MAX)) + { + RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n", + pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages); + if (!g_fPGMR0DynMapTestRunning) + VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0); + *ppv = NULL; + STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a); + return VERR_PGM_DYNMAP_FAILED; + } + } + + /* + * Add the page to the auto reference set. + * + * The typical usage pattern means that the same pages will be mapped + * several times in the same set. We can catch most of these + * remappings by looking a few pages back into the set. (The searching + * and set optimizing path will hardly ever be used when doing this.) + */ + AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8); + int32_t i = pSet->cEntries; + if (i-- < 5) + { + unsigned iEntry = pSet->cEntries++; + pSet->aEntries[iEntry].cRefs = 1; + pSet->aEntries[iEntry].cUnrefs = 0; + pSet->aEntries[iEntry].cInlinedRefs = 0; + pSet->aEntries[iEntry].iPage = iPage; + pSet->aEntries[iEntry].pvPage = pvPage; + pSet->aEntries[iEntry].HCPhys = HCPhys; + pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n", + pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction)); + } + /* Any of the last 5 pages? */ + else if ( pSet->aEntries[i - 0].iPage == iPage + && pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i - 0].cRefs++; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction)); + } + else if ( pSet->aEntries[i - 1].iPage == iPage + && pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i - 1].cRefs++; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction)); + } + else if ( pSet->aEntries[i - 2].iPage == iPage + && pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i - 2].cRefs++; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction)); + } + else if ( pSet->aEntries[i - 3].iPage == iPage + && pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i - 3].cRefs++; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction)); + } + else if ( pSet->aEntries[i - 4].iPage == iPage + && pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i - 4].cRefs++; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction)); + } + /* Don't bother searching unless we're above a 60% load. */ + else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100)) + { + unsigned iEntry = pSet->cEntries++; + pSet->aEntries[iEntry].cRefs = 1; + pSet->aEntries[iEntry].cUnrefs = 0; + pSet->aEntries[iEntry].cInlinedRefs = 0; + pSet->aEntries[iEntry].iPage = iPage; + pSet->aEntries[iEntry].pvPage = pvPage; + pSet->aEntries[iEntry].HCPhys = HCPhys; + pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction)); + } + else + { + /* Search the rest of the set. */ + Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries)); + i -= 4; + while (i-- > 0) + if ( pSet->aEntries[i].iPage == iPage + && pSet->aEntries[i].cRefs < UINT16_MAX - 1) + { + pSet->aEntries[i].cRefs++; + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits); + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction)); + break; + } + if (i < 0) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses); +#if 0 /* this is very bogus */ + if (pSet->iSubset < pSet->cEntries) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes); + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]); + pgmDynMapFlushSubset(pSet); + } +#endif + + if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries))) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize); + pgmDynMapOptimizeAutoSet(pSet); + } + + if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries))) + { + unsigned iEntry = pSet->cEntries++; + pSet->aEntries[iEntry].cRefs = 1; + pSet->aEntries[iEntry].cUnrefs = 0; + pSet->aEntries[iEntry].cInlinedRefs = 0; + pSet->aEntries[iEntry].iPage = iPage; + pSet->aEntries[iEntry].pvPage = pvPage; + pSet->aEntries[iEntry].HCPhys = HCPhys; + pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry; + LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction)); + } + else + { + /* We're screwed. */ + pgmRZDynMapReleasePage(pThis, iPage, 1); + + RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n"); + if (!g_fPGMR0DynMapTestRunning) + VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0); + *ppv = NULL; + STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a); + return VERR_PGM_DYNMAP_FULL_SET; + } + } + } + + *ppv = pvPage; + STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a); + return VINF_SUCCESS; +} + + +#if 0 /*def DEBUG*/ +/** For pgmR0DynMapTest3PerCpu. */ +typedef struct PGMR0DYNMAPTEST +{ + uint32_t u32Expect; + uint32_t *pu32; + uint32_t volatile cFailures; +} PGMR0DYNMAPTEST; +typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST; + +/** + * Checks that the content of the page is the same on all CPUs, i.e. that there + * are no CPU specific PTs or similar nasty stuff involved. + * + * @param idCpu The current CPU. + * @param pvUser1 Pointer a PGMR0DYNMAPTEST structure. + * @param pvUser2 Unused, ignored. + */ +static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2) +{ + PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1; + ASMInvalidatePage(pTest->pu32); + if (*pTest->pu32 != pTest->u32Expect) + ASMAtomicIncU32(&pTest->cFailures); + NOREF(pvUser2); NOREF(idCpu); +} + + +/** + * Performs some basic tests in debug builds. + */ +static int pgmR0DynMapTest(PVM pVM) +{ + LogRel(("pgmR0DynMapTest: ****** START ******\n")); + PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet; + PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet); + uint32_t i; + + /* + * Assert internal integrity first. + */ + LogRel(("Test #0\n")); + int rc = PGMR0DynMapAssertIntegrity(); + if (RT_FAILURE(rc)) + return rc; + + void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed; + pVM->pgm.s.pvR0DynMapUsed = pThis; + g_fPGMR0DynMapTestRunning = true; + + /* + * Simple test, map CR3 twice and check that we're getting the + * same mapping address back. + */ + LogRel(("Test #1\n")); + ASMIntDisable(); + PGMRZDynMapStartAutoSet(&pVM->aCpus[0]); + + uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK; + void *pv = (void *)(intptr_t)-1; + void *pv2 = (void *)(intptr_t)-2; + rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv RTLOG_COMMA_SRC_POS); + int rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS); + ASMIntEnable(); + if ( RT_SUCCESS(rc2) + && RT_SUCCESS(rc) + && pv == pv2) + { + LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries))); + rc = PGMR0DynMapAssertIntegrity(); + + /* + * Check that the simple set overflow code works by filling it + * with more CR3 mappings. + */ + LogRel(("Test #2\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++) + { + pv2 = (void *)(intptr_t)-4; + rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS); + } + ASMIntEnable(); + if (RT_FAILURE(rc) || pv != pv2) + { + LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i)); + if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE; + } + else if (pSet->cEntries != 5) + { + LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2)); + rc = VERR_PGM_DYNMAP_IPE; + } + else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1 + || pSet->aEntries[3].cRefs != UINT16_MAX - 1 + || pSet->aEntries[2].cRefs != 1 + || pSet->aEntries[1].cRefs != 1 + || pSet->aEntries[0].cRefs != 1) + { + LogRel(("failed(%d): bad set dist: ", __LINE__)); + for (i = 0; i < pSet->cEntries; i++) + LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs)); + LogRel(("\n")); + rc = VERR_PGM_DYNMAP_IPE; + } + if (RT_SUCCESS(rc)) + rc = PGMR0DynMapAssertIntegrity(); + if (RT_SUCCESS(rc)) + { + /* + * Trigger an set optimization run (exactly). + */ + LogRel(("Test #3\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + pv2 = NULL; + for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++) + { + pv2 = (void *)(intptr_t)(-5 - i); + rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS); + } + ASMIntEnable(); + if (RT_FAILURE(rc) || pv == pv2) + { + LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i)); + if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE; + } + else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries)) + { + LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries))); + rc = VERR_PGM_DYNMAP_IPE; + } + LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries))); + if (RT_SUCCESS(rc)) + rc = PGMR0DynMapAssertIntegrity(); + if (RT_SUCCESS(rc)) + { + /* + * Trigger an overflow error. + */ + LogRel(("Test #4\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++) + { + rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS); + if (RT_SUCCESS(rc)) + rc = PGMR0DynMapAssertIntegrity(); + if (RT_FAILURE(rc)) + break; + } + ASMIntEnable(); + if (rc == VERR_PGM_DYNMAP_FULL_SET) + { + /* flush the set. */ + LogRel(("Test #5\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]); + PGMRZDynMapStartAutoSet(&pVM->aCpus[0]); + ASMIntEnable(); + + rc = PGMR0DynMapAssertIntegrity(); + } + else + { + LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__, + rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i)); + if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE; + } + } + } + } + else + { + LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2)); + if (RT_SUCCESS(rc)) + rc = rc2; + } + + /* + * Check that everyone sees the same stuff. + */ + if (RT_SUCCESS(rc)) + { + LogRel(("Test #5\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0); + rc = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS); + if (RT_SUCCESS(rc)) + { + PGMR0DYNMAPTEST Test; + uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u; + Test.pu32 = (uint32_t *)((uintptr_t)pv | ((uintptr_t)pu32Real & PAGE_OFFSET_MASK)); + Test.u32Expect = *pu32Real; + ASMAtomicWriteU32(&Test.cFailures, 0); + ASMIntEnable(); + + rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL); + if (RT_FAILURE(rc)) + LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc)); + else if (Test.cFailures) + { + LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__, + Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32)); + rc = VERR_PGM_DYNMAP_IPE; + } + else + LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", + pu32Real, Test.pu32, Test.u32Expect, *Test.pu32)); + } + else + { + ASMIntEnable(); + LogRel(("failed(%d): rc=%Rrc\n", rc)); + } + } + + /* + * Clean up. + */ + LogRel(("Cleanup.\n")); + ASMIntDisable(); + PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]); + PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]); + PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]); + ASMIntEnable(); + + if (RT_SUCCESS(rc)) + rc = PGMR0DynMapAssertIntegrity(); + else + PGMR0DynMapAssertIntegrity(); + + g_fPGMR0DynMapTestRunning = false; + LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc, + pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries))); + pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved; + LogRel(("pgmR0DynMapTest: ****** END ******\n")); + return rc; +} +#endif /* DEBUG */ + |