diff options
Diffstat (limited to 'src/VBox/VMM/include/PGMInternal.h')
| -rw-r--r-- | src/VBox/VMM/include/PGMInternal.h | 3884 |
1 files changed, 3884 insertions, 0 deletions
diff --git a/src/VBox/VMM/include/PGMInternal.h b/src/VBox/VMM/include/PGMInternal.h new file mode 100644 index 00000000..5ebeca1e --- /dev/null +++ b/src/VBox/VMM/include/PGMInternal.h @@ -0,0 +1,3884 @@ +/* $Id: PGMInternal.h $ */ +/** @file + * PGM - Internal header file. + */ + +/* + * Copyright (C) 2006-2022 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <https://www.gnu.org/licenses>. + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifndef VMM_INCLUDED_SRC_include_PGMInternal_h +#define VMM_INCLUDED_SRC_include_PGMInternal_h +#ifndef RT_WITHOUT_PRAGMA_ONCE +# pragma once +#endif + +#include <VBox/cdefs.h> +#include <VBox/types.h> +#include <VBox/err.h> +#include <VBox/dbg.h> +#include <VBox/vmm/stam.h> +#include <VBox/param.h> +#include <VBox/vmm/vmm.h> +#include <VBox/vmm/mm.h> +#include <VBox/vmm/pdmcritsect.h> +#include <VBox/vmm/pdmapi.h> +#include <VBox/dis.h> +#include <VBox/vmm/dbgf.h> +#include <VBox/log.h> +#include <VBox/vmm/gmm.h> +#include <VBox/vmm/hm.h> +#include <iprt/asm.h> +#include <iprt/assert.h> +#include <iprt/avl.h> +#include <iprt/critsect.h> +#include <iprt/list-off32.h> +#include <iprt/sha.h> +#include <iprt/cpp/hardavlrange.h> + + + +/** @defgroup grp_pgm_int Internals + * @ingroup grp_pgm + * @internal + * @{ + */ + + +/** @name PGM Compile Time Config + * @{ + */ + +/** + * Check and skip global PDEs for non-global flushes + */ +#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH + +/** + * Optimization for PAE page tables that are modified often + */ +//#if 0 /* disabled again while debugging */ +#define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT +//#endif + +/** + * Large page support enabled only on 64 bits hosts; applies to nested paging only. + */ +#define PGM_WITH_LARGE_PAGES + +/** + * Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and + * VMX_EXIT_EPT_MISCONFIG. + */ +#define PGM_WITH_MMIO_OPTIMIZATIONS + +/** + * Sync N pages instead of a whole page table + */ +#define PGM_SYNC_N_PAGES + +/** + * Number of pages to sync during a page fault + * + * When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here + * causes a lot of unnecessary extents and also is slower than taking more \#PFs. + * + * Note that \#PFs are much more expensive in the VT-x/AMD-V case due to + * world switch overhead, so let's sync more. + */ +#ifdef IN_RING0 +/* Chose 32 based on the compile test in @bugref{4219}; 64 shows worse stats. + * 32 again shows better results than 16; slightly more overhead in the \#PF handler, + * but ~5% fewer faults. + */ +# define PGM_SYNC_NR_PAGES 32 +#else +# define PGM_SYNC_NR_PAGES 8 +#endif + +/** + * Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t)) + */ +#define PGM_MAX_PHYSCACHE_ENTRIES 64 +#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1) + + +/** @def PGMPOOL_CFG_MAX_GROW + * The maximum number of pages to add to the pool in one go. + */ +#define PGMPOOL_CFG_MAX_GROW (_2M >> GUEST_PAGE_SHIFT) /** @todo or HOST_PAGE_SHIFT ? */ + +/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL + * Enables some extra assertions for virtual handlers (mainly phys2virt related). + */ +#ifdef VBOX_STRICT +# define VBOX_STRICT_PGM_HANDLER_VIRTUAL +#endif + +/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC + * Enables the experimental lazy page allocation code. */ +#ifdef DOXYGEN_RUNNING +# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC +#endif + +/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES + * Enables real write monitoring of pages, i.e. mapping them read-only and + * only making them writable when getting a write access \#PF. */ +#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES + +/** @def VBOX_WITH_PGM_NEM_MODE + * Enabled the NEM memory management mode in PGM. See PGM::fNemMode for + * details. */ +#ifdef DOXYGEN_RUNNING +# define VBOX_WITH_PGM_NEM_MODE +#endif + +/** @} */ + + +/** @name PDPT and PML4 flags. + * These are placed in the three bits available for system programs in + * the PDPT and PML4 entries. + * @{ */ +/** The entry is a permanent one and it's must always be present. + * Never free such an entry. */ +#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10) +/** PGM specific bits in PML4 entries. */ +#define PGM_PML4_FLAGS 0 +/** PGM specific bits in PDPT entries. */ +#define PGM_PDPT_FLAGS (PGM_PLXFLAGS_PERMANENT) +/** @} */ + +/** @name Page directory flags. + * These are placed in the three bits available for system programs in + * the page directory entries. + * @{ */ +/** Indicates the original entry was a big page. + * @remarks This is currently only used for statistics and can be recycled. */ +#define PGM_PDFLAGS_BIG_PAGE RT_BIT_64(9) +/** Made read-only to facilitate dirty bit tracking. */ +#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11) +/** @} */ + +/** @name Page flags. + * These are placed in the three bits available for system programs in + * the page entries. + * @{ */ +/** Made read-only to facilitate dirty bit tracking. */ +#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9) + +#ifndef PGM_PTFLAGS_CSAM_VALIDATED +/** Scanned and approved by CSAM (tm). + * NOTE: Must be identical to the one defined in CSAMInternal.h!! + * @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/vmm/pgm.h. */ +#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11) +#endif + +/** @} */ + +/** @name Defines used to indicate the shadow and guest paging in the templates. + * @{ */ +#define PGM_TYPE_REAL 1 +#define PGM_TYPE_PROT 2 +#define PGM_TYPE_32BIT 3 +#define PGM_TYPE_PAE 4 +#define PGM_TYPE_AMD64 5 +#define PGM_TYPE_NESTED_32BIT 6 +#define PGM_TYPE_NESTED_PAE 7 +#define PGM_TYPE_NESTED_AMD64 8 +#define PGM_TYPE_EPT 9 +#define PGM_TYPE_NONE 10 /**< Dummy shadow paging mode for NEM. */ +#define PGM_TYPE_END (PGM_TYPE_NONE + 1) +#define PGM_TYPE_FIRST_SHADOW PGM_TYPE_32BIT /**< The first type used by shadow paging. */ +/** @} */ + +/** @name Defines used to indicate the second-level + * address translation (SLAT) modes in the templates. + * @{ */ +#define PGM_SLAT_TYPE_DIRECT (PGM_TYPE_END + 1) +#define PGM_SLAT_TYPE_EPT (PGM_TYPE_END + 2) +#define PGM_SLAT_TYPE_32BIT (PGM_TYPE_END + 3) +#define PGM_SLAT_TYPE_PAE (PGM_TYPE_END + 4) +#define PGM_SLAT_TYPE_AMD64 (PGM_TYPE_END + 5) +/** @} */ + +/** Macro for checking if the guest is using paging. + * @param uGstType PGM_TYPE_* + * @param uShwType PGM_TYPE_* + * @remark ASSUMES certain order of the PGM_TYPE_* values. + */ +#define PGM_WITH_PAGING(uGstType, uShwType) \ + ( (uGstType) >= PGM_TYPE_32BIT \ + && (uShwType) < PGM_TYPE_NESTED_32BIT) + +/** Macro for checking if the guest supports the NX bit. + * @param uGstType PGM_TYPE_* + * @param uShwType PGM_TYPE_* + * @remark ASSUMES certain order of the PGM_TYPE_* values. + */ +#define PGM_WITH_NX(uGstType, uShwType) \ + ( (uGstType) >= PGM_TYPE_PAE \ + && (uShwType) < PGM_TYPE_NESTED_32BIT) + +/** Macro for checking for nested. + * @param uType PGM_TYPE_* + */ +#define PGM_TYPE_IS_NESTED(uType) \ + ( (uType) == PGM_TYPE_NESTED_32BIT \ + || (uType) == PGM_TYPE_NESTED_PAE \ + || (uType) == PGM_TYPE_NESTED_AMD64) + +/** Macro for checking for nested or EPT. + * @param uType PGM_TYPE_* + */ +#define PGM_TYPE_IS_NESTED_OR_EPT(uType) \ + ( (uType) == PGM_TYPE_NESTED_32BIT \ + || (uType) == PGM_TYPE_NESTED_PAE \ + || (uType) == PGM_TYPE_NESTED_AMD64 \ + || (uType) == PGM_TYPE_EPT) + + + +/** @def PGM_HCPHYS_2_PTR + * Maps a HC physical page pool address to a virtual address. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param HCPhys The HC physical address to map to a virtual one. + * @param ppv Where to store the virtual address. No need to cast + * this. + * + * @remark There is no need to assert on the result. + */ +#define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) pgmPoolHCPhys2Ptr(pVM, HCPhys, (void **)(ppv)) + +/** @def PGM_GCPHYS_2_PTR_V2 + * Maps a GC physical page address to a virtual address. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhys The GC physical address to map to a virtual one. + * @param ppv Where to store the virtual address. No need to cast this. + * + * @remark Use with care as we don't have so much dynamic mapping space in + * ring-0 on 32-bit darwin and in RC. + * @remark There is no need to assert on the result. + */ +#define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \ + pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */ + +/** @def PGM_GCPHYS_2_PTR + * Maps a GC physical page address to a virtual address. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address to map to a virtual one. + * @param ppv Where to store the virtual address. No need to cast this. + * + * @remark Use with care as we don't have so much dynamic mapping space in + * ring-0 on 32-bit darwin and in RC. + * @remark There is no need to assert on the result. + */ +#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv) + +/** @def PGM_GCPHYS_2_PTR_BY_VMCPU + * Maps a GC physical page address to a virtual address. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhys The GC physical address to map to a virtual one. + * @param ppv Where to store the virtual address. No need to cast this. + * + * @remark Use with care as we don't have so much dynamic mapping space in + * ring-0 on 32-bit darwin and in RC. + * @remark There is no need to assert on the result. + */ +#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv) + +/** @def PGM_GCPHYS_2_PTR_EX + * Maps a unaligned GC physical page address to a virtual address. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address to map to a virtual one. + * @param ppv Where to store the virtual address. No need to cast this. + * + * @remark Use with care as we don't have so much dynamic mapping space in + * ring-0 on 32-bit darwin and in RC. + * @remark There is no need to assert on the result. + */ +#define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \ + pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */ + +/** @def PGM_DYNMAP_UNUSED_HINT + * Hints to the dynamic mapping code in RC and R0/darwin that the specified page + * is no longer used. + * + * For best effect only apply this to the page that was mapped most recently. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pvPage The pool page. + */ +#define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) do {} while (0) + +/** @def PGM_DYNMAP_UNUSED_HINT_VM + * Hints to the dynamic mapping code in RC and R0/darwin that the specified page + * is no longer used. + * + * For best effect only apply this to the page that was mapped most recently. + * + * @param pVM The cross context VM structure. + * @param pvPage The pool page. + */ +#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage) PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage) + + +/** @def PGM_INVL_PG + * Invalidates a page. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCVirt The virtual address of the page to invalidate. + */ +#ifdef IN_RING0 +# define PGM_INVL_PG(pVCpu, GCVirt) HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt)) +#elif defined(IN_RING3) +# define PGM_INVL_PG(pVCpu, GCVirt) HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt)) +#else +# error "Not IN_RING0 or IN_RING3!" +#endif + +/** @def PGM_INVL_PG_ALL_VCPU + * Invalidates a page on all VCPUs + * + * @param pVM The cross context VM structure. + * @param GCVirt The virtual address of the page to invalidate. + */ +#ifdef IN_RING0 +# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt)) +#else +# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt)) +#endif + +/** @def PGM_INVL_BIG_PG + * Invalidates a 4MB page directory entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCVirt The virtual address within the page directory to invalidate. + */ +#ifdef IN_RING0 +# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HMFlushTlb(pVCpu) +#else +# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HMFlushTlb(pVCpu) +#endif + +/** @def PGM_INVL_VCPU_TLBS() + * Invalidates the TLBs of the specified VCPU + * + * @param pVCpu The cross context virtual CPU structure. + */ +#ifdef IN_RING0 +# define PGM_INVL_VCPU_TLBS(pVCpu) HMFlushTlb(pVCpu) +#else +# define PGM_INVL_VCPU_TLBS(pVCpu) HMFlushTlb(pVCpu) +#endif + +/** @def PGM_INVL_ALL_VCPU_TLBS() + * Invalidates the TLBs of all VCPUs + * + * @param pVM The cross context VM structure. + */ +#ifdef IN_RING0 +# define PGM_INVL_ALL_VCPU_TLBS(pVM) HMFlushTlbOnAllVCpus(pVM) +#else +# define PGM_INVL_ALL_VCPU_TLBS(pVM) HMFlushTlbOnAllVCpus(pVM) +#endif + + +/** @name Safer Shadow PAE PT/PTE + * For helping avoid misinterpreting invalid PAE/AMD64 page table entries as + * present. + * + * @{ + */ +#if 1 +/** + * For making sure that u1Present and X86_PTE_P checks doesn't mistake + * invalid entries for present. + * @sa X86PTEPAE. + */ +typedef union PGMSHWPTEPAE +{ + /** Unsigned integer view */ + X86PGPAEUINT uCareful; + /* Not other views. */ +} PGMSHWPTEPAE; + +# define PGMSHWPTEPAE_IS_P(Pte) ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P ) +# define PGMSHWPTEPAE_IS_RW(Pte) ( !!((Pte).uCareful & X86_PTE_RW)) +# define PGMSHWPTEPAE_IS_US(Pte) ( !!((Pte).uCareful & X86_PTE_US)) +# define PGMSHWPTEPAE_IS_A(Pte) ( !!((Pte).uCareful & X86_PTE_A)) +# define PGMSHWPTEPAE_IS_D(Pte) ( !!((Pte).uCareful & X86_PTE_D)) +# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) ) +# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_RW | X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P | X86_PTE_RW) ) +# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).uCareful ) +# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).uCareful & X86_PTE_PAE_PG_MASK ) +# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).uCareful ) /**< Use with care. */ +# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).uCareful = (uVal); } while (0) +# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).uCareful = (Pte2).uCareful; } while (0) +# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0) +# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0) +# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0) +# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).uCareful |= X86_PTE_RW; } while (0) + +/** + * For making sure that u1Present and X86_PTE_P checks doesn't mistake + * invalid entries for present. + * @sa X86PTPAE. + */ +typedef struct PGMSHWPTPAE +{ + PGMSHWPTEPAE a[X86_PG_PAE_ENTRIES]; +} PGMSHWPTPAE; + +#else +typedef X86PTEPAE PGMSHWPTEPAE; +typedef X86PTPAE PGMSHWPTPAE; +# define PGMSHWPTEPAE_IS_P(Pte) ( (Pte).n.u1Present ) +# define PGMSHWPTEPAE_IS_RW(Pte) ( (Pte).n.u1Write ) +# define PGMSHWPTEPAE_IS_US(Pte) ( (Pte).n.u1User ) +# define PGMSHWPTEPAE_IS_A(Pte) ( (Pte).n.u1Accessed ) +# define PGMSHWPTEPAE_IS_D(Pte) ( (Pte).n.u1Dirty ) +# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) ) +# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) ) +# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).u ) +# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).u & X86_PTE_PAE_PG_MASK ) +# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).u ) /**< Use with care. */ +# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).u = (uVal); } while (0) +# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).u = (Pte2).u; } while (0) +# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0) +# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0) +# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0) +# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).u |= X86_PTE_RW; } while (0) + +#endif + +/** Pointer to a shadow PAE PTE. */ +typedef PGMSHWPTEPAE *PPGMSHWPTEPAE; +/** Pointer to a const shadow PAE PTE. */ +typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE; + +/** Pointer to a shadow PAE page table. */ +typedef PGMSHWPTPAE *PPGMSHWPTPAE; +/** Pointer to a const shadow PAE page table. */ +typedef PGMSHWPTPAE const *PCPGMSHWPTPAE; +/** @} */ + + +/** The physical access handler type handle count (power of two). */ +#define PGMPHYSHANDLERTYPE_COUNT 0x20 +/** Mask for getting the array index from an access handler type handle. + * The other handle bits are random and non-zero to avoid mixups due to zero + * initialized fields. */ +#define PGMPHYSHANDLERTYPE_IDX_MASK 0x1f + +/** + * Physical page access handler type registration, ring-0 part. + */ +typedef struct PGMPHYSHANDLERTYPEINTR0 +{ + /** The handle value for verfication. */ + PGMPHYSHANDLERTYPE hType; + /** The kind of accesses we're handling. */ + PGMPHYSHANDLERKIND enmKind; + /** The PGM_PAGE_HNDL_PHYS_STATE_XXX value corresponding to enmKind. */ + uint8_t uState; + /** Whether to keep the PGM lock when calling the handler. + * @sa PGMPHYSHANDLER_F_KEEP_PGM_LOCK */ + bool fKeepPgmLock; + /** Set if this is registered by a device instance and uUser should be + * translated from a device instance ID to a pointer. + * @sa PGMPHYSHANDLER_F_R0_DEVINS_IDX */ + bool fRing0DevInsIdx; + /** See PGMPHYSHANDLER_F_NOT_IN_HM. */ + bool fNotInHm : 1; + /** Pointer to the ring-0 callback function. */ + R0PTRTYPE(PFNPGMPHYSHANDLER) pfnHandler; + /** Pointer to the ring-0 callback function for \#PFs, can be NULL. */ + R0PTRTYPE(PFNPGMRZPHYSPFHANDLER) pfnPfHandler; + /** Description / Name. For easing debugging. */ + R0PTRTYPE(const char *) pszDesc; +} PGMPHYSHANDLERTYPEINTR0; +/** Pointer to a physical access handler type registration. */ +typedef PGMPHYSHANDLERTYPEINTR0 *PPGMPHYSHANDLERTYPEINTR0; + +/** + * Physical page access handler type registration, shared/ring-3 part. + */ +typedef struct PGMPHYSHANDLERTYPEINTR3 +{ + /** The handle value for verfication. */ + PGMPHYSHANDLERTYPE hType; + /** The kind of accesses we're handling. */ + PGMPHYSHANDLERKIND enmKind; + /** The PGM_PAGE_HNDL_PHYS_STATE_XXX value corresponding to enmKind. */ + uint8_t uState; + /** Whether to keep the PGM lock when calling the handler. + * @sa PGMPHYSHANDLER_F_KEEP_PGM_LOCK */ + bool fKeepPgmLock; + /** Set if this is registered by a device instance and uUser should be + * translated from a device instance ID to a pointer. + * @sa PGMPHYSHANDLER_F_R0_DEVINS_IDX */ + bool fRing0DevInsIdx; + /** Set by ring-0 if the handler is ring-0 enabled (for debug). */ + bool fRing0Enabled : 1; + /** See PGMPHYSHANDLER_F_NOT_IN_HM. */ + bool fNotInHm : 1; + /** Pointer to the ring-3 callback function. */ + R3PTRTYPE(PFNPGMPHYSHANDLER) pfnHandler; + /** Description / Name. For easing debugging. */ + R3PTRTYPE(const char *) pszDesc; +} PGMPHYSHANDLERTYPEINTR3; +/** Pointer to a physical access handler type registration. */ +typedef PGMPHYSHANDLERTYPEINTR3 *PPGMPHYSHANDLERTYPEINTR3; + +/** Pointer to a physical access handler type record for the current context. */ +typedef CTX_SUFF(PPGMPHYSHANDLERTYPEINT) PPGMPHYSHANDLERTYPEINT; +/** Pointer to a const physical access handler type record for the current context. */ +typedef CTX_SUFF(PGMPHYSHANDLERTYPEINT) const *PCPGMPHYSHANDLERTYPEINT; +/** Dummy physical access handler type record. */ +extern CTX_SUFF(PGMPHYSHANDLERTYPEINT) const g_pgmHandlerPhysicalDummyType; + + +/** + * Physical page access handler structure. + * + * This is used to keep track of physical address ranges + * which are being monitored in some kind of way. + */ +typedef struct PGMPHYSHANDLER +{ + /** @name Tree stuff. + * @{ */ + /** First address. */ + RTGCPHYS Key; + /** Last address. */ + RTGCPHYS KeyLast; + uint32_t idxLeft; + uint32_t idxRight; + uint8_t cHeight; + /** @} */ + uint8_t abPadding[3]; + /** Number of pages to update. */ + uint32_t cPages; + /** Set if we have pages that have been aliased. */ + uint32_t cAliasedPages; + /** Set if we have pages that have temporarily been disabled. */ + uint32_t cTmpOffPages; + /** Registered handler type handle. + * @note Marked volatile to prevent re-reading after validation. */ + PGMPHYSHANDLERTYPE volatile hType; + /** User argument for the handlers. */ + uint64_t uUser; + /** Description / Name. For easing debugging. */ + R3PTRTYPE(const char *) pszDesc; + /** Profiling of this handler. + * @note VBOX_WITH_STATISTICS only, but included to keep structure stable. */ + STAMPROFILE Stat; +} PGMPHYSHANDLER; +AssertCompileSize(PGMPHYSHANDLER, 12*8); +/** Pointer to a physical page access handler structure. */ +typedef PGMPHYSHANDLER *PPGMPHYSHANDLER; + +/** + * Gets the type record for a physical handler (no reference added). + * @returns PCPGMPHYSHANDLERTYPEINT, can be NULL + * @param a_pVM The cross context VM structure. + * @param a_pPhysHandler Pointer to the physical handler structure + * (PGMPHYSHANDLER). + */ +#define PGMPHYSHANDLER_GET_TYPE(a_pVM, a_pPhysHandler) \ + pgmHandlerPhysicalTypeHandleToPtr(a_pVM, (a_pPhysHandler) ? (a_pPhysHandler)->hType : NIL_PGMPHYSHANDLERTYPE) + +/** + * Gets the type record for a physical handler, never returns NULL. + * + * @returns PCPGMPHYSHANDLERTYPEINT, never NULL. + * @param a_pVM The cross context VM structure. + * @param a_pPhysHandler Pointer to the physical handler structure + * (PGMPHYSHANDLER). + */ +#define PGMPHYSHANDLER_GET_TYPE_NO_NULL(a_pVM, a_pPhysHandler) \ + pgmHandlerPhysicalTypeHandleToPtr2(a_pVM, (a_pPhysHandler) ? (a_pPhysHandler)->hType : NIL_PGMPHYSHANDLERTYPE) + +/** Physical access handler allocator. */ +typedef RTCHardAvlTreeSlabAllocator<PGMPHYSHANDLER> PGMPHYSHANDLERALLOCATOR; + +/** Physical access handler tree. */ +typedef RTCHardAvlRangeTree<PGMPHYSHANDLER, RTGCPHYS> PGMPHYSHANDLERTREE; +/** Pointer to a physical access handler tree. */ +typedef PGMPHYSHANDLERTREE *PPGMPHYSHANDLERTREE; + + +/** + * A Physical Guest Page tracking structure. + * + * The format of this structure is complicated because we have to fit a lot + * of information into as few bits as possible. The format is also subject + * to change (there is one coming up soon). Which means that for we'll be + * using PGM_PAGE_GET_*, PGM_PAGE_IS_ and PGM_PAGE_SET_* macros for *all* + * accesses to the structure. + */ +typedef union PGMPAGE +{ + /** Structured view. */ + struct + { + /** 1:0 - The physical handler state (PGM_PAGE_HNDL_PHYS_STATE_*). */ + uint64_t u2HandlerPhysStateY : 2; + /** 2 - Don't apply the physical handler in HM mode (nested APIC hack). */ + uint64_t fHandlerPhysNotInHm : 1; + /** 3 - Flag indicating that a write monitored page was written to + * when set. */ + uint64_t fWrittenToY : 1; + /** 7:4 - Unused. */ + uint64_t u2Unused0 : 4; + /** 9:8 - Paging structure needed to map the page + * (PGM_PAGE_PDE_TYPE_*). */ + uint64_t u2PDETypeY : 2; + /** 11:10 - NEM state bits. */ + uint64_t u2NemStateY : 2; + /** 12:48 - The host physical frame number (shift left to get the + * address). */ + uint64_t HCPhysFN : 36; + /** 50:48 - The page state. */ + uint64_t uStateY : 3; + /** 51:53 - The page type (PGMPAGETYPE). */ + uint64_t uTypeY : 3; + /** 63:54 - PTE index for usage tracking (page pool). */ + uint64_t u10PteIdx : 10; + + /** The GMM page ID. + * @remarks In the current implementation, MMIO2 and pages aliased to + * MMIO2 pages will be exploiting this field to calculate the + * ring-3 mapping address corresponding to the page. + * Later we may consider including MMIO2 management into GMM. */ + uint32_t idPage; + /** Usage tracking (page pool). */ + uint16_t u16TrackingY; + /** The number of read locks on this page. */ + uint8_t cReadLocksY; + /** The number of write locks on this page. */ + uint8_t cWriteLocksY; + } s; + + /** 64-bit integer view. */ + uint64_t au64[2]; + /** 16-bit view. */ + uint32_t au32[4]; + /** 16-bit view. */ + uint16_t au16[8]; + /** 8-bit view. */ + uint8_t au8[16]; +} PGMPAGE; +AssertCompileSize(PGMPAGE, 16); +/** Pointer to a physical guest page. */ +typedef PGMPAGE *PPGMPAGE; +/** Pointer to a const physical guest page. */ +typedef const PGMPAGE *PCPGMPAGE; +/** Pointer to a physical guest page pointer. */ +typedef PPGMPAGE *PPPGMPAGE; + + +/** + * Clears the page structure. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_CLEAR(a_pPage) \ + do { \ + (a_pPage)->au64[0] = 0; \ + (a_pPage)->au64[1] = 0; \ + } while (0) + +/** + * Initializes the page structure. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_HCPhys The host physical address of the page. + * @param a_idPage The (GMM) page ID of the page. + * @param a_uType The page type (PGMPAGETYPE). + * @param a_uState The page state (PGM_PAGE_STATE_XXX). + */ +#define PGM_PAGE_INIT(a_pPage, a_HCPhys, a_idPage, a_uType, a_uState) \ + do { \ + RTHCPHYS SetHCPhysTmp = (a_HCPhys); \ + AssertFatalMsg(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000)), ("%RHp\n", SetHCPhysTmp)); \ + (a_pPage)->au64[0] = SetHCPhysTmp; \ + (a_pPage)->au64[1] = 0; \ + (a_pPage)->s.idPage = (a_idPage); \ + (a_pPage)->s.uStateY = (a_uState); \ + (a_pPage)->s.uTypeY = (a_uType); \ + } while (0) + +/** + * Initializes the page structure of a ZERO page. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_pVM The VM handle (for getting the zero page address). + * @param a_uType The page type (PGMPAGETYPE). + */ +#define PGM_PAGE_INIT_ZERO(a_pPage, a_pVM, a_uType) \ + PGM_PAGE_INIT((a_pPage), (a_pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (a_uType), PGM_PAGE_STATE_ZERO) + + +/** @name The Page state, PGMPAGE::uStateY. + * @{ */ +/** The zero page. + * This is a per-VM page that's never ever mapped writable. */ +#define PGM_PAGE_STATE_ZERO 0U +/** A allocated page. + * This is a per-VM page allocated from the page pool (or wherever + * we get MMIO2 pages from if the type is MMIO2). + */ +#define PGM_PAGE_STATE_ALLOCATED 1U +/** A allocated page that's being monitored for writes. + * The shadow page table mappings are read-only. When a write occurs, the + * fWrittenTo member is set, the page remapped as read-write and the state + * moved back to allocated. */ +#define PGM_PAGE_STATE_WRITE_MONITORED 2U +/** The page is shared, aka. copy-on-write. + * This is a page that's shared with other VMs. */ +#define PGM_PAGE_STATE_SHARED 3U +/** The page is ballooned, so no longer available for this VM. */ +#define PGM_PAGE_STATE_BALLOONED 4U +/** @} */ + + +/** Asserts lock ownership in some of the PGM_PAGE_XXX macros. */ +#if defined(VBOX_STRICT) && 0 /** @todo triggers in pgmRZDynMapGCPageV2Inlined */ +# define PGM_PAGE_ASSERT_LOCK(a_pVM) PGM_LOCK_ASSERT_OWNER(a_pVM) +#else +# define PGM_PAGE_ASSERT_LOCK(a_pVM) do { } while (0) +#endif + +/** + * Gets the page state. + * @returns page state (PGM_PAGE_STATE_*). + * @param a_pPage Pointer to the physical guest page tracking structure. + * + * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict + * builds. + */ +#define PGM_PAGE_GET_STATE_NA(a_pPage) ( (a_pPage)->s.uStateY ) +#if defined(__GNUC__) && defined(VBOX_STRICT) +# define PGM_PAGE_GET_STATE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_STATE_NA(a_pPage); }) +#else +# define PGM_PAGE_GET_STATE PGM_PAGE_GET_STATE_NA +#endif + +/** + * Sets the page state. + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_uState The new page state. + */ +#define PGM_PAGE_SET_STATE(a_pVM, a_pPage, a_uState) \ + do { (a_pPage)->s.uStateY = (a_uState); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + + +/** + * Gets the host physical address of the guest page. + * @returns host physical address (RTHCPHYS). + * @param a_pPage Pointer to the physical guest page tracking structure. + * + * @remarks In strict builds on gcc platforms, this macro will make some ugly + * assumption about a valid pVM variable/parameter being in the + * current context. It will use this pVM variable to assert that the + * PGM lock is held. Use the PGM_PAGE_GET_HCPHYS_NA in contexts where + * pVM is not around. + */ +#if 0 +# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->s.HCPhysFN << 12 ) +# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA +#else +# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->au64[0] & UINT64_C(0x0000fffffffff000) ) +# if defined(__GNUC__) && defined(VBOX_STRICT) +# define PGM_PAGE_GET_HCPHYS(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_HCPHYS_NA(a_pPage); }) +# else +# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA +# endif +#endif + +/** + * Sets the host physical address of the guest page. + * + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_HCPhys The new host physical address. + */ +#define PGM_PAGE_SET_HCPHYS(a_pVM, a_pPage, a_HCPhys) \ + do { \ + RTHCPHYS const SetHCPhysTmp = (a_HCPhys); \ + AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \ + (a_pPage)->s.HCPhysFN = SetHCPhysTmp >> 12; \ + PGM_PAGE_ASSERT_LOCK(a_pVM); \ + } while (0) + +/** + * Get the Page ID. + * @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_PAGEID(a_pPage) ( (uint32_t)(a_pPage)->s.idPage ) + +/** + * Sets the Page ID. + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_idPage The new page ID. + */ +#define PGM_PAGE_SET_PAGEID(a_pVM, a_pPage, a_idPage) \ + do { \ + (a_pPage)->s.idPage = (a_idPage); \ + PGM_PAGE_ASSERT_LOCK(a_pVM); \ + } while (0) + +/** + * Get the Chunk ID. + * @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_CHUNKID(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) >> GMM_CHUNKID_SHIFT ) + +/** + * Get the index of the page within the allocation chunk. + * @returns The page index. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_PAGE_IN_CHUNK(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) & GMM_PAGEID_IDX_MASK ) + +/** + * Gets the page type. + * @returns The page type. + * @param a_pPage Pointer to the physical guest page tracking structure. + * + * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict + * builds. + */ +#define PGM_PAGE_GET_TYPE_NA(a_pPage) ( (a_pPage)->s.uTypeY ) +#if defined(__GNUC__) && defined(VBOX_STRICT) +# define PGM_PAGE_GET_TYPE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TYPE_NA(a_pPage); }) +#else +# define PGM_PAGE_GET_TYPE PGM_PAGE_GET_TYPE_NA +#endif + +/** + * Sets the page type. + * + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_enmType The new page type (PGMPAGETYPE). + */ +#define PGM_PAGE_SET_TYPE(a_pVM, a_pPage, a_enmType) \ + do { (a_pPage)->s.uTypeY = (a_enmType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** + * Gets the page table index + * @returns The page table index. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_PTE_INDEX(a_pPage) ( (a_pPage)->s.u10PteIdx ) + +/** + * Sets the page table index. + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_iPte New page table index. + */ +#define PGM_PAGE_SET_PTE_INDEX(a_pVM, a_pPage, a_iPte) \ + do { (a_pPage)->s.u10PteIdx = (a_iPte); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** + * Checks if the page is marked for MMIO, no MMIO2 aliasing. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO ) + +/** + * Checks if the page is marked for MMIO, including both aliases. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_MMIO_OR_ALIAS(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \ + || (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO2_ALIAS_MMIO \ + || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO \ + ) + +/** + * Checks if the page is marked for MMIO, including special aliases. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \ + || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO ) + +/** + * Checks if the page is a special aliased MMIO page. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO ) + +/** + * Checks if the page is backed by the ZERO page. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_ZERO(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ZERO ) + +/** + * Checks if the page is backed by a SHARED page. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_SHARED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_SHARED ) + +/** + * Checks if the page is ballooned. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_BALLOONED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_BALLOONED ) + +/** + * Checks if the page is allocated. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_ALLOCATED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ALLOCATED ) + +/** + * Marks the page as written to (for GMM change monitoring). + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_SET_WRITTEN_TO(a_pVM, a_pPage) \ + do { (a_pPage)->s.fWrittenToY = 1; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** + * Clears the written-to indicator. + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_CLEAR_WRITTEN_TO(a_pVM, a_pPage) \ + do { (a_pPage)->s.fWrittenToY = 0; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** + * Checks if the page was marked as written-to. + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_WRITTEN_TO(a_pPage) ( (a_pPage)->s.fWrittenToY ) + + +/** @name PT usage values (PGMPAGE::u2PDEType). + * + * @{ */ +/** Either as a PT or PDE. */ +#define PGM_PAGE_PDE_TYPE_DONTCARE 0 +/** Must use a page table to map the range. */ +#define PGM_PAGE_PDE_TYPE_PT 1 +/** Can use a page directory entry to map the continuous range. */ +#define PGM_PAGE_PDE_TYPE_PDE 2 +/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */ +#define PGM_PAGE_PDE_TYPE_PDE_DISABLED 3 +/** @} */ + +/** + * Set the PDE type of the page + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_uType PGM_PAGE_PDE_TYPE_*. + */ +#define PGM_PAGE_SET_PDE_TYPE(a_pVM, a_pPage, a_uType) \ + do { (a_pPage)->s.u2PDETypeY = (a_uType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** + * Checks if the page was marked being part of a large page + * @returns true/false. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_PDE_TYPE(a_pPage) ( (a_pPage)->s.u2PDETypeY ) + +/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY). + * + * @remarks The values are assigned in order of priority, so we can calculate + * the correct state for a page with different handlers installed. + * @{ */ +/** No handler installed. */ +#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0 +/** Monitoring is temporarily disabled. */ +#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1 +/** Write access is monitored. */ +#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2 +/** All access is monitored. */ +#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3 +/** @} */ + +/** + * Gets the physical access handler state of a page. + * @returns PGM_PAGE_HNDL_PHYS_STATE_* value. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) ( (a_pPage)->s.u2HandlerPhysStateY ) + +/** + * Sets the physical access handler state of a page. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_uState The new state value. + * @param a_fNotInHm The PGMPHYSHANDLER_F_NOT_HM bit. + */ +#define PGM_PAGE_SET_HNDL_PHYS_STATE(a_pPage, a_uState, a_fNotInHm) \ + do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); (a_pPage)->s.fHandlerPhysNotInHm = (a_fNotInHm); } while (0) + +/** + * Sets the physical access handler state of a page. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_uState The new state value. + */ +#define PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(a_pPage, a_uState) \ + do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); } while (0) + +/** + * Checks if the page has any physical access handlers, including temporarily disabled ones. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(a_pPage) \ + ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE ) + +/** + * Checks if the page has any active physical access handlers. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(a_pPage) \ + ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE ) + +/** + * Checks if the page has any access handlers, including temporarily disabled ones. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \ + ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE ) + +/** + * Checks if the page has any active access handlers. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \ + (PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE ) + +/** + * Checks if the page has any active access handlers catching all accesses. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \ + ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL ) + +/** @def PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM + * Checks if the physical handlers of the page should be ignored in shadow page + * tables and such. + * @returns true/false + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(a_pPage) ((a_pPage)->s.fHandlerPhysNotInHm) + +/** @def PGM_PAGE_GET_TRACKING + * Gets the packed shadow page pool tracking data associated with a guest page. + * @returns uint16_t containing the data. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_TRACKING_NA(a_pPage) ( (a_pPage)->s.u16TrackingY ) +#if defined(__GNUC__) && defined(VBOX_STRICT) +# define PGM_PAGE_GET_TRACKING(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TRACKING_NA(a_pPage); }) +#else +# define PGM_PAGE_GET_TRACKING PGM_PAGE_GET_TRACKING_NA +#endif + +/** @def PGM_PAGE_SET_TRACKING + * Sets the packed shadow page pool tracking data associated with a guest page. + * @param a_pVM The VM handle, only used for lock ownership assertions. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_u16TrackingData The tracking data to store. + */ +#define PGM_PAGE_SET_TRACKING(a_pVM, a_pPage, a_u16TrackingData) \ + do { (a_pPage)->s.u16TrackingY = (a_u16TrackingData); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) + +/** @def PGM_PAGE_GET_TD_CREFS + * Gets the @a cRefs tracking data member. + * @returns cRefs. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_TD_CREFS(a_pPage) \ + ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK) +#define PGM_PAGE_GET_TD_CREFS_NA(a_pPage) \ + ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK) + +/** @def PGM_PAGE_GET_TD_IDX + * Gets the @a idx tracking data member. + * @returns idx. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_TD_IDX(a_pPage) \ + ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK) +#define PGM_PAGE_GET_TD_IDX_NA(a_pPage) \ + ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK) + + +/** Max number of locks on a page. */ +#define PGM_PAGE_MAX_LOCKS UINT8_C(254) + +/** Get the read lock count. + * @returns count. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_READ_LOCKS(a_pPage) ( (a_pPage)->s.cReadLocksY ) + +/** Get the write lock count. + * @returns count. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_WRITE_LOCKS(a_pPage) ( (a_pPage)->s.cWriteLocksY ) + +/** Decrement the read lock counter. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_DEC_READ_LOCKS(a_pPage) do { --(a_pPage)->s.cReadLocksY; } while (0) + +/** Decrement the write lock counter. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_DEC_WRITE_LOCKS(a_pPage) do { --(a_pPage)->s.cWriteLocksY; } while (0) + +/** Increment the read lock counter. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_INC_READ_LOCKS(a_pPage) do { ++(a_pPage)->s.cReadLocksY; } while (0) + +/** Increment the write lock counter. + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_INC_WRITE_LOCKS(a_pPage) do { ++(a_pPage)->s.cWriteLocksY; } while (0) + + +/** Gets the NEM state. + * @returns NEM state value (two bits). + * @param a_pPage Pointer to the physical guest page tracking structure. + */ +#define PGM_PAGE_GET_NEM_STATE(a_pPage) ((a_pPage)->s.u2NemStateY) + +/** Sets the NEM state. + * @param a_pPage Pointer to the physical guest page tracking structure. + * @param a_u2State The NEM state value (specific to NEM impl.). + */ +#define PGM_PAGE_SET_NEM_STATE(a_pPage, a_u2State) \ + do { Assert((a_u2State) < 4); (a_pPage)->s.u2NemStateY = (a_u2State); } while (0) + + +#if 0 +/** Enables sanity checking of write monitoring using CRC-32. */ +# define PGMLIVESAVERAMPAGE_WITH_CRC32 +#endif + +/** + * Per page live save tracking data. + */ +typedef struct PGMLIVESAVERAMPAGE +{ + /** Number of times it has been dirtied. */ + uint32_t cDirtied : 24; + /** Whether it is currently dirty. */ + uint32_t fDirty : 1; + /** Ignore the page. + * This is used for pages that has been MMIO, MMIO2 or ROM pages once. We will + * deal with these after pausing the VM and DevPCI have said it bit about + * remappings. */ + uint32_t fIgnore : 1; + /** Was a ZERO page last time around. */ + uint32_t fZero : 1; + /** Was a SHARED page last time around. */ + uint32_t fShared : 1; + /** Whether the page is/was write monitored in a previous pass. */ + uint32_t fWriteMonitored : 1; + /** Whether the page is/was write monitored earlier in this pass. */ + uint32_t fWriteMonitoredJustNow : 1; + /** Bits reserved for future use. */ + uint32_t u2Reserved : 2; +#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32 + /** CRC-32 for the page. This is for internal consistency checks. */ + uint32_t u32Crc; +#endif +} PGMLIVESAVERAMPAGE; +#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32 +AssertCompileSize(PGMLIVESAVERAMPAGE, 8); +#else +AssertCompileSize(PGMLIVESAVERAMPAGE, 4); +#endif +/** Pointer to the per page live save tracking data. */ +typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE; + +/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */ +#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0 + + +/** + * RAM range for GC Phys to HC Phys conversion. + * + * Can be used for HC Virt to GC Phys and HC Virt to HC Phys + * conversions too, but we'll let MM handle that for now. + * + * This structure is used by linked lists in both GC and HC. + */ +typedef struct PGMRAMRANGE +{ + /** Start of the range. Page aligned. */ + RTGCPHYS GCPhys; + /** Size of the range. (Page aligned of course). */ + RTGCPHYS cb; + /** Pointer to the next RAM range - for R3. */ + R3PTRTYPE(struct PGMRAMRANGE *) pNextR3; + /** Pointer to the next RAM range - for R0. */ + R0PTRTYPE(struct PGMRAMRANGE *) pNextR0; + /** PGM_RAM_RANGE_FLAGS_* flags. */ + uint32_t fFlags; + /** NEM specific info, UINT32_MAX if not used. */ + uint32_t uNemRange; + /** Last address in the range (inclusive). Page aligned (-1). */ + RTGCPHYS GCPhysLast; + /** Start of the HC mapping of the range. This is only used for MMIO2 and in NEM mode. */ + R3PTRTYPE(void *) pvR3; + /** Live save per page tracking data. */ + R3PTRTYPE(PPGMLIVESAVERAMPAGE) paLSPages; + /** The range description. */ + R3PTRTYPE(const char *) pszDesc; + /** Pointer to self - R0 pointer. */ + R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0; + + /** Pointer to the left search three node - ring-3 context. */ + R3PTRTYPE(struct PGMRAMRANGE *) pLeftR3; + /** Pointer to the right search three node - ring-3 context. */ + R3PTRTYPE(struct PGMRAMRANGE *) pRightR3; + /** Pointer to the left search three node - ring-0 context. */ + R0PTRTYPE(struct PGMRAMRANGE *) pLeftR0; + /** Pointer to the right search three node - ring-0 context. */ + R0PTRTYPE(struct PGMRAMRANGE *) pRightR0; + + /** Padding to make aPage aligned on sizeof(PGMPAGE). */ +#if HC_ARCH_BITS == 32 + uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 0]; +#endif + /** Array of physical guest page tracking structures. + * @note Number of entries is PGMRAMRANGE::cb / GUEST_PAGE_SIZE. */ + PGMPAGE aPages[1]; +} PGMRAMRANGE; +/** Pointer to RAM range for GC Phys to HC Phys conversion. */ +typedef PGMRAMRANGE *PPGMRAMRANGE; + +/** @name PGMRAMRANGE::fFlags + * @{ */ +/** The RAM range is floating around as an independent guest mapping. */ +#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20) +/** Ad hoc RAM range for an ROM mapping. */ +#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM RT_BIT(21) +/** Ad hoc RAM range for an MMIO mapping. */ +#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO RT_BIT(22) +/** Ad hoc RAM range for an MMIO2 or pre-registered MMIO mapping. */ +#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX RT_BIT(23) +/** @} */ + +/** Tests if a RAM range is an ad hoc one or not. + * @returns true/false. + * @param pRam The RAM range. + */ +#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \ + (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX) ) ) + +/** The number of entries in the RAM range TLBs (there is one for each + * context). Must be a power of two. */ +#define PGM_RAMRANGE_TLB_ENTRIES 8 + +/** + * Calculates the RAM range TLB index for the physical address. + * + * @returns RAM range TLB index. + * @param a_GCPhys The guest physical address. + */ +#define PGM_RAMRANGE_TLB_IDX(a_GCPhys) ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) ) + +/** + * Calculates the ring-3 address for a_GCPhysPage if the RAM range has a + * mapping address. + */ +#define PGM_RAMRANGE_CALC_PAGE_R3PTR(a_pRam, a_GCPhysPage) \ + ( (a_pRam)->pvR3 ? (R3PTRTYPE(uint8_t *))(a_pRam)->pvR3 + (a_GCPhysPage) - (a_pRam)->GCPhys : NULL ) + + +/** + * Per page tracking structure for ROM image. + * + * A ROM image may have a shadow page, in which case we may have two pages + * backing it. This structure contains the PGMPAGE for both while + * PGMRAMRANGE have a copy of the active one. It is important that these + * aren't out of sync in any regard other than page pool tracking data. + */ +typedef struct PGMROMPAGE +{ + /** The page structure for the virgin ROM page. */ + PGMPAGE Virgin; + /** The page structure for the shadow RAM page. */ + PGMPAGE Shadow; + /** The current protection setting. */ + PGMROMPROT enmProt; + /** Live save status information. Makes use of unused alignment space. */ + struct + { + /** The previous protection value. */ + uint8_t u8Prot; + /** Written to flag set by the handler. */ + bool fWrittenTo; + /** Whether the shadow page is dirty or not. */ + bool fDirty; + /** Whether it was dirtied in the recently. */ + bool fDirtiedRecently; + } LiveSave; +} PGMROMPAGE; +AssertCompileSizeAlignment(PGMROMPAGE, 8); +/** Pointer to a ROM page tracking structure. */ +typedef PGMROMPAGE *PPGMROMPAGE; + + +/** + * A registered ROM image. + * + * This is needed to keep track of ROM image since they generally intrude + * into a PGMRAMRANGE. It also keeps track of additional info like the + * two page sets (read-only virgin and read-write shadow), the current + * state of each page. + * + * Because access handlers cannot easily be executed in a different + * context, the ROM ranges needs to be accessible and in all contexts. + */ +typedef struct PGMROMRANGE +{ + /** Pointer to the next range - R3. */ + R3PTRTYPE(struct PGMROMRANGE *) pNextR3; + /** Pointer to the next range - R0. */ + R0PTRTYPE(struct PGMROMRANGE *) pNextR0; + /** Pointer to the this range - R0. */ + R0PTRTYPE(struct PGMROMRANGE *) pSelfR0; + /** Address of the range. */ + RTGCPHYS GCPhys; + /** Address of the last byte in the range. */ + RTGCPHYS GCPhysLast; + /** Size of the range. */ + RTGCPHYS cb; + /** The flags (PGMPHYS_ROM_FLAGS_*). */ + uint8_t fFlags; + /** The saved state range ID. */ + uint8_t idSavedState; + /** Alignment padding. */ + uint8_t au8Alignment[2]; + /** The size bits pvOriginal points to. */ + uint32_t cbOriginal; + /** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified. + * This is used for strictness checks. */ + R3PTRTYPE(const void *) pvOriginal; + /** The ROM description. */ + R3PTRTYPE(const char *) pszDesc; +#ifdef VBOX_WITH_PGM_NEM_MODE + /** In simplified memory mode this provides alternate backing for shadowed ROMs. + * - PGMROMPROT_READ_ROM_WRITE_IGNORE: Shadow + * - PGMROMPROT_READ_ROM_WRITE_RAM: Shadow + * - PGMROMPROT_READ_RAM_WRITE_IGNORE: ROM + * - PGMROMPROT_READ_RAM_WRITE_RAM: ROM */ + R3PTRTYPE(uint8_t *) pbR3Alternate; + RTR3PTR pvAlignment2; +#endif + /** The per page tracking structures. */ + PGMROMPAGE aPages[1]; +} PGMROMRANGE; +/** Pointer to a ROM range. */ +typedef PGMROMRANGE *PPGMROMRANGE; + + +/** + * Live save per page data for an MMIO2 page. + * + * Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring + * of MMIO2 pages. The current approach is using some optimistic SHA-1 + + * CRC-32 for detecting changes as well as special handling of zero pages. This + * is a TEMPORARY measure which isn't perfect, but hopefully it is good enough + * for speeding things up. (We're using SHA-1 and not SHA-256 or SHA-512 + * because of speed (2.5x and 6x slower).) + * + * @todo Implement dirty MMIO2 page reporting that can be enabled during live + * save but normally is disabled. Since we can write monitor guest + * accesses on our own, we only need this for host accesses. Shouldn't be + * too difficult for DevVGA, VMMDev might be doable, the planned + * networking fun will be fun since it involves ring-0. + */ +typedef struct PGMLIVESAVEMMIO2PAGE +{ + /** Set if the page is considered dirty. */ + bool fDirty; + /** The number of scans this page has remained unchanged for. + * Only updated for dirty pages. */ + uint8_t cUnchangedScans; + /** Whether this page was zero at the last scan. */ + bool fZero; + /** Alignment padding. */ + bool fReserved; + /** CRC-32 for the first half of the page. + * This is used together with u32CrcH2 to quickly detect changes in the page + * during the non-final passes. */ + uint32_t u32CrcH1; + /** CRC-32 for the second half of the page. */ + uint32_t u32CrcH2; + /** SHA-1 for the saved page. + * This is used in the final pass to skip pages without changes. */ + uint8_t abSha1Saved[RTSHA1_HASH_SIZE]; +} PGMLIVESAVEMMIO2PAGE; +/** Pointer to a live save status data for an MMIO2 page. */ +typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE; + +/** + * A registered MMIO2 (= Device RAM) range. + * + * There are a few reason why we need to keep track of these registrations. One + * of them is the deregistration & cleanup stuff, while another is that the + * PGMRAMRANGE associated with such a region may have to be removed from the ram + * range list. + * + * Overlapping with a RAM range has to be 100% or none at all. The pages in the + * existing RAM range must not be ROM nor MMIO. A guru meditation will be + * raised if a partial overlap or an overlap of ROM pages is encountered. On an + * overlap we will free all the existing RAM pages and put in the ram range + * pages instead. + */ +typedef struct PGMREGMMIO2RANGE +{ + /** The owner of the range. (a device) */ + PPDMDEVINSR3 pDevInsR3; + /** Pointer to the ring-3 mapping of the allocation. */ + RTR3PTR pvR3; +#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM + /** Pointer to the ring-0 mapping of the allocation. */ + RTR0PTR pvR0; +#endif + /** Pointer to the next range - R3. */ + R3PTRTYPE(struct PGMREGMMIO2RANGE *) pNextR3; + /** Flags (PGMREGMMIO2RANGE_F_XXX). */ + uint16_t fFlags; + /** The sub device number (internal PCI config (CFGM) number). */ + uint8_t iSubDev; + /** The PCI region number. */ + uint8_t iRegion; + /** The saved state range ID. */ + uint8_t idSavedState; + /** MMIO2 range identifier, for page IDs (PGMPAGE::s.idPage). */ + uint8_t idMmio2; + /** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */ +#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM + uint8_t abAlignment[HC_ARCH_BITS == 32 ? 6 + 4 : 2]; +#else + uint8_t abAlignment[HC_ARCH_BITS == 32 ? 6 + 8 : 2 + 8]; +#endif + /** The real size. + * This may be larger than indicated by RamRange.cb if the range has been + * reduced during saved state loading. */ + RTGCPHYS cbReal; + /** Pointer to the physical handler for MMIO. + * If NEM is responsible for tracking dirty pages in simple memory mode, this + * will be NULL. */ + R3PTRTYPE(PPGMPHYSHANDLER) pPhysHandlerR3; + /** Live save per page tracking data for MMIO2. */ + R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE) paLSPages; + /** The associated RAM range. */ + PGMRAMRANGE RamRange; +} PGMREGMMIO2RANGE; +AssertCompileMemberAlignment(PGMREGMMIO2RANGE, RamRange, 16); +/** Pointer to a MMIO2 or pre-registered MMIO range. */ +typedef PGMREGMMIO2RANGE *PPGMREGMMIO2RANGE; + +/** @name PGMREGMMIO2RANGE_F_XXX - Registered MMIO2 range flags. + * @{ */ +/** Set if this is the first chunk in the MMIO2 range. */ +#define PGMREGMMIO2RANGE_F_FIRST_CHUNK UINT16_C(0x0001) +/** Set if this is the last chunk in the MMIO2 range. */ +#define PGMREGMMIO2RANGE_F_LAST_CHUNK UINT16_C(0x0002) +/** Set if the whole range is mapped. */ +#define PGMREGMMIO2RANGE_F_MAPPED UINT16_C(0x0004) +/** Set if it's overlapping, clear if not. */ +#define PGMREGMMIO2RANGE_F_OVERLAPPING UINT16_C(0x0008) +/** This mirrors the PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES creation flag.*/ +#define PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES UINT16_C(0x0010) +/** Set if the access handler is registered. */ +#define PGMREGMMIO2RANGE_F_IS_TRACKING UINT16_C(0x0020) +/** Set if dirty page tracking is currently enabled. */ +#define PGMREGMMIO2RANGE_F_TRACKING_ENABLED UINT16_C(0x0040) +/** Set if there are dirty pages in the range. */ +#define PGMREGMMIO2RANGE_F_IS_DIRTY UINT16_C(0x0080) +/** @} */ + + +/** @name Internal MMIO2 constants. + * @{ */ +/** The maximum number of MMIO2 ranges. */ +#define PGM_MMIO2_MAX_RANGES 32 +/** The maximum number of pages in a MMIO2 range. */ +#define PGM_MMIO2_MAX_PAGE_COUNT UINT32_C(0x01000000) +/** Makes a MMIO2 page ID out of a MMIO2 range ID and page index number. */ +#define PGM_MMIO2_PAGEID_MAKE(a_idMmio2, a_iPage) ( ((uint32_t)(a_idMmio2) << 24) | (uint32_t)(a_iPage) ) +/** Gets the MMIO2 range ID from an MMIO2 page ID. */ +#define PGM_MMIO2_PAGEID_GET_MMIO2_ID(a_idPage) ( (uint8_t)((a_idPage) >> 24) ) +/** Gets the MMIO2 page index from an MMIO2 page ID. */ +#define PGM_MMIO2_PAGEID_GET_IDX(a_idPage) ( ((a_idPage) & UINT32_C(0x00ffffff)) ) +/** @} */ + + + +/** + * PGMPhysRead/Write cache entry + */ +typedef struct PGMPHYSCACHEENTRY +{ + /** R3 pointer to physical page. */ + R3PTRTYPE(uint8_t *) pbR3; + /** GC Physical address for cache entry */ + RTGCPHYS GCPhys; +#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32 + RTGCPHYS u32Padding0; /**< alignment padding. */ +#endif +} PGMPHYSCACHEENTRY; + +/** + * PGMPhysRead/Write cache to reduce REM memory access overhead + */ +typedef struct PGMPHYSCACHE +{ + /** Bitmap of valid cache entries */ + uint64_t aEntries; + /** Cache entries */ + PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES]; +} PGMPHYSCACHE; + + +/** @name Ring-3 page mapping TLBs + * @{ */ + +/** Pointer to an allocation chunk ring-3 mapping. */ +typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP; +/** Pointer to an allocation chunk ring-3 mapping pointer. */ +typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP; + +/** + * Ring-3 tracking structure for an allocation chunk ring-3 mapping. + * + * The primary tree (Core) uses the chunk id as key. + */ +typedef struct PGMCHUNKR3MAP +{ + /** The key is the chunk id. */ + AVLU32NODECORE Core; + /** The time (ChunkR3Map.iNow) this chunk was last used. Used for unmap + * selection. */ + uint32_t iLastUsed; + /** The current reference count. */ + uint32_t volatile cRefs; + /** The current permanent reference count. */ + uint32_t volatile cPermRefs; + /** The mapping address. */ + void *pv; +} PGMCHUNKR3MAP; + +/** + * Allocation chunk ring-3 mapping TLB entry. + */ +typedef struct PGMCHUNKR3MAPTLBE +{ + /** The chunk id. */ + uint32_t volatile idChunk; +#if HC_ARCH_BITS == 64 + uint32_t u32Padding; /**< alignment padding. */ +#endif + /** The chunk map. */ + R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk; +} PGMCHUNKR3MAPTLBE; +/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */ +typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE; + +/** The number of TLB entries in PGMCHUNKR3MAPTLB. + * @remark Must be a power of two value. */ +#define PGM_CHUNKR3MAPTLB_ENTRIES 64 + +/** + * Allocation chunk ring-3 mapping TLB. + * + * @remarks We use a TLB to speed up lookups by avoiding walking the AVL. + * At first glance this might look kinda odd since AVL trees are + * supposed to give the most optimal lookup times of all trees + * due to their balancing. However, take a tree with 1023 nodes + * in it, that's 10 levels, meaning that most searches has to go + * down 9 levels before they find what they want. This isn't fast + * compared to a TLB hit. There is the factor of cache misses, + * and of course the problem with trees and branch prediction. + * This is why we use TLBs in front of most of the trees. + * + * @todo Generalize this TLB + AVL stuff, shouldn't be all that + * difficult when we switch to the new inlined AVL trees (from kStuff). + */ +typedef struct PGMCHUNKR3MAPTLB +{ + /** The TLB entries. */ + PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES]; +} PGMCHUNKR3MAPTLB; + +/** + * Calculates the index of a guest page in the Ring-3 Chunk TLB. + * @returns Chunk TLB index. + * @param idChunk The Chunk ID. + */ +#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) ) + + +/** + * Ring-3 guest page mapping TLB entry. + * @remarks used in ring-0 as well at the moment. + */ +typedef struct PGMPAGER3MAPTLBE +{ + /** Address of the page. */ + RTGCPHYS volatile GCPhys; + /** The guest page. */ + R3PTRTYPE(PPGMPAGE) volatile pPage; + /** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */ + R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap; + /** The address */ + R3PTRTYPE(void *) volatile pv; +#if HC_ARCH_BITS == 32 + uint32_t u32Padding; /**< alignment padding. */ +#endif +} PGMPAGER3MAPTLBE; +/** Pointer to an entry in the HC physical TLB. */ +typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE; + + +/** The number of entries in the ring-3 guest page mapping TLB. + * @remarks The value must be a power of two. */ +#define PGM_PAGER3MAPTLB_ENTRIES 256 + +/** + * Ring-3 guest page mapping TLB. + * @remarks used in ring-0 as well at the moment. + */ +typedef struct PGMPAGER3MAPTLB +{ + /** The TLB entries. */ + PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES]; +} PGMPAGER3MAPTLB; +/** Pointer to the ring-3 guest page mapping TLB. */ +typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB; + +/** + * Calculates the index of the TLB entry for the specified guest page. + * @returns Physical TLB index. + * @param GCPhys The guest physical address. + */ +#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> GUEST_PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) ) + +/** @} */ + + +/** @name Ring-0 page mapping TLB + * @{ */ +/** + * Ring-0 guest page mapping TLB entry. + */ +typedef struct PGMPAGER0MAPTLBE +{ + /** Address of the page. */ + RTGCPHYS volatile GCPhys; + /** The guest page. */ + R0PTRTYPE(PPGMPAGE) volatile pPage; + /** The address */ + R0PTRTYPE(void *) volatile pv; +} PGMPAGER0MAPTLBE; +/** Pointer to an entry in the HC physical TLB. */ +typedef PGMPAGER0MAPTLBE *PPGMPAGER0MAPTLBE; + + +/** The number of entries in the ring-3 guest page mapping TLB. + * @remarks The value must be a power of two. */ +#define PGM_PAGER0MAPTLB_ENTRIES 256 + +/** + * Ring-3 guest page mapping TLB. + * @remarks used in ring-0 as well at the moment. + */ +typedef struct PGMPAGER0MAPTLB +{ + /** The TLB entries. */ + PGMPAGER0MAPTLBE aEntries[PGM_PAGER0MAPTLB_ENTRIES]; +} PGMPAGER0MAPTLB; +/** Pointer to the ring-3 guest page mapping TLB. */ +typedef PGMPAGER0MAPTLB *PPGMPAGER0MAPTLB; + +/** + * Calculates the index of the TLB entry for the specified guest page. + * @returns Physical TLB index. + * @param GCPhys The guest physical address. + */ +#define PGM_PAGER0MAPTLB_IDX(GCPhys) ( ((GCPhys) >> GUEST_PAGE_SHIFT) & (PGM_PAGER0MAPTLB_ENTRIES - 1) ) +/** @} */ + + +/** @name Context neutral page mapper TLB. + * + * Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr + * code is writting in a kind of context neutral way. Time will show whether + * this actually makes sense or not... + * + * @todo this needs to be reconsidered and dropped/redone since the ring-0 + * context ends up using a global mapping cache on some platforms + * (darwin). + * + * @{ */ +/** @typedef PPGMPAGEMAPTLB + * The page mapper TLB pointer type for the current context. */ +/** @typedef PPGMPAGEMAPTLB + * The page mapper TLB entry pointer type for the current context. */ +/** @typedef PPGMPAGEMAPTLB + * The page mapper TLB entry pointer pointer type for the current context. */ +/** @def PGM_PAGEMAPTLB_ENTRIES + * The number of TLB entries in the page mapper TLB for the current context. */ +/** @def PGM_PAGEMAPTLB_IDX + * Calculate the TLB index for a guest physical address. + * @returns The TLB index. + * @param GCPhys The guest physical address. */ +/** @typedef PPGMPAGEMAP + * Pointer to a page mapper unit for current context. */ +/** @typedef PPPGMPAGEMAP + * Pointer to a page mapper unit pointer for current context. */ +#if defined(IN_RING0) +typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB; +typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE; +typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE; +# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES +# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys) +typedef struct PGMCHUNKR0MAP *PPGMPAGEMAP; +typedef struct PGMCHUNKR0MAP **PPPGMPAGEMAP; +#else +typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB; +typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE; +typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE; +# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES +# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys) +typedef PPGMCHUNKR3MAP PPGMPAGEMAP; +typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP; +#endif +/** @} */ + + +/** @name PGM Pool Indexes. + * Aka. the unique shadow page identifier. + * @{ */ +/** NIL page pool IDX. */ +#define NIL_PGMPOOL_IDX 0 +/** The first normal index. There used to be 5 fictive pages up front, now + * there is only the NIL page. */ +#define PGMPOOL_IDX_FIRST 1 +/** The last valid index. (inclusive, 14 bits) */ +#define PGMPOOL_IDX_LAST 0x3fff +/** @} */ + +/** The NIL index for the parent chain. */ +#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff) +#define NIL_PGMPOOL_PRESENT_INDEX ((uint16_t)0xffff) + +/** + * Node in the chain linking a shadowed page to it's parent (user). + */ +#pragma pack(1) +typedef struct PGMPOOLUSER +{ + /** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */ + uint16_t iNext; + /** The user page index. */ + uint16_t iUser; + /** Index into the user table. */ + uint32_t iUserTable; +} PGMPOOLUSER, *PPGMPOOLUSER; +typedef const PGMPOOLUSER *PCPGMPOOLUSER; +#pragma pack() + + +/** The NIL index for the phys ext chain. */ +#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff) +/** The NIL pte index for a phys ext chain slot. */ +#define NIL_PGMPOOL_PHYSEXT_IDX_PTE ((uint16_t)0xffff) + +/** + * Node in the chain of physical cross reference extents. + * @todo Calling this an 'extent' is not quite right, find a better name. + * @todo find out the optimal size of the aidx array + */ +#pragma pack(1) +typedef struct PGMPOOLPHYSEXT +{ + /** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */ + uint16_t iNext; + /** Alignment. */ + uint16_t u16Align; + /** The user page index. */ + uint16_t aidx[3]; + /** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */ + uint16_t apte[3]; +} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT; +typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT; +#pragma pack() + + +/** + * The kind of page that's being shadowed. + */ +typedef enum PGMPOOLKIND +{ + /** The virtual invalid 0 entry. */ + PGMPOOLKIND_INVALID = 0, + /** The entry is free (=unused). */ + PGMPOOLKIND_FREE, + + /** Shw: 32-bit page table; Gst: no paging. */ + PGMPOOLKIND_32BIT_PT_FOR_PHYS, + /** Shw: 32-bit page table; Gst: 32-bit page table. */ + PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT, + /** Shw: 32-bit page table; Gst: 4MB page. */ + PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB, + /** Shw: PAE page table; Gst: no paging. */ + PGMPOOLKIND_PAE_PT_FOR_PHYS, + /** Shw: PAE page table; Gst: 32-bit page table. */ + PGMPOOLKIND_PAE_PT_FOR_32BIT_PT, + /** Shw: PAE page table; Gst: Half of a 4MB page. */ + PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB, + /** Shw: PAE page table; Gst: PAE page table. */ + PGMPOOLKIND_PAE_PT_FOR_PAE_PT, + /** Shw: PAE page table; Gst: 2MB page. */ + PGMPOOLKIND_PAE_PT_FOR_PAE_2MB, + + /** Shw: 32-bit page directory. Gst: 32-bit page directory. */ + PGMPOOLKIND_32BIT_PD, + /** Shw: 32-bit page directory. Gst: no paging. */ + PGMPOOLKIND_32BIT_PD_PHYS, + /** Shw: PAE page directory 0; Gst: 32-bit page directory. */ + PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD, + /** Shw: PAE page directory 1; Gst: 32-bit page directory. */ + PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD, + /** Shw: PAE page directory 2; Gst: 32-bit page directory. */ + PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD, + /** Shw: PAE page directory 3; Gst: 32-bit page directory. */ + PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD, + /** Shw: PAE page directory; Gst: PAE page directory. */ + PGMPOOLKIND_PAE_PD_FOR_PAE_PD, + /** Shw: PAE page directory; Gst: no paging. Note: +NP. */ + PGMPOOLKIND_PAE_PD_PHYS, + + /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */ + PGMPOOLKIND_PAE_PDPT_FOR_32BIT, + /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */ + PGMPOOLKIND_PAE_PDPT, + /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */ + PGMPOOLKIND_PAE_PDPT_PHYS, + + /** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */ + PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT, + /** Shw: 64-bit page directory pointer table; Gst: no paging. */ + PGMPOOLKIND_64BIT_PDPT_FOR_PHYS, + /** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */ + PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD, + /** Shw: 64-bit page directory table; Gst: no paging. */ + PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 24 */ + + /** Shw: 64-bit PML4; Gst: 64-bit PML4. */ + PGMPOOLKIND_64BIT_PML4, + + /** Shw: EPT page directory pointer table; Gst: no paging. */ + PGMPOOLKIND_EPT_PDPT_FOR_PHYS, + /** Shw: EPT page directory table; Gst: no paging. */ + PGMPOOLKIND_EPT_PD_FOR_PHYS, + /** Shw: EPT page table; Gst: no paging. */ + PGMPOOLKIND_EPT_PT_FOR_PHYS, + + /** Shw: Root Nested paging table. */ + PGMPOOLKIND_ROOT_NESTED, + + /** Shw: EPT page table; Gst: EPT page table. */ + PGMPOOLKIND_EPT_PT_FOR_EPT_PT, + /** Shw: EPT page table; Gst: 2MB page. */ + PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, + /** Shw: EPT page directory table; Gst: EPT page directory. */ + PGMPOOLKIND_EPT_PD_FOR_EPT_PD, + /** Shw: EPT page directory pointer table; Gst: EPT page directory pointer table. */ + PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, + /** Shw: EPT PML4; Gst: EPT PML4. */ + PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4, + + /** The last valid entry. */ + PGMPOOLKIND_LAST = PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4 +} PGMPOOLKIND; + +/** + * The access attributes of the page; only applies to big pages. + */ +typedef enum +{ + PGMPOOLACCESS_DONTCARE = 0, + PGMPOOLACCESS_USER_RW, + PGMPOOLACCESS_USER_R, + PGMPOOLACCESS_USER_RW_NX, + PGMPOOLACCESS_USER_R_NX, + PGMPOOLACCESS_SUPERVISOR_RW, + PGMPOOLACCESS_SUPERVISOR_R, + PGMPOOLACCESS_SUPERVISOR_RW_NX, + PGMPOOLACCESS_SUPERVISOR_R_NX +} PGMPOOLACCESS; + +/** + * The tracking data for a page in the pool. + */ +typedef struct PGMPOOLPAGE +{ + /** AVL node code with the (HC) physical address of this page. */ + AVLOHCPHYSNODECORE Core; + /** Pointer to the R3 mapping of the page. */ + R3PTRTYPE(void *) pvPageR3; + /** Pointer to the R0 mapping of the page. */ + R0PTRTYPE(void *) pvPageR0; + /** The guest physical address. */ + RTGCPHYS GCPhys; + /** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */ + uint8_t enmKind; + /** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */ + uint8_t enmAccess; + /** This supplements enmKind and enmAccess */ + bool fA20Enabled : 1; + + /** Used to indicate that the page is zeroed. */ + bool fZeroed : 1; + /** Used to indicate that a PT has non-global entries. */ + bool fSeenNonGlobal : 1; + /** Used to indicate that we're monitoring writes to the guest page. */ + bool fMonitored : 1; + /** Used to indicate that the page is in the cache (e.g. in the GCPhys hash). + * (All pages are in the age list.) */ + bool fCached : 1; + /** This is used by the R3 access handlers when invoked by an async thread. + * It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */ + bool volatile fReusedFlushPending : 1; + /** Used to mark the page as dirty (write monitoring is temporarily + * off). */ + bool fDirty : 1; + bool fPadding1 : 1; + bool fPadding2; + + /** The index of this page. */ + uint16_t idx; + /** The next entry in the list this page currently resides in. + * It's either in the free list or in the GCPhys hash. */ + uint16_t iNext; + /** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */ + uint16_t iUserHead; + /** The number of present entries. */ + uint16_t cPresent; + /** The first entry in the table which is present. */ + uint16_t iFirstPresent; + /** The number of modifications to the monitored page. */ + uint16_t cModifications; + /** The next modified page. NIL_PGMPOOL_IDX if tail. */ + uint16_t iModifiedNext; + /** The previous modified page. NIL_PGMPOOL_IDX if head. */ + uint16_t iModifiedPrev; + /** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */ + uint16_t iMonitoredNext; + /** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */ + uint16_t iMonitoredPrev; + /** The next page in the age list. */ + uint16_t iAgeNext; + /** The previous page in the age list. */ + uint16_t iAgePrev; + /** Index into PGMPOOL::aDirtyPages if fDirty is set. */ + uint8_t idxDirtyEntry; + + /** @name Access handler statistics to determine whether the guest is + * (re)initializing a page table. + * @{ */ + RTGCPTR GCPtrLastAccessHandlerRip; + RTGCPTR GCPtrLastAccessHandlerFault; + uint64_t cLastAccessHandler; + /** @} */ + /** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages. */ + uint32_t volatile cLocked; +#if GC_ARCH_BITS == 64 + uint32_t u32Alignment3; +#endif +# ifdef VBOX_STRICT + RTGCPTR GCPtrDirtyFault; +# endif +} PGMPOOLPAGE; +/** Pointer to a pool page. */ +typedef PGMPOOLPAGE *PPGMPOOLPAGE; +/** Pointer to a const pool page. */ +typedef PGMPOOLPAGE const *PCPGMPOOLPAGE; +/** Pointer to a pool page pointer. */ +typedef PGMPOOLPAGE **PPPGMPOOLPAGE; + + +/** The hash table size. */ +# define PGMPOOL_HASH_SIZE 0x8000 +/** The hash function. */ +# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> GUEST_PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) ) + + +/** + * The shadow page pool instance data. + * + * It's all one big allocation made at init time, except for the + * pages that is. The user nodes follows immediately after the + * page structures. + */ +typedef struct PGMPOOL +{ + /** The VM handle - R3 Ptr. */ + PVMR3 pVMR3; + /** The VM handle - R0 Ptr. */ + R0PTRTYPE(PVMCC) pVMR0; + /** The ring-3 pointer to this structure. */ + R3PTRTYPE(struct PGMPOOL *) pPoolR3; + /** The ring-0 pointer to this structure. */ + R0PTRTYPE(struct PGMPOOL *) pPoolR0; + /** The max pool size. This includes the special IDs. */ + uint16_t cMaxPages; + /** The current pool size. */ + uint16_t cCurPages; + /** The head of the free page list. */ + uint16_t iFreeHead; + /* Padding. */ + uint16_t u16Padding; + /** Head of the chain of free user nodes. */ + uint16_t iUserFreeHead; + /** The number of user nodes we've allocated. */ + uint16_t cMaxUsers; + /** The number of present page table entries in the entire pool. */ + uint32_t cPresent; + /** Pointer to the array of user nodes - R3 pointer. */ + R3PTRTYPE(PPGMPOOLUSER) paUsersR3; + /** Pointer to the array of user nodes - R0 pointer. */ + R0PTRTYPE(PPGMPOOLUSER) paUsersR0; + /** Head of the chain of free phys ext nodes. */ + uint16_t iPhysExtFreeHead; + /** The number of user nodes we've allocated. */ + uint16_t cMaxPhysExts; + uint32_t u32Padding0b; + /** Pointer to the array of physical xref extent nodes - R3 pointer. */ + R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3; + /** Pointer to the array of physical xref extent nodes - R0 pointer. */ + R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0; + /** Hash table for GCPhys addresses. */ + uint16_t aiHash[PGMPOOL_HASH_SIZE]; + /** The head of the age list. */ + uint16_t iAgeHead; + /** The tail of the age list. */ + uint16_t iAgeTail; + /** Set if the cache is enabled. */ + bool fCacheEnabled; + /** Alignment padding. */ + bool afPadding1[3]; + /** Head of the list of modified pages. */ + uint16_t iModifiedHead; + /** The current number of modified pages. */ + uint16_t cModifiedPages; + /** Alignment padding. */ + uint32_t u32Padding2; + /** Physical access handler type registration handle. */ + PGMPHYSHANDLERTYPE hAccessHandlerType; + /** Next available slot (in aDirtyPages). */ + uint32_t idxFreeDirtyPage; + /** Number of active dirty pages. */ + uint32_t cDirtyPages; + /** Array of current dirty pgm pool page indices. */ + uint16_t aidxDirtyPages[16]; + /** Array running in parallel to aidxDirtyPages with the page data. */ + struct + { + uint64_t aPage[512]; + } aDirtyPages[16]; + + /** The number of pages currently in use. */ + uint16_t cUsedPages; +#ifdef VBOX_WITH_STATISTICS + /** The high water mark for cUsedPages. */ + uint16_t cUsedPagesHigh; + uint32_t Alignment1; /**< Align the next member on a 64-bit boundary. */ + /** Profiling pgmPoolAlloc(). */ + STAMPROFILEADV StatAlloc; + /** Profiling pgmR3PoolClearDoIt(). */ + STAMPROFILE StatClearAll; + /** Profiling pgmR3PoolReset(). */ + STAMPROFILE StatR3Reset; + /** Profiling pgmPoolFlushPage(). */ + STAMPROFILE StatFlushPage; + /** Profiling pgmPoolFree(). */ + STAMPROFILE StatFree; + /** Counting explicit flushes by PGMPoolFlushPage(). */ + STAMCOUNTER StatForceFlushPage; + /** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */ + STAMCOUNTER StatForceFlushDirtyPage; + /** Counting flushes for reused pages. */ + STAMCOUNTER StatForceFlushReused; + /** Profiling time spent zeroing pages. */ + STAMPROFILE StatZeroPage; + /** Profiling of pgmPoolTrackDeref. */ + STAMPROFILE StatTrackDeref; + /** Profiling pgmTrackFlushGCPhysPT. */ + STAMPROFILE StatTrackFlushGCPhysPT; + /** Profiling pgmTrackFlushGCPhysPTs. */ + STAMPROFILE StatTrackFlushGCPhysPTs; + /** Profiling pgmTrackFlushGCPhysPTsSlow. */ + STAMPROFILE StatTrackFlushGCPhysPTsSlow; + /** Number of times we've been out of user records. */ + STAMCOUNTER StatTrackFreeUpOneUser; + /** Nr of flushed entries. */ + STAMCOUNTER StatTrackFlushEntry; + /** Nr of updated entries. */ + STAMCOUNTER StatTrackFlushEntryKeep; + /** Profiling deref activity related tracking GC physical pages. */ + STAMPROFILE StatTrackDerefGCPhys; + /** Number of linear searches for a HCPhys in the ram ranges. */ + STAMCOUNTER StatTrackLinearRamSearches; + /** The number of failing pgmPoolTrackPhysExtAlloc calls. */ + STAMCOUNTER StamTrackPhysExtAllocFailures; + + /** Profiling the RC/R0 \#PF access handler. */ + STAMPROFILE StatMonitorPfRZ; + /** Profiling the RC/R0 access we've handled (except REP STOSD). */ + STAMPROFILE StatMonitorPfRZHandled; + /** Times we've failed interpreting the instruction. */ + STAMCOUNTER StatMonitorPfRZEmulateInstr; + /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */ + STAMPROFILE StatMonitorPfRZFlushPage; + /** Times we've detected a page table reinit. */ + STAMCOUNTER StatMonitorPfRZFlushReinit; + /** Counting flushes for pages that are modified too often. */ + STAMCOUNTER StatMonitorPfRZFlushModOverflow; + /** Times we've detected fork(). */ + STAMCOUNTER StatMonitorPfRZFork; + /** Times we've failed interpreting a patch code instruction. */ + STAMCOUNTER StatMonitorPfRZIntrFailPatch1; + /** Times we've failed interpreting a patch code instruction during flushing. */ + STAMCOUNTER StatMonitorPfRZIntrFailPatch2; + /** The number of times we've seen rep prefixes we can't handle. */ + STAMCOUNTER StatMonitorPfRZRepPrefix; + /** Profiling the REP STOSD cases we've handled. */ + STAMPROFILE StatMonitorPfRZRepStosd; + + /** Profiling the R0/RC regular access handler. */ + STAMPROFILE StatMonitorRZ; + /** Profiling the pgmPoolFlushPage calls made from the regular access handler in R0/RC. */ + STAMPROFILE StatMonitorRZFlushPage; + /** Per access size counts indexed by size minus 1, last for larger. */ + STAMCOUNTER aStatMonitorRZSizes[16+3]; + /** Missaligned access counts indexed by offset - 1. */ + STAMCOUNTER aStatMonitorRZMisaligned[7]; + + /** Nr of handled PT faults. */ + STAMCOUNTER StatMonitorRZFaultPT; + /** Nr of handled PD faults. */ + STAMCOUNTER StatMonitorRZFaultPD; + /** Nr of handled PDPT faults. */ + STAMCOUNTER StatMonitorRZFaultPDPT; + /** Nr of handled PML4 faults. */ + STAMCOUNTER StatMonitorRZFaultPML4; + + /** Profiling the R3 access handler. */ + STAMPROFILE StatMonitorR3; + /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */ + STAMPROFILE StatMonitorR3FlushPage; + /** Per access size counts indexed by size minus 1, last for larger. */ + STAMCOUNTER aStatMonitorR3Sizes[16+3]; + /** Missaligned access counts indexed by offset - 1. */ + STAMCOUNTER aStatMonitorR3Misaligned[7]; + /** Nr of handled PT faults. */ + STAMCOUNTER StatMonitorR3FaultPT; + /** Nr of handled PD faults. */ + STAMCOUNTER StatMonitorR3FaultPD; + /** Nr of handled PDPT faults. */ + STAMCOUNTER StatMonitorR3FaultPDPT; + /** Nr of handled PML4 faults. */ + STAMCOUNTER StatMonitorR3FaultPML4; + + /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */ + STAMCOUNTER StatResetDirtyPages; + /** Times we've called pgmPoolAddDirtyPage. */ + STAMCOUNTER StatDirtyPage; + /** Times we've had to flush duplicates for dirty page management. */ + STAMCOUNTER StatDirtyPageDupFlush; + /** Times we've had to flush because of overflow. */ + STAMCOUNTER StatDirtyPageOverFlowFlush; + + /** The high water mark for cModifiedPages. */ + uint16_t cModifiedPagesHigh; + uint16_t Alignment2[3]; /**< Align the next member on a 64-bit boundary. */ + + /** The number of cache hits. */ + STAMCOUNTER StatCacheHits; + /** The number of cache misses. */ + STAMCOUNTER StatCacheMisses; + /** The number of times we've got a conflict of 'kind' in the cache. */ + STAMCOUNTER StatCacheKindMismatches; + /** Number of times we've been out of pages. */ + STAMCOUNTER StatCacheFreeUpOne; + /** The number of cacheable allocations. */ + STAMCOUNTER StatCacheCacheable; + /** The number of uncacheable allocations. */ + STAMCOUNTER StatCacheUncacheable; +#else + uint32_t Alignment3; /**< Align the next member on a 64-bit boundary. */ +#endif + /** Profiling PGMR0PoolGrow(). */ + STAMPROFILE StatGrow; + /** The AVL tree for looking up a page by its HC physical address. */ + AVLOHCPHYSTREE HCPhysTree; + uint32_t Alignment4; /**< Align the next member on a 64-bit boundary. */ + /** Array of pages. (cMaxPages in length) + * The Id is the index into thist array. + */ + PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST]; +} PGMPOOL, *PPGMPOOL, **PPPGMPOOL; +AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8); +AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8); +AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8); +#ifdef VBOX_WITH_STATISTICS +AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8); +#endif +AssertCompileMemberAlignment(PGMPOOL, aPages, 8); + + +/** @def PGMPOOL_PAGE_2_PTR + * Maps a pool page pool into the current context. + * + * @returns VBox status code. + * @param a_pVM Pointer to the VM. + * @param a_pPage The pool page. + * + * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the + * small page window employeed by that function. Be careful. + * @remark There is no need to assert on the result. + */ +#if defined(VBOX_STRICT) || 1 /* temporarily going strict here */ +# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageStrict(a_pPage, __FUNCTION__) +DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE a_pPage, const char *pszCaller) +{ + RT_NOREF(pszCaller); + AssertPtr(a_pPage); + AssertMsg(RT_VALID_PTR(a_pPage->CTX_SUFF(pvPage)), + ("enmKind=%d idx=%#x HCPhys=%RHp GCPhys=%RGp pvPageR3=%p pvPageR0=%p caller=%s\n", + a_pPage->enmKind, a_pPage->idx, a_pPage->Core.Key, a_pPage->GCPhys, a_pPage->pvPageR3, a_pPage->pvPageR0, pszCaller)); + return a_pPage->CTX_SUFF(pvPage); +} +#else +# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage) ((a_pPage)->CTX_SUFF(pvPage)) +#endif + + +/** @def PGMPOOL_PAGE_2_PTR_V2 + * Maps a pool page pool into the current context, taking both VM and VMCPU. + * + * @returns VBox status code. + * @param a_pVM Pointer to the VM. + * @param a_pVCpu The current CPU. + * @param a_pPage The pool page. + * + * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the + * small page window employeed by that function. Be careful. + * @remark There is no need to assert on the result. + */ +#define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) PGMPOOL_PAGE_2_PTR((a_pVM), (a_pPage)) + + +/** @def PGMPOOL_PAGE_IS_NESTED + * Checks whether the given pool page is a nested-guest pool page. + * + * @returns @c true if a nested-guest pool page, @c false otherwise. + * @param a_pPage The pool page. + * @todo We can optimize the conditionals later. + */ +#define PGMPOOL_PAGE_IS_NESTED(a_pPage) PGMPOOL_PAGE_IS_KIND_NESTED((a_pPage)->enmKind) +#define PGMPOOL_PAGE_IS_KIND_NESTED(a_enmKind) ( (a_enmKind) == PGMPOOLKIND_EPT_PT_FOR_EPT_PT \ + || (a_enmKind) == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB \ + || (a_enmKind) == PGMPOOLKIND_EPT_PD_FOR_EPT_PD \ + || (a_enmKind) == PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT \ + || (a_enmKind) == PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4) + +/** @name Per guest page tracking data. + * This is currently as a 16-bit word in the PGMPAGE structure, the idea though + * is to use more bits for it and split it up later on. But for now we'll play + * safe and change as little as possible. + * + * The 16-bit word has two parts: + * + * The first 14-bit forms the @a idx field. It is either the index of a page in + * the shadow page pool, or and index into the extent list. + * + * The 2 topmost bits makes up the @a cRefs field, which counts the number of + * shadow page pool references to the page. If cRefs equals + * PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent + * (misnomer) table and not the shadow page pool. + * + * See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set + * the 16-bit word. + * + * @{ */ +/** The shift count for getting to the cRefs part. */ +#define PGMPOOL_TD_CREFS_SHIFT 14 +/** The mask applied after shifting the tracking data down by + * PGMPOOL_TD_CREFS_SHIFT. */ +#define PGMPOOL_TD_CREFS_MASK 0x3 +/** The cRefs value used to indicate that the idx is the head of a + * physical cross reference list. */ +#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK +/** The shift used to get idx. */ +#define PGMPOOL_TD_IDX_SHIFT 0 +/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */ +#define PGMPOOL_TD_IDX_MASK 0x3fff +/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are + * simply too many mappings of this page. */ +#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK + +/** @def PGMPOOL_TD_MAKE + * Makes a 16-bit tracking data word. + * + * @returns tracking data. + * @param cRefs The @a cRefs field. Must be within bounds! + * @param idx The @a idx field. Must also be within bounds! */ +#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) | (idx) ) + +/** @def PGMPOOL_TD_GET_CREFS + * Get the @a cRefs field from a tracking data word. + * + * @returns The @a cRefs field + * @param u16 The tracking data word. + * @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a + * non-zero @a u16. */ +#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK ) + +/** @def PGMPOOL_TD_GET_IDX + * Get the @a idx field from a tracking data word. + * + * @returns The @a idx field + * @param u16 The tracking data word. */ +#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK ) +/** @} */ + + + +/** @name A20 gate macros + * @{ */ +#define PGM_WITH_A20 +#ifdef PGM_WITH_A20 +# define PGM_A20_IS_ENABLED(a_pVCpu) ((a_pVCpu)->pgm.s.fA20Enabled) +# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) ((a_GCPhys) & (a_pVCpu)->pgm.s.GCPhysA20Mask) +# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) \ + do { a_GCPhysVar &= (a_pVCpu)->pgm.s.GCPhysA20Mask; } while (0) +# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) Assert(PGM_A20_APPLY(pVCpu, a_GCPhys) == (a_GCPhys)) +#else +# define PGM_A20_IS_ENABLED(a_pVCpu) (true) +# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) (a_GCPhys) +# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) do { } while (0) +# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) do { } while (0) +#endif +/** @} */ + + +/** + * Guest page table walk for the AMD64 mode. + */ +typedef struct PGMPTWALKGSTAMD64 +{ + PX86PML4 pPml4; + PX86PML4E pPml4e; + X86PML4E Pml4e; + + PX86PDPT pPdpt; + PX86PDPE pPdpe; + X86PDPE Pdpe; + + PX86PDPAE pPd; + PX86PDEPAE pPde; + X86PDEPAE Pde; + + PX86PTPAE pPt; + PX86PTEPAE pPte; + X86PTEPAE Pte; +} PGMPTWALKGSTAMD64; +/** Pointer to a AMD64 guest page table walk. */ +typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64; +/** Pointer to a const AMD64 guest page table walk. */ +typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64; + +/** + * Guest page table walk for the EPT mode. + */ +typedef struct PGMPTWALKGSTEPT +{ + PEPTPML4 pPml4; + PEPTPML4E pPml4e; + EPTPML4E Pml4e; + + PEPTPDPT pPdpt; + PEPTPDPTE pPdpte; + EPTPDPTE Pdpte; + + PEPTPD pPd; + PEPTPDE pPde; + EPTPDE Pde; + + PEPTPT pPt; + PEPTPTE pPte; + EPTPTE Pte; +} PGMPTWALKGSTEPT; +/** Pointer to an EPT guest page table walk. */ +typedef PGMPTWALKGSTEPT *PPGMPTWALKGSTEPT; +/** Pointer to a const EPT guest page table walk. */ +typedef PGMPTWALKGSTEPT const *PCPGMPTWALKGSTEPT; + +/** + * Guest page table walk for the PAE mode. + */ +typedef struct PGMPTWALKGSTPAE +{ + PX86PDPT pPdpt; + PX86PDPE pPdpe; + X86PDPE Pdpe; + + PX86PDPAE pPd; + PX86PDEPAE pPde; + X86PDEPAE Pde; + + PX86PTPAE pPt; + PX86PTEPAE pPte; + X86PTEPAE Pte; +} PGMPTWALKGSTPAE; +/** Pointer to a PAE guest page table walk. */ +typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE; +/** Pointer to a const AMD64 guest page table walk. */ +typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE; + +/** + * Guest page table walk for the 32-bit mode. + */ +typedef struct PGMPTWALKGST32BIT +{ + PX86PD pPd; + PX86PDE pPde; + X86PDE Pde; + + PX86PT pPt; + PX86PTE pPte; + X86PTE Pte; +} PGMPTWALKGST32BIT; +/** Pointer to a 32-bit guest page table walk. */ +typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT; +/** Pointer to a const 32-bit guest page table walk. */ +typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT; + +/** + * Which part of PGMPTWALKGST that is valid. + */ +typedef enum PGMPTWALKGSTTYPE +{ + /** Customary invalid 0 value. */ + PGMPTWALKGSTTYPE_INVALID = 0, + /** PGMPTWALKGST::u.Amd64 is valid. */ + PGMPTWALKGSTTYPE_AMD64, + /** PGMPTWALKGST::u.Pae is valid. */ + PGMPTWALKGSTTYPE_PAE, + /** PGMPTWALKGST::u.Legacy is valid. */ + PGMPTWALKGSTTYPE_32BIT, + /** PGMPTWALKGST::u.Ept is valid. */ + PGMPTWALKGSTTYPE_EPT, + /** Customary 32-bit type hack. */ + PGMPTWALKGSTTYPE_32BIT_HACK = 0x7fff0000 +} PGMPTWALKGSTTYPE; + +/** + * Combined guest page table walk result. + */ +typedef struct PGMPTWALKGST +{ + union + { + /** The page walker for AMD64. */ + PGMPTWALKGSTAMD64 Amd64; + /** The page walker for PAE (32-bit). */ + PGMPTWALKGSTPAE Pae; + /** The page walker for 32-bit paging (called legacy due to C naming + * convension). */ + PGMPTWALKGST32BIT Legacy; + /** The page walker for EPT (SLAT). */ + PGMPTWALKGSTEPT Ept; + } u; + /** Indicates which part of the union is valid. */ + PGMPTWALKGSTTYPE enmType; +} PGMPTWALKGST; +/** Pointer to a combined guest page table walk result. */ +typedef PGMPTWALKGST *PPGMPTWALKGST; +/** Pointer to a read-only combined guest page table walk result. */ +typedef PGMPTWALKGST const *PCPGMPTWALKGST; + + +/** @name Paging mode macros + * @{ + */ +#ifdef IN_RING3 +# define PGM_CTX(a,b) a##R3##b +# define PGM_CTX_STR(a,b) a "R3" b +# define PGM_CTX_DECL(type) DECLCALLBACK(type) +#elif defined(IN_RING0) +# define PGM_CTX(a,b) a##R0##b +# define PGM_CTX_STR(a,b) a "R0" b +# define PGM_CTX_DECL(type) VMMDECL(type) +#else +# error "Not IN_RING3 or IN_RING0!" +#endif + +#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name) +#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name +#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name +#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name) +#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name +#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name +#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name) +#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name +#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name +#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name) +#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name +#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name +#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name) +#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name +#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name +#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name) + +#define PGM_GST_SLAT_NAME_EPT(name) PGM_CTX(pgm,GstSlatEpt##name) +#define PGM_GST_SLAT_NAME_RC_EPT_STR(name) "pgmRCGstSlatEpt" #name +#define PGM_GST_SLAT_NAME_R0_EPT_STR(name) "pgmR0GstSlatEpt" #name +#define PGM_GST_SLAT_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_SLAT_NAME(name) + +#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name) +#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name +#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name +#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name) +#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name +#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name +#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name) +#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name +#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name +#define PGM_SHW_NAME_NESTED_32BIT(name) PGM_CTX(pgm,ShwNested32Bit##name) +#define PGM_SHW_NAME_RC_NESTED_32BIT_STR(name) "pgmRCShwNested32Bit" #name +#define PGM_SHW_NAME_R0_NESTED_32BIT_STR(name) "pgmR0ShwNested32Bit" #name +#define PGM_SHW_NAME_NESTED_PAE(name) PGM_CTX(pgm,ShwNestedPAE##name) +#define PGM_SHW_NAME_RC_NESTED_PAE_STR(name) "pgmRCShwNestedPAE" #name +#define PGM_SHW_NAME_R0_NESTED_PAE_STR(name) "pgmR0ShwNestedPAE" #name +#define PGM_SHW_NAME_NESTED_AMD64(name) PGM_CTX(pgm,ShwNestedAMD64##name) +#define PGM_SHW_NAME_RC_NESTED_AMD64_STR(name) "pgmRCShwNestedAMD64" #name +#define PGM_SHW_NAME_R0_NESTED_AMD64_STR(name) "pgmR0ShwNestedAMD64" #name +#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name) +#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name +#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name +#define PGM_SHW_NAME_NONE(name) PGM_CTX(pgm,ShwNone##name) +#define PGM_SHW_NAME_RC_NONE_STR(name) "pgmRCShwNone" #name +#define PGM_SHW_NAME_R0_NONE_STR(name) "pgmR0ShwNone" #name +#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name) + +/* Shw_Gst */ +#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name) +#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name) +#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name) +#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name) +#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name) +#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name) +#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name) +#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name) +#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name) +#define PGM_BTH_NAME_NESTED_32BIT_REAL(name) PGM_CTX(pgm,BthNested32BitReal##name) +#define PGM_BTH_NAME_NESTED_32BIT_PROT(name) PGM_CTX(pgm,BthNested32BitProt##name) +#define PGM_BTH_NAME_NESTED_32BIT_32BIT(name) PGM_CTX(pgm,BthNested32Bit32Bit##name) +#define PGM_BTH_NAME_NESTED_32BIT_PAE(name) PGM_CTX(pgm,BthNested32BitPAE##name) +#define PGM_BTH_NAME_NESTED_32BIT_AMD64(name) PGM_CTX(pgm,BthNested32BitAMD64##name) +#define PGM_BTH_NAME_NESTED_PAE_REAL(name) PGM_CTX(pgm,BthNestedPAEReal##name) +#define PGM_BTH_NAME_NESTED_PAE_PROT(name) PGM_CTX(pgm,BthNestedPAEProt##name) +#define PGM_BTH_NAME_NESTED_PAE_32BIT(name) PGM_CTX(pgm,BthNestedPAE32Bit##name) +#define PGM_BTH_NAME_NESTED_PAE_PAE(name) PGM_CTX(pgm,BthNestedPAEPAE##name) +#define PGM_BTH_NAME_NESTED_PAE_AMD64(name) PGM_CTX(pgm,BthNestedPAEAMD64##name) +#define PGM_BTH_NAME_NESTED_AMD64_REAL(name) PGM_CTX(pgm,BthNestedAMD64Real##name) +#define PGM_BTH_NAME_NESTED_AMD64_PROT(name) PGM_CTX(pgm,BthNestedAMD64Prot##name) +#define PGM_BTH_NAME_NESTED_AMD64_32BIT(name) PGM_CTX(pgm,BthNestedAMD6432Bit##name) +#define PGM_BTH_NAME_NESTED_AMD64_PAE(name) PGM_CTX(pgm,BthNestedAMD64PAE##name) +#define PGM_BTH_NAME_NESTED_AMD64_AMD64(name) PGM_CTX(pgm,BthNestedAMD64AMD64##name) +#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name) +#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name) +#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name) +#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name) +#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name) +#define PGM_BTH_NAME_NONE_REAL(name) PGM_CTX(pgm,BthNoneReal##name) +#define PGM_BTH_NAME_NONE_PROT(name) PGM_CTX(pgm,BthNoneProt##name) +#define PGM_BTH_NAME_NONE_32BIT(name) PGM_CTX(pgm,BthNone32Bit##name) +#define PGM_BTH_NAME_NONE_PAE(name) PGM_CTX(pgm,BthNonePAE##name) +#define PGM_BTH_NAME_NONE_AMD64(name) PGM_CTX(pgm,BthNoneAMD64##name) + +#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name +#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name +#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name +#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name +#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name +#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name +#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name +#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name +#define PGM_BTH_NAME_RC_NESTED_32BIT_REAL_STR(name) "pgmRCBthNested32BitReal" #name +#define PGM_BTH_NAME_RC_NESTED_32BIT_PROT_STR(name) "pgmRCBthNested32BitProt" #name +#define PGM_BTH_NAME_RC_NESTED_32BIT_32BIT_STR(name) "pgmRCBthNested32Bit32Bit" #name +#define PGM_BTH_NAME_RC_NESTED_32BIT_PAE_STR(name) "pgmRCBthNested32BitPAE" #name +#define PGM_BTH_NAME_RC_NESTED_32BIT_AMD64_STR(name) "pgmRCBthNested32BitAMD64" #name +#define PGM_BTH_NAME_RC_NESTED_PAE_REAL_STR(name) "pgmRCBthNestedPAEReal" #name +#define PGM_BTH_NAME_RC_NESTED_PAE_PROT_STR(name) "pgmRCBthNestedPAEProt" #name +#define PGM_BTH_NAME_RC_NESTED_PAE_32BIT_STR(name) "pgmRCBthNestedPAE32Bit" #name +#define PGM_BTH_NAME_RC_NESTED_PAE_PAE_STR(name) "pgmRCBthNestedPAEPAE" #name +#define PGM_BTH_NAME_RC_NESTED_PAE_AMD64_STR(name) "pgmRCBthNestedPAEAMD64" #name +#define PGM_BTH_NAME_RC_NESTED_AMD64_REAL_STR(name) "pgmRCBthNestedAMD64Real" #name +#define PGM_BTH_NAME_RC_NESTED_AMD64_PROT_STR(name) "pgmRCBthNestedAMD64Prot" #name +#define PGM_BTH_NAME_RC_NESTED_AMD64_32BIT_STR(name) "pgmRCBthNestedAMD6432Bit" #name +#define PGM_BTH_NAME_RC_NESTED_AMD64_PAE_STR(name) "pgmRCBthNestedAMD64PAE" #name +#define PGM_BTH_NAME_RC_NESTED_AMD64_AMD64_STR(name) "pgmRCBthNestedAMD64AMD64" #name +#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name +#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name +#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name +#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name +#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name + +#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name +#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name +#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name +#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name +#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name +#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name +#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name +#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name +#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name +#define PGM_BTH_NAME_R0_NESTED_32BIT_REAL_STR(name) "pgmR0BthNested32BitReal" #name +#define PGM_BTH_NAME_R0_NESTED_32BIT_PROT_STR(name) "pgmR0BthNested32BitProt" #name +#define PGM_BTH_NAME_R0_NESTED_32BIT_32BIT_STR(name) "pgmR0BthNested32Bit32Bit" #name +#define PGM_BTH_NAME_R0_NESTED_32BIT_PAE_STR(name) "pgmR0BthNested32BitPAE" #name +#define PGM_BTH_NAME_R0_NESTED_32BIT_AMD64_STR(name) "pgmR0BthNested32BitAMD64" #name +#define PGM_BTH_NAME_R0_NESTED_PAE_REAL_STR(name) "pgmR0BthNestedPAEReal" #name +#define PGM_BTH_NAME_R0_NESTED_PAE_PROT_STR(name) "pgmR0BthNestedPAEProt" #name +#define PGM_BTH_NAME_R0_NESTED_PAE_32BIT_STR(name) "pgmR0BthNestedPAE32Bit" #name +#define PGM_BTH_NAME_R0_NESTED_PAE_PAE_STR(name) "pgmR0BthNestedPAEPAE" #name +#define PGM_BTH_NAME_R0_NESTED_PAE_AMD64_STR(name) "pgmR0BthNestedPAEAMD64" #name +#define PGM_BTH_NAME_R0_NESTED_AMD64_REAL_STR(name) "pgmR0BthNestedAMD64Real" #name +#define PGM_BTH_NAME_R0_NESTED_AMD64_PROT_STR(name) "pgmR0BthNestedAMD64Prot" #name +#define PGM_BTH_NAME_R0_NESTED_AMD64_32BIT_STR(name) "pgmR0BthNestedAMD6432Bit" #name +#define PGM_BTH_NAME_R0_NESTED_AMD64_PAE_STR(name) "pgmR0BthNestedAMD64PAE" #name +#define PGM_BTH_NAME_R0_NESTED_AMD64_AMD64_STR(name) "pgmR0BthNestedAMD64AMD64" #name +#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name +#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name +#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name +#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name +#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name + +#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name) +/** @} */ + + +/** + * Function pointers for guest paging. + */ +typedef struct PGMMODEDATAGST +{ + /** The guest mode type. */ + uint32_t uType; + DECLCALLBACKMEMBER(int, pfnGetPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)); + DECLCALLBACKMEMBER(int, pfnModifyPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask)); + DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)); + DECLCALLBACKMEMBER(int, pfnExit,(PVMCPUCC pVCpu)); +#ifdef IN_RING3 + DECLCALLBACKMEMBER(int, pfnRelocate,(PVMCPUCC pVCpu, RTGCPTR offDelta)); /**< Only in ring-3. */ +#endif +} PGMMODEDATAGST; + +/** The length of g_aPgmGuestModeData. */ +#if VBOX_WITH_64_BITS_GUESTS +# define PGM_GUEST_MODE_DATA_ARRAY_SIZE (PGM_TYPE_AMD64 + 1) +#else +# define PGM_GUEST_MODE_DATA_ARRAY_SIZE (PGM_TYPE_PAE + 1) +#endif +/** The guest mode data array. */ +extern PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE]; + + +/** + * Function pointers for shadow paging. + */ +typedef struct PGMMODEDATASHW +{ + /** The shadow mode type. */ + uint32_t uType; + DECLCALLBACKMEMBER(int, pfnGetPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)); + DECLCALLBACKMEMBER(int, pfnModifyPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, + uint64_t fMask, uint32_t fOpFlags)); + DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu)); + DECLCALLBACKMEMBER(int, pfnExit,(PVMCPUCC pVCpu)); +#ifdef IN_RING3 + DECLCALLBACKMEMBER(int, pfnRelocate,(PVMCPUCC pVCpu, RTGCPTR offDelta)); /**< Only in ring-3. */ +#endif +} PGMMODEDATASHW; + +/** The length of g_aPgmShadowModeData. */ +#define PGM_SHADOW_MODE_DATA_ARRAY_SIZE PGM_TYPE_END +/** The shadow mode data array. */ +extern PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE]; + + +/** + * Function pointers for guest+shadow paging. + */ +typedef struct PGMMODEDATABTH +{ + /** The shadow mode type. */ + uint32_t uShwType; + /** The guest mode type. */ + uint32_t uGstType; + + DECLCALLBACKMEMBER(int, pfnInvalidatePage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)); + DECLCALLBACKMEMBER(int, pfnSyncCR3,(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)); + DECLCALLBACKMEMBER(int, pfnPrefetchPage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)); + DECLCALLBACKMEMBER(int, pfnVerifyAccessSyncPage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError)); + DECLCALLBACKMEMBER(int, pfnMapCR3,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)); + DECLCALLBACKMEMBER(int, pfnUnmapCR3,(PVMCPUCC pVCpu)); + DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)); +#ifndef IN_RING3 + DECLCALLBACKMEMBER(int, pfnTrap0eHandler,(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)); + DECLCALLBACKMEMBER(int, pfnNestedTrap0eHandler,(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNested, + bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk, + bool *pfLockTaken)); +#endif +#ifdef VBOX_STRICT + DECLCALLBACKMEMBER(unsigned, pfnAssertCR3,(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)); +#endif +} PGMMODEDATABTH; + +/** The length of g_aPgmBothModeData. */ +#define PGM_BOTH_MODE_DATA_ARRAY_SIZE ((PGM_TYPE_END - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END) +/** The guest+shadow mode data array. */ +extern PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE]; + + +#ifdef VBOX_WITH_STATISTICS +/** + * PGM statistics. + */ +typedef struct PGMSTATS +{ + /* R3 only: */ + STAMCOUNTER StatR3DetectedConflicts; /**< R3: Number of times PGMR3MapHasConflicts() detected a conflict. */ + STAMPROFILE StatR3ResolveConflict; /**< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). */ + + /* R3+RZ */ + STAMCOUNTER StatRZChunkR3MapTlbHits; /**< RC/R0: Ring-3/0 chunk mapper TLB hits. */ + STAMCOUNTER StatRZChunkR3MapTlbMisses; /**< RC/R0: Ring-3/0 chunk mapper TLB misses. */ + STAMCOUNTER StatRZPageMapTlbHits; /**< RC/R0: Ring-3/0 page mapper TLB hits. */ + STAMCOUNTER StatRZPageMapTlbMisses; /**< RC/R0: Ring-3/0 page mapper TLB misses. */ + STAMCOUNTER StatPageMapTlbFlushes; /**< ALL: Ring-3/0 page mapper TLB flushes. */ + STAMCOUNTER StatPageMapTlbFlushEntry; /**< ALL: Ring-3/0 page mapper TLB flushes. */ + STAMCOUNTER StatR3ChunkR3MapTlbHits; /**< R3: Ring-3/0 chunk mapper TLB hits. */ + STAMCOUNTER StatR3ChunkR3MapTlbMisses; /**< R3: Ring-3/0 chunk mapper TLB misses. */ + STAMCOUNTER StatR3PageMapTlbHits; /**< R3: Ring-3/0 page mapper TLB hits. */ + STAMCOUNTER StatR3PageMapTlbMisses; /**< R3: Ring-3/0 page mapper TLB misses. */ + STAMCOUNTER StatRZRamRangeTlbHits; /**< RC/R0: RAM range TLB hits. */ + STAMCOUNTER StatRZRamRangeTlbMisses; /**< RC/R0: RAM range TLB misses. */ + STAMCOUNTER StatR3RamRangeTlbHits; /**< R3: RAM range TLB hits. */ + STAMCOUNTER StatR3RamRangeTlbMisses; /**< R3: RAM range TLB misses. */ + STAMCOUNTER StatR3PhysHandlerReset; /**< R3: The number of times PGMHandlerPhysicalReset is called. */ + STAMCOUNTER StatRZPhysHandlerReset; /**< RC/R0: The number of times PGMHandlerPhysicalReset is called. */ + STAMCOUNTER StatR3PhysHandlerLookupHits; /**< R3: Number of cache hits when looking up physical handlers. */ + STAMCOUNTER StatR3PhysHandlerLookupMisses; /**< R3: Number of cache misses when looking up physical handlers. */ + STAMCOUNTER StatRZPhysHandlerLookupHits; /**< RC/R0: Number of cache hits when lookup up physical handlers. */ + STAMCOUNTER StatRZPhysHandlerLookupMisses; /**< RC/R0: Number of cache misses when looking up physical handlers */ + STAMCOUNTER StatRZPageReplaceShared; /**< RC/R0: Times a shared page has been replaced by a private one. */ + STAMCOUNTER StatRZPageReplaceZero; /**< RC/R0: Times the zero page has been replaced by a private one. */ +/// @todo STAMCOUNTER StatRZPageHandyAllocs; /**< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. */ + STAMCOUNTER StatR3PageReplaceShared; /**< R3: Times a shared page has been replaced by a private one. */ + STAMCOUNTER StatR3PageReplaceZero; /**< R3: Times the zero page has been replaced by a private one. */ +/// @todo STAMCOUNTER StatR3PageHandyAllocs; /**< R3: The number of times we've executed GMMR3AllocateHandyPages. */ + + /* RC only: */ + STAMCOUNTER StatRCInvlPgConflict; /**< RC: Number of times PGMInvalidatePage() detected a mapping conflict. */ + STAMCOUNTER StatRCInvlPgSyncMonCR3; /**< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. */ + + STAMCOUNTER StatRZPhysRead; + STAMCOUNTER StatRZPhysReadBytes; + STAMCOUNTER StatRZPhysWrite; + STAMCOUNTER StatRZPhysWriteBytes; + STAMCOUNTER StatR3PhysRead; + STAMCOUNTER StatR3PhysReadBytes; + STAMCOUNTER StatR3PhysWrite; + STAMCOUNTER StatR3PhysWriteBytes; + STAMCOUNTER StatRCPhysRead; + STAMCOUNTER StatRCPhysReadBytes; + STAMCOUNTER StatRCPhysWrite; + STAMCOUNTER StatRCPhysWriteBytes; + + STAMCOUNTER StatRZPhysSimpleRead; + STAMCOUNTER StatRZPhysSimpleReadBytes; + STAMCOUNTER StatRZPhysSimpleWrite; + STAMCOUNTER StatRZPhysSimpleWriteBytes; + STAMCOUNTER StatR3PhysSimpleRead; + STAMCOUNTER StatR3PhysSimpleReadBytes; + STAMCOUNTER StatR3PhysSimpleWrite; + STAMCOUNTER StatR3PhysSimpleWriteBytes; + STAMCOUNTER StatRCPhysSimpleRead; + STAMCOUNTER StatRCPhysSimpleReadBytes; + STAMCOUNTER StatRCPhysSimpleWrite; + STAMCOUNTER StatRCPhysSimpleWriteBytes; + + STAMCOUNTER StatTrackVirgin; /**< The number of first time shadowings. */ + STAMCOUNTER StatTrackAliased; /**< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. */ + STAMCOUNTER StatTrackAliasedMany; /**< The number of times we're tracking using cRef2. */ + STAMCOUNTER StatTrackAliasedLots; /**< The number of times we're hitting pages which has overflowed cRef2. */ + STAMCOUNTER StatTrackNoExtentsLeft; /**< The number of times the extent list was exhausted. */ + STAMCOUNTER StatTrackOverflows; /**< The number of times the extent list grows to long. */ + STAMPROFILE StatTrackDeref; /**< Profiling of SyncPageWorkerTrackDeref (expensive). */ + + STAMPROFILE StatLargePageAlloc2; /**< Time spent setting up newly allocated large pages. */ + STAMPROFILE StatLargePageSetup; /**< Time spent setting up newly allocated large pages. */ + /** pgmPhysIsValidLargePage profiling - R3 */ + STAMPROFILE StatR3IsValidLargePage; + /** pgmPhysIsValidLargePage profiling - RZ*/ + STAMPROFILE StatRZIsValidLargePage; + + STAMPROFILE StatChunkAging; + STAMPROFILE StatChunkFindCandidate; + STAMPROFILE StatChunkUnmap; + STAMPROFILE StatChunkMap; +} PGMSTATS; +#endif /* VBOX_WITH_STATISTICS */ + + +/** + * PGM Data (part of VM) + */ +typedef struct PGM +{ + /** The zero page. */ + uint8_t abZeroPg[RT_MAX(HOST_PAGE_SIZE, GUEST_PAGE_SIZE)]; + /** The MMIO placeholder page. */ + uint8_t abMmioPg[RT_MAX(HOST_PAGE_SIZE, GUEST_PAGE_SIZE)]; + + /** @name The zero page (abPagePg). + * @{ */ + /** The host physical address of the zero page. */ + RTHCPHYS HCPhysZeroPg; + /** @}*/ + + /** @name The Invalid MMIO page (abMmioPg). + * This page is filled with 0xfeedface. + * @{ */ + /** The host physical address of the invalid MMIO page. */ + RTHCPHYS HCPhysMmioPg; + /** The host pysical address of the invalid MMIO page plus all invalid + * physical address bits set. This is used to trigger X86_TRAP_PF_RSVD. + * @remarks Check fLessThan52PhysicalAddressBits before use. */ + RTHCPHYS HCPhysInvMmioPg; + /** @} */ + + /** @cfgm{/RamPreAlloc, boolean, false} + * Indicates whether the base RAM should all be allocated before starting + * the VM (default), or if it should be allocated when first written to. + */ + bool fRamPreAlloc; +#ifdef VBOX_WITH_PGM_NEM_MODE + /** Set if we're operating in NEM memory mode. + * + * NEM mode implies that memory is allocated in big chunks for each RAM range + * rather than on demand page by page. Memory is also not locked and PGM has + * therefore no physical addresses for them. Page sharing is out of the + * question. Ballooning depends on the native execution engine, but probably + * pointless as well. */ + bool fNemMode; +# define PGM_IS_IN_NEM_MODE(a_pVM) ((a_pVM)->pgm.s.fNemMode) +#else +# define PGM_IS_IN_NEM_MODE(a_pVM) (false) +#endif + /** Indicates whether write monitoring is currently in use. + * This is used to prevent conflicts between live saving and page sharing + * detection. */ + bool fPhysWriteMonitoringEngaged; + /** Set if the CPU has less than 52-bit physical address width. + * This is used */ + bool fLessThan52PhysicalAddressBits; + /** Set when nested paging is active. + * This is meant to save calls to HMIsNestedPagingActive and let the + * compilers optimize the code better. Whether we use nested paging or + * not is something we find out during VMM initialization and we won't + * change this later on. */ + bool fNestedPaging; + /** We're not in a state which permits writes to guest memory. + * (Only used in strict builds.) */ + bool fNoMorePhysWrites; + /** @cfgm{/PageFusionAllowed, boolean, false} + * Whether page fusion is allowed. */ + bool fPageFusionAllowed; + /** @cfgm{/PGM/PciPassThrough, boolean, false} + * Whether PCI passthrough is enabled. */ + bool fPciPassthrough; + /** The number of MMIO2 regions (serves as the next MMIO2 ID). */ + uint8_t cMmio2Regions; + /** Restore original ROM page content when resetting after loading state. + * The flag is set by pgmR3LoadRomRanges and cleared at reset. This + * enables the VM to start using an updated ROM without requiring powering + * down the VM, just rebooting or resetting it. */ + bool fRestoreRomPagesOnReset; + /** Whether to automatically clear all RAM pages on reset. */ + bool fZeroRamPagesOnReset; + /** Large page enabled flag. */ + bool fUseLargePages; + /** Alignment padding. */ +#ifndef VBOX_WITH_PGM_NEM_MODE + bool afAlignment3[1]; +#endif + /** The host paging mode. (This is what SUPLib reports.) */ + SUPPAGINGMODE enmHostMode; + bool afAlignment3b[2]; + + /** Generation ID for the RAM ranges. This member is incremented everytime + * a RAM range is linked or unlinked. */ + uint32_t volatile idRamRangesGen; + + /** Physical access handler type for ROM protection. */ + PGMPHYSHANDLERTYPE hRomPhysHandlerType; + /** Physical access handler type for MMIO2 dirty page tracing. */ + PGMPHYSHANDLERTYPE hMmio2DirtyPhysHandlerType; + + /** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */ + RTGCPHYS GCPhys4MBPSEMask; + /** Mask containing the invalid bits of a guest physical address. + * @remarks this does not stop at bit 52. */ + RTGCPHYS GCPhysInvAddrMask; + + + /** RAM range TLB for R3. */ + R3PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES]; + /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3. + * This is sorted by physical address and contains no overlapping ranges. */ + R3PTRTYPE(PPGMRAMRANGE) pRamRangesXR3; + /** Root of the RAM range search tree for ring-3. */ + R3PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR3; + /** Shadow Page Pool - R3 Ptr. */ + R3PTRTYPE(PPGMPOOL) pPoolR3; + /** Pointer to the list of ROM ranges - for R3. + * This is sorted by physical address and contains no overlapping ranges. */ + R3PTRTYPE(PPGMROMRANGE) pRomRangesR3; + /** Pointer to the list of MMIO2 ranges - for R3. + * Registration order. */ + R3PTRTYPE(PPGMREGMMIO2RANGE) pRegMmioRangesR3; + /** MMIO2 lookup array for ring-3. Indexed by idMmio2 minus 1. */ + R3PTRTYPE(PPGMREGMMIO2RANGE) apMmio2RangesR3[PGM_MMIO2_MAX_RANGES]; + + /** RAM range TLB for R0. */ + R0PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES]; + /** R0 pointer corresponding to PGM::pRamRangesXR3. */ + R0PTRTYPE(PPGMRAMRANGE) pRamRangesXR0; + /** Root of the RAM range search tree for ring-0. */ + R0PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR0; + /** Shadow Page Pool - R0 Ptr. */ + R0PTRTYPE(PPGMPOOL) pPoolR0; + /** R0 pointer corresponding to PGM::pRomRangesR3. */ + R0PTRTYPE(PPGMROMRANGE) pRomRangesR0; + /** MMIO2 lookup array for ring-0. Indexed by idMmio2 minus 1. */ + R0PTRTYPE(PPGMREGMMIO2RANGE) apMmio2RangesR0[PGM_MMIO2_MAX_RANGES]; + + /** Hack: Number of deprecated page mapping locks taken by the current lock + * owner via pgmPhysGCPhys2CCPtrInternalDepr. */ + uint32_t cDeprecatedPageLocks; + + /** Registered physical access handler types. */ + uint32_t cPhysHandlerTypes; + /** Physical access handler types. + * Initialized to callback causing guru meditations and invalid enmKind. */ + PGMPHYSHANDLERTYPEINTR3 aPhysHandlerTypes[PGMPHYSHANDLERTYPE_COUNT]; + /** Physical handler allocator, ring-3 edition. */ +#ifdef IN_RING3 + PGMPHYSHANDLERALLOCATOR PhysHandlerAllocator; +#else + RTCHardAvlTreeSlabAllocatorR3_T PhysHandlerAllocator; +#endif + /** The pointer to the ring-3 mapping of the physical access handler tree. */ + R3PTRTYPE(PPGMPHYSHANDLERTREE) pPhysHandlerTree; + /** Caching the last physical handler we looked. */ + uint32_t idxLastPhysHandler; + + uint32_t au64Padding3[5]; + + /** PGM critical section. + * This protects the physical, ram ranges, and the page flag updating (some of + * it anyway). + */ + PDMCRITSECT CritSectX; + + /** + * Data associated with managing the ring-3 mappings of the allocation chunks. + */ + struct + { + /** The chunk mapping TLB. */ + PGMCHUNKR3MAPTLB Tlb; + /** The chunk tree, ordered by chunk id. */ + R3PTRTYPE(PAVLU32NODECORE) pTree; +#if HC_ARCH_BITS == 32 + uint32_t u32Alignment0; +#endif + /** The number of mapped chunks. */ + uint32_t c; + /** @cfgm{/PGM/MaxRing3Chunks, uint32_t, host dependent} + * The maximum number of mapped chunks. On 64-bit this is unlimited by default, + * on 32-bit it defaults to 1 or 3 GB depending on the host. */ + uint32_t cMax; + /** The current time. This is incremented whenever a chunk is inserted. */ + uint32_t iNow; + /** Alignment padding. */ + uint32_t au32Alignment1[3]; + } ChunkR3Map; + + /** The page mapping TLB for ring-3. */ + PGMPAGER3MAPTLB PhysTlbR3; + /** The page mapping TLB for ring-0. */ + PGMPAGER0MAPTLB PhysTlbR0; + + /** The number of handy pages. */ + uint32_t cHandyPages; + + /** The number of large handy pages. */ + uint32_t cLargeHandyPages; + + /** + * Array of handy pages. + * + * This array is used in a two way communication between pgmPhysAllocPage + * and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as + * an intermediary. + * + * The size of this array is important, see pgmPhysEnsureHandyPage for details. + * (The current size of 32 pages, means 128 KB of handy memory.) + */ + GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES]; + + /** + * Array of large handy pages. (currently size 1) + * + * This array is used in a two way communication between pgmPhysAllocLargePage + * and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as + * an intermediary. + */ + GMMPAGEDESC aLargeHandyPage[1]; + /** When to try allocate large pages again after a failure. */ + uint64_t nsLargePageRetry; + /** Number of repeated long allocation times. */ + uint32_t cLargePageLongAllocRepeats; + uint32_t uPadding5; + + /** + * Live save data. + */ + struct + { + /** Per type statistics. */ + struct + { + /** The number of ready pages. */ + uint32_t cReadyPages; + /** The number of dirty pages. */ + uint32_t cDirtyPages; + /** The number of ready zero pages. */ + uint32_t cZeroPages; + /** The number of write monitored pages. */ + uint32_t cMonitoredPages; + } Rom, + Mmio2, + Ram; + /** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */ + uint32_t cIgnoredPages; + /** Indicates that a live save operation is active. */ + bool fActive; + /** Padding. */ + bool afReserved[2]; + /** The next history index. */ + uint8_t iDirtyPagesHistory; + /** History of the total amount of dirty pages. */ + uint32_t acDirtyPagesHistory[64]; + /** Short term dirty page average. */ + uint32_t cDirtyPagesShort; + /** Long term dirty page average. */ + uint32_t cDirtyPagesLong; + /** The number of saved pages. This is used to get some kind of estimate of the + * link speed so we can decide when we're done. It is reset after the first + * 7 passes so the speed estimate doesn't get inflated by the initial set of + * zero pages. */ + uint64_t cSavedPages; + /** The nanosecond timestamp when cSavedPages was 0. */ + uint64_t uSaveStartNS; + /** Pages per second (for statistics). */ + uint32_t cPagesPerSecond; + uint32_t cAlignment; + } LiveSave; + + /** @name Error injection. + * @{ */ + /** Inject handy page allocation errors pretending we're completely out of + * memory. */ + bool volatile fErrInjHandyPages; + /** Padding. */ + bool afReserved[3]; + /** @} */ + + /** @name Release Statistics + * @{ */ + uint32_t cAllPages; /**< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) */ + uint32_t cPrivatePages; /**< The number of private pages. */ + uint32_t cSharedPages; /**< The number of shared pages. */ + uint32_t cReusedSharedPages; /**< The number of reused shared pages. */ + uint32_t cZeroPages; /**< The number of zero backed pages. */ + uint32_t cPureMmioPages; /**< The number of pure MMIO pages. */ + uint32_t cMonitoredPages; /**< The number of write monitored pages. */ + uint32_t cWrittenToPages; /**< The number of previously write monitored pages. */ + uint32_t cWriteLockedPages; /**< The number of write locked pages. */ + uint32_t cReadLockedPages; /**< The number of read locked pages. */ + uint32_t cBalloonedPages; /**< The number of ballooned pages. */ + uint32_t cMappedChunks; /**< Number of times we mapped a chunk. */ + uint32_t cUnmappedChunks; /**< Number of times we unmapped a chunk. */ + uint32_t cLargePages; /**< The number of large pages. */ + uint32_t cLargePagesDisabled; /**< The number of disabled large pages. */ +/* uint32_t aAlignment4[1]; */ + + STAMPROFILE StatLargePageAlloc; /**< Time spent by the host OS for large page allocation. */ + STAMCOUNTER StatLargePageAllocFailed; /**< Count allocation failures. */ + STAMCOUNTER StatLargePageOverflow; /**< The number of times allocating a large pages takes more than the allowed period. */ + STAMCOUNTER StatLargePageReused; /**< The number of large pages we've reused.*/ + STAMCOUNTER StatLargePageRefused; /**< The number of times we couldn't use a large page.*/ + STAMCOUNTER StatLargePageRecheck; /**< The number of times we rechecked a disabled large page.*/ + STAMCOUNTER StatLargePageTlbFlush; /**< The number of a full VCPU TLB flush was required after allocation. */ + STAMCOUNTER StatLargePageZeroEvict; /**< The number of zero page mappings we had to evict when allocating a large page. */ + + STAMPROFILE StatShModCheck; /**< Profiles shared module checks. */ + + STAMPROFILE StatMmio2QueryAndResetDirtyBitmap; /**< Profiling PGMR3PhysMmio2QueryAndResetDirtyBitmap. */ + /** @} */ + +#ifdef VBOX_WITH_STATISTICS + /** These are optional statistics that used to be on the hyper heap. */ + PGMSTATS Stats; +#endif +} PGM; +#ifndef IN_TSTVMSTRUCTGC /* HACK */ +AssertCompileMemberAlignment(PGM, CritSectX, 8); +AssertCompileMemberAlignment(PGM, ChunkR3Map, 16); +AssertCompileMemberAlignment(PGM, PhysTlbR3, 32); /** @todo 32 byte alignment! */ +AssertCompileMemberAlignment(PGM, PhysTlbR0, 32); +AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8); +AssertCompileMemberAlignment(PGM, aHandyPages, 8); +#endif /* !IN_TSTVMSTRUCTGC */ +/** Pointer to the PGM instance data. */ +typedef PGM *PPGM; + + +#ifdef VBOX_WITH_STATISTICS +/** + * Per CPU statistis for PGM (used to be on the heap). + */ +typedef struct PGMCPUSTATS +{ + /* Common */ + STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /**< SyncPT - PD distribution. */ + STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /**< SyncPage - PD distribution. */ + + /* R0 only: */ + STAMPROFILE StatR0NpMiscfg; /**< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. */ + STAMCOUNTER StatR0NpMiscfgSyncPage; /**< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). */ + + /* RZ only: */ + STAMPROFILE StatRZTrap0e; /**< RC/R0: PGMTrap0eHandler() profiling. */ + STAMPROFILE StatRZTrap0eTime2Ballooned; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. */ + STAMPROFILE StatRZTrap0eTime2CSAM; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. */ + STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. */ + STAMPROFILE StatRZTrap0eTime2GuestTrap; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. */ + STAMPROFILE StatRZTrap0eTime2HndPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. */ + STAMPROFILE StatRZTrap0eTime2HndUnhandled; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. */ + STAMPROFILE StatRZTrap0eTime2InvalidPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. */ + STAMPROFILE StatRZTrap0eTime2MakeWritable; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. */ + STAMPROFILE StatRZTrap0eTime2Misc; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. */ + STAMPROFILE StatRZTrap0eTime2OutOfSync; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. */ + STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. */ + STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. */ + STAMPROFILE StatRZTrap0eTime2SyncPT; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. */ + STAMPROFILE StatRZTrap0eTime2WPEmulation; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. */ + STAMPROFILE StatRZTrap0eTime2Wp0RoUsHack; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be enabled. */ + STAMPROFILE StatRZTrap0eTime2Wp0RoUsUnhack; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be disabled. */ + STAMCOUNTER StatRZTrap0eConflicts; /**< RC/R0: The number of times \#PF was caused by an undetected conflict. */ + STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /**< RC/R0: Number of out-of-sync handled pages. */ + STAMCOUNTER StatRZTrap0eHandlersPhysAll; /**< RC/R0: Number of traps due to physical all-access handlers. */ + STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt; /**< RC/R0: Number of the physical all-access handler traps using the optimization. */ + STAMCOUNTER StatRZTrap0eHandlersPhysWrite; /**< RC/R0: Number of traps due to write-physical access handlers. */ + STAMCOUNTER StatRZTrap0eHandlersUnhandled; /**< RC/R0: Number of traps due to access outside range of monitored page(s). */ + STAMCOUNTER StatRZTrap0eHandlersInvalid; /**< RC/R0: Number of traps due to access to invalid physical memory. */ + STAMCOUNTER StatRZTrap0eUSNotPresentRead; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eUSWrite; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eUSReserved; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eUSNXE; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eUSRead; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eSVNotPresentRead; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eSVWrite; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eSVReserved; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eSNXE; /**< RC/R0: \#PF err kind */ + STAMCOUNTER StatRZTrap0eGuestPF; /**< RC/R0: Real guest \#PFs. */ + STAMCOUNTER StatRZTrap0eWPEmulInRZ; /**< RC/R0: WP=0 virtualization trap, handled. */ + STAMCOUNTER StatRZTrap0eWPEmulToR3; /**< RC/R0: WP=0 virtualization trap, chickened out. */ + STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /**< RC/R0: PD distribution of the \#PFs. */ + STAMCOUNTER StatRZGuestCR3WriteHandled; /**< RC/R0: The number of times WriteHandlerCR3() was successfully called. */ + STAMCOUNTER StatRZGuestCR3WriteUnhandled; /**< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. */ + STAMCOUNTER StatRZGuestCR3WriteConflict; /**< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. */ + STAMCOUNTER StatRZGuestROMWriteHandled; /**< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. */ + STAMCOUNTER StatRZGuestROMWriteUnhandled; /**< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler */ + STAMCOUNTER StatRZDynMapMigrateInvlPg; /**< RZ: invlpg in PGMR0DynMapMigrateAutoSet. */ + STAMPROFILE StatRZDynMapGCPageInl; /**< RZ: Calls to pgmRZDynMapGCPageInlined. */ + STAMCOUNTER StatRZDynMapGCPageInlHits; /**< RZ: Hash table lookup hits. */ + STAMCOUNTER StatRZDynMapGCPageInlMisses; /**< RZ: Misses that falls back to the code common. */ + STAMCOUNTER StatRZDynMapGCPageInlRamHits; /**< RZ: 1st ram range hits. */ + STAMCOUNTER StatRZDynMapGCPageInlRamMisses; /**< RZ: 1st ram range misses, takes slow path. */ + STAMPROFILE StatRZDynMapHCPageInl; /**< RZ: Calls to pgmRZDynMapHCPageInlined. */ + STAMCOUNTER StatRZDynMapHCPageInlHits; /**< RZ: Hash table lookup hits. */ + STAMCOUNTER StatRZDynMapHCPageInlMisses; /**< RZ: Misses that falls back to the code common. */ + STAMPROFILE StatRZDynMapHCPage; /**< RZ: Calls to pgmRZDynMapHCPageCommon. */ + STAMCOUNTER StatRZDynMapSetOptimize; /**< RZ: Calls to pgmRZDynMapOptimizeAutoSet. */ + STAMCOUNTER StatRZDynMapSetSearchFlushes; /**< RZ: Set search restoring to subset flushes. */ + STAMCOUNTER StatRZDynMapSetSearchHits; /**< RZ: Set search hits. */ + STAMCOUNTER StatRZDynMapSetSearchMisses; /**< RZ: Set search misses. */ + STAMCOUNTER StatRZDynMapPage; /**< RZ: Calls to pgmR0DynMapPage. */ + STAMCOUNTER StatRZDynMapPageHits0; /**< RZ: Hits at iPage+0. */ + STAMCOUNTER StatRZDynMapPageHits1; /**< RZ: Hits at iPage+1. */ + STAMCOUNTER StatRZDynMapPageHits2; /**< RZ: Hits at iPage+2. */ + STAMCOUNTER StatRZDynMapPageInvlPg; /**< RZ: invlpg. */ + STAMCOUNTER StatRZDynMapPageSlow; /**< RZ: Calls to pgmR0DynMapPageSlow. */ + STAMCOUNTER StatRZDynMapPageSlowLoopHits; /**< RZ: Hits in the pgmR0DynMapPageSlow search loop. */ + STAMCOUNTER StatRZDynMapPageSlowLoopMisses; /**< RZ: Misses in the pgmR0DynMapPageSlow search loop. */ + //STAMCOUNTER StatRZDynMapPageSlowLostHits; /**< RZ: Lost hits. */ + STAMCOUNTER StatRZDynMapSubsets; /**< RZ: Times PGMDynMapPushAutoSubset was called. */ + STAMCOUNTER StatRZDynMapPopFlushes; /**< RZ: Times PGMDynMapPopAutoSubset flushes the subset. */ + STAMCOUNTER aStatRZDynMapSetFilledPct[11]; /**< RZ: Set fill distribution, percent. */ + + /* HC - R3 and (maybe) R0: */ + + /* RZ & R3: */ + STAMPROFILE StatRZSyncCR3; /**< RC/R0: PGMSyncCR3() profiling. */ + STAMPROFILE StatRZSyncCR3Handlers; /**< RC/R0: Profiling of the PGMSyncCR3() update handler section. */ + STAMCOUNTER StatRZSyncCR3Global; /**< RC/R0: The number of global CR3 syncs. */ + STAMCOUNTER StatRZSyncCR3NotGlobal; /**< RC/R0: The number of non-global CR3 syncs. */ + STAMCOUNTER StatRZSyncCR3DstCacheHit; /**< RC/R0: The number of times we got some kind of cache hit on a page table. */ + STAMCOUNTER StatRZSyncCR3DstFreed; /**< RC/R0: The number of times we've had to free a shadow entry. */ + STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /**< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. */ + STAMCOUNTER StatRZSyncCR3DstNotPresent; /**< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. */ + STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /**< RC/R0: The number of times a global page directory wasn't flushed. */ + STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /**< RC/R0: The number of times a page table with only global entries wasn't flushed. */ + STAMPROFILE StatRZSyncPT; /**< RC/R0: PGMSyncPT() profiling. */ + STAMCOUNTER StatRZSyncPTFailed; /**< RC/R0: The number of times PGMSyncPT() failed. */ + STAMCOUNTER StatRZSyncPT4K; /**< RC/R0: Number of 4KB syncs. */ + STAMCOUNTER StatRZSyncPT4M; /**< RC/R0: Number of 4MB syncs. */ + STAMCOUNTER StatRZSyncPagePDNAs; /**< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */ + STAMCOUNTER StatRZSyncPagePDOutOfSync; /**< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. */ + STAMCOUNTER StatRZAccessedPage; /**< RC/R0: The number of pages marked not present for accessed bit emulation. */ + STAMPROFILE StatRZDirtyBitTracking; /**< RC/R0: Profiling the dirty bit tracking in CheckPageFault(). */ + STAMCOUNTER StatRZDirtyPage; /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */ + STAMCOUNTER StatRZDirtyPageBig; /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */ + STAMCOUNTER StatRZDirtyPageSkipped; /**< RC/R0: The number of pages already dirty or readonly. */ + STAMCOUNTER StatRZDirtyPageTrap; /**< RC/R0: The number of traps generated for dirty bit tracking. */ + STAMCOUNTER StatRZDirtyPageStale; /**< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) */ + STAMCOUNTER StatRZDirtyTrackRealPF; /**< RC/R0: The number of real pages faults during dirty bit tracking. */ + STAMCOUNTER StatRZDirtiedPage; /**< RC/R0: The number of pages marked dirty because of write accesses. */ + STAMCOUNTER StatRZPageAlreadyDirty; /**< RC/R0: The number of pages already marked dirty because of write accesses. */ + STAMPROFILE StatRZInvalidatePage; /**< RC/R0: PGMInvalidatePage() profiling. */ + STAMCOUNTER StatRZInvalidatePage4KBPages; /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. */ + STAMCOUNTER StatRZInvalidatePage4MBPages; /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. */ + STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /**< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. */ + STAMCOUNTER StatRZInvalidatePagePDNAs; /**< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. */ + STAMCOUNTER StatRZInvalidatePagePDNPs; /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */ + STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */ + STAMCOUNTER StatRZInvalidatePageSizeChanges ; /**< RC/R0: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */ + STAMCOUNTER StatRZInvalidatePageSkipped; /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */ + STAMCOUNTER StatRZPageOutOfSyncUser; /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */ + STAMCOUNTER StatRZPageOutOfSyncSupervisor; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */ + STAMCOUNTER StatRZPageOutOfSyncUserWrite; /**< RC/R0: The number of times user page is out of sync was detected in \#PF. */ + STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. */ + STAMCOUNTER StatRZPageOutOfSyncBallloon; /**< RC/R0: The number of times a ballooned page was accessed (read). */ + STAMPROFILE StatRZPrefetch; /**< RC/R0: PGMPrefetchPage. */ + STAMPROFILE StatRZFlushTLB; /**< RC/R0: Profiling of the PGMFlushTLB() body. */ + STAMCOUNTER StatRZFlushTLBNewCR3; /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */ + STAMCOUNTER StatRZFlushTLBNewCR3Global; /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */ + STAMCOUNTER StatRZFlushTLBSameCR3; /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */ + STAMCOUNTER StatRZFlushTLBSameCR3Global; /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */ + STAMPROFILE StatRZGstModifyPage; /**< RC/R0: Profiling of the PGMGstModifyPage() body */ + + STAMPROFILE StatR3SyncCR3; /**< R3: PGMSyncCR3() profiling. */ + STAMPROFILE StatR3SyncCR3Handlers; /**< R3: Profiling of the PGMSyncCR3() update handler section. */ + STAMCOUNTER StatR3SyncCR3Global; /**< R3: The number of global CR3 syncs. */ + STAMCOUNTER StatR3SyncCR3NotGlobal; /**< R3: The number of non-global CR3 syncs. */ + STAMCOUNTER StatR3SyncCR3DstFreed; /**< R3: The number of times we've had to free a shadow entry. */ + STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /**< R3: The number of times we've had to free a shadow entry for which the source entry was not present. */ + STAMCOUNTER StatR3SyncCR3DstNotPresent; /**< R3: The number of times we've encountered a not present shadow entry for a present guest entry. */ + STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /**< R3: The number of times a global page directory wasn't flushed. */ + STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /**< R3: The number of times a page table with only global entries wasn't flushed. */ + STAMCOUNTER StatR3SyncCR3DstCacheHit; /**< R3: The number of times we got some kind of cache hit on a page table. */ + STAMPROFILE StatR3SyncPT; /**< R3: PGMSyncPT() profiling. */ + STAMCOUNTER StatR3SyncPTFailed; /**< R3: The number of times PGMSyncPT() failed. */ + STAMCOUNTER StatR3SyncPT4K; /**< R3: Number of 4KB syncs. */ + STAMCOUNTER StatR3SyncPT4M; /**< R3: Number of 4MB syncs. */ + STAMCOUNTER StatR3SyncPagePDNAs; /**< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */ + STAMCOUNTER StatR3SyncPagePDOutOfSync; /**< R3: The number of time we've encountered an out-of-sync PD in SyncPage. */ + STAMCOUNTER StatR3AccessedPage; /**< R3: The number of pages marked not present for accessed bit emulation. */ + STAMPROFILE StatR3DirtyBitTracking; /**< R3: Profiling the dirty bit tracking in CheckPageFault(). */ + STAMCOUNTER StatR3DirtyPage; /**< R3: The number of pages marked read-only for dirty bit tracking. */ + STAMCOUNTER StatR3DirtyPageBig; /**< R3: The number of pages marked read-only for dirty bit tracking. */ + STAMCOUNTER StatR3DirtyPageSkipped; /**< R3: The number of pages already dirty or readonly. */ + STAMCOUNTER StatR3DirtyPageTrap; /**< R3: The number of traps generated for dirty bit tracking. */ + STAMCOUNTER StatR3DirtyTrackRealPF; /**< R3: The number of real pages faults during dirty bit tracking. */ + STAMCOUNTER StatR3DirtiedPage; /**< R3: The number of pages marked dirty because of write accesses. */ + STAMCOUNTER StatR3PageAlreadyDirty; /**< R3: The number of pages already marked dirty because of write accesses. */ + STAMPROFILE StatR3InvalidatePage; /**< R3: PGMInvalidatePage() profiling. */ + STAMCOUNTER StatR3InvalidatePage4KBPages; /**< R3: The number of times PGMInvalidatePage() was called for a 4KB page. */ + STAMCOUNTER StatR3InvalidatePage4MBPages; /**< R3: The number of times PGMInvalidatePage() was called for a 4MB page. */ + STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /**< R3: The number of times PGMInvalidatePage() skipped a 4MB page. */ + STAMCOUNTER StatR3InvalidatePagePDNAs; /**< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. */ + STAMCOUNTER StatR3InvalidatePagePDNPs; /**< R3: The number of times PGMInvalidatePage() was called for a not present page directory. */ + STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */ + STAMCOUNTER StatR3InvalidatePageSizeChanges ; /**< R3: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */ + STAMCOUNTER StatR3InvalidatePageSkipped; /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */ + STAMCOUNTER StatR3PageOutOfSyncUser; /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */ + STAMCOUNTER StatR3PageOutOfSyncSupervisor; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */ + STAMCOUNTER StatR3PageOutOfSyncUserWrite; /**< R3: The number of times user page is out of sync was detected in \#PF. */ + STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF. */ + STAMCOUNTER StatR3PageOutOfSyncBallloon; /**< R3: The number of times a ballooned page was accessed (read). */ + STAMPROFILE StatR3Prefetch; /**< R3: PGMPrefetchPage. */ + STAMPROFILE StatR3FlushTLB; /**< R3: Profiling of the PGMFlushTLB() body. */ + STAMCOUNTER StatR3FlushTLBNewCR3; /**< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */ + STAMCOUNTER StatR3FlushTLBNewCR3Global; /**< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */ + STAMCOUNTER StatR3FlushTLBSameCR3; /**< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */ + STAMCOUNTER StatR3FlushTLBSameCR3Global; /**< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */ + STAMPROFILE StatR3GstModifyPage; /**< R3: Profiling of the PGMGstModifyPage() body */ +} PGMCPUSTATS; +#endif /* VBOX_WITH_STATISTICS */ + + +/** + * PGMCPU Data (part of VMCPU). + */ +typedef struct PGMCPU +{ + /** A20 gate mask. + * Our current approach to A20 emulation is to let REM do it and don't bother + * anywhere else. The interesting Guests will be operating with it enabled anyway. + * But whould need arrise, we'll subject physical addresses to this mask. */ + RTGCPHYS GCPhysA20Mask; + /** A20 gate state - boolean! */ + bool fA20Enabled; + /** Mirror of the EFER.NXE bit. Managed by PGMNotifyNxeChanged. */ + bool fNoExecuteEnabled; + /** Whether the guest CR3 and PAE PDPEs have been mapped when guest PAE mode is + * active. */ + bool fPaePdpesAndCr3MappedR3; + bool fPaePdpesAndCr3MappedR0; + + /** What needs syncing (PGM_SYNC_*). + * This is used to queue operations for PGMSyncCR3, PGMInvalidatePage, + * PGMFlushTLB, and PGMR3Load. */ + uint32_t fSyncFlags; + + /** The shadow paging mode. */ + PGMMODE enmShadowMode; + /** The guest paging mode. */ + PGMMODE enmGuestMode; + /** The guest second level address translation mode. */ + PGMSLAT enmGuestSlatMode; + /** Guest mode data table index (PGM_TYPE_XXX). */ + uint8_t volatile idxGuestModeData; + /** Shadow mode data table index (PGM_TYPE_XXX). */ + uint8_t volatile idxShadowModeData; + /** Both mode data table index (complicated). */ + uint8_t volatile idxBothModeData; + /** Alignment padding. */ + uint8_t abPadding[1]; + + /** The guest CR3. + * When SLAT is active, this is the translated physical address. + * When SLAT is inactive, this is the physical address in CR3. */ + RTGCPHYS GCPhysCR3; + + /** The nested-guest CR3. + * When SLAT is active, this is CR3 prior to translation. + * When SLAT is inactive, this is unused (and NIL_RTGCPHYS). */ + RTGCPHYS GCPhysNstGstCR3; + + /** The cached guest CR3 when it has been mapped in PAE mode. + * This allows us to skip remapping the CR3 and PAE PDPEs + * (in PGMFlushTLB or similar) when it was already done as + * part of MOV CRx instruction emulation. + */ + RTGCPHYS GCPhysPaeCR3; + + /** @name 32-bit Guest Paging. + * @{ */ + /** The guest's page directory, R3 pointer. */ + R3PTRTYPE(PX86PD) pGst32BitPdR3; + /** The guest's page directory, R0 pointer. */ + R0PTRTYPE(PX86PD) pGst32BitPdR0; + /** Mask containing the MBZ bits of a big page PDE. */ + uint32_t fGst32BitMbzBigPdeMask; + /** Set if the page size extension (PSE) is enabled. */ + bool fGst32BitPageSizeExtension; + /** Alignment padding. */ + bool afAlignment2[3]; + /** @} */ + + /** @name PAE Guest Paging. + * @{ */ + /** The guest's page directory pointer table, R3 pointer. */ + R3PTRTYPE(PX86PDPT) pGstPaePdptR3; + /** The guest's page directory pointer table, R0 pointer. */ + R0PTRTYPE(PX86PDPT) pGstPaePdptR0; + + /** The guest's page directories, R3 pointers. + * These are individual pointers and don't have to be adjacent. + * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */ + R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4]; + /** The guest's page directories, R0 pointers. + * Same restrictions as apGstPaePDsR3. */ + R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4]; + /** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */ + RTGCPHYS aGCPhysGstPaePDs[4]; + /** Mask containing the MBZ PTE bits. */ + uint64_t fGstPaeMbzPteMask; + /** Mask containing the MBZ PDE bits. */ + uint64_t fGstPaeMbzPdeMask; + /** Mask containing the MBZ big page PDE bits. */ + uint64_t fGstPaeMbzBigPdeMask; + /** Mask containing the MBZ PDPE bits. */ + uint64_t fGstPaeMbzPdpeMask; + /** @} */ + + /** @name AMD64 Guest Paging. + * @{ */ + /** The guest's page directory pointer table, R3 pointer. */ + R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3; + /** The guest's page directory pointer table, R0 pointer. */ + R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0; + /** Mask containing the MBZ PTE bits. */ + uint64_t fGstAmd64MbzPteMask; + /** Mask containing the MBZ PDE bits. */ + uint64_t fGstAmd64MbzPdeMask; + /** Mask containing the MBZ big page PDE bits. */ + uint64_t fGstAmd64MbzBigPdeMask; + /** Mask containing the MBZ PDPE bits. */ + uint64_t fGstAmd64MbzPdpeMask; + /** Mask containing the MBZ big page PDPE bits. */ + uint64_t fGstAmd64MbzBigPdpeMask; + /** Mask containing the MBZ PML4E bits. */ + uint64_t fGstAmd64MbzPml4eMask; + /** Mask containing the PDPE bits that we shadow. */ + uint64_t fGstAmd64ShadowedPdpeMask; + /** Mask containing the PML4E bits that we shadow. */ + uint64_t fGstAmd64ShadowedPml4eMask; + /** @} */ + + /** @name PAE and AMD64 Guest Paging. + * @{ */ + /** Mask containing the PTE bits that we shadow. */ + uint64_t fGst64ShadowedPteMask; + /** Mask containing the PDE bits that we shadow. */ + uint64_t fGst64ShadowedPdeMask; + /** Mask containing the big page PDE bits that we shadow in the PDE. */ + uint64_t fGst64ShadowedBigPdeMask; + /** Mask containing the big page PDE bits that we shadow in the PTE. */ + uint64_t fGst64ShadowedBigPde4PteMask; + /** @} */ + + /** @name EPT Guest Paging. + * @{ */ + /** The guest's EPT PML4 table, R3 pointer. */ + R3PTRTYPE(PEPTPML4) pGstEptPml4R3; + /** The guest's EPT PML4 table, R0 pointer. */ + R0PTRTYPE(PEPTPML4) pGstEptPml4R0; + /** The guest's EPT pointer (copy of virtual VMCS). */ + uint64_t uEptPtr; + /** Copy of the VM's IA32_VMX_EPT_VPID_CAP VPID MSR for faster access. Doesn't + * change through the lifetime of the VM. */ + uint64_t uEptVpidCapMsr; + /** Mask containing the MBZ PTE bits. */ + uint64_t fGstEptMbzPteMask; + /** Mask containing the MBZ PDE bits. */ + uint64_t fGstEptMbzPdeMask; + /** Mask containing the MBZ big page (2M) PDE bits. */ + uint64_t fGstEptMbzBigPdeMask; + /** Mask containing the MBZ PDPTE bits. */ + uint64_t fGstEptMbzPdpteMask; + /** Mask containing the MBZ big page (1G) PDPTE bits. */ + uint64_t fGstEptMbzBigPdpteMask; + /** Mask containing the MBZ PML4E bits. */ + uint64_t fGstEptMbzPml4eMask; + /** Mask to determine whether an entry is present. */ + uint64_t fGstEptPresentMask; + + /** Mask containing the EPT PTE bits we shadow. */ + uint64_t fGstEptShadowedPteMask; + /** Mask containing the EPT PDE bits we shadow. */ + uint64_t fGstEptShadowedPdeMask; + /** Mask containing the EPT PDE (2M) bits we shadow. */ + uint64_t fGstEptShadowedBigPdeMask; + /** Mask containing the EPT PDPTE bits we shadow. */ + uint64_t fGstEptShadowedPdpteMask; + /** Mask containing the EPT PML4E bits we shadow. */ + uint64_t fGstEptShadowedPml4eMask; + /** @} */ + + /** Pointer to the page of the current active CR3 - R3 Ptr. */ + R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3; + /** Pointer to the page of the current active CR3 - R0 Ptr. */ + R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0; + + /** For saving stack space, the disassembler state is allocated here instead of + * on the stack. */ + DISCPUSTATE DisState; + + /** Counts the number of times the netware WP0+RO+US hack has been applied. */ + uint64_t cNetwareWp0Hacks; + + /** Count the number of pgm pool access handler calls. */ + uint64_t cPoolAccessHandler; + + /** @name Release Statistics + * @{ */ + /** The number of times the guest has switched mode since last reset or statistics reset. */ + STAMCOUNTER cGuestModeChanges; + /** The number of times the guest has switched mode since last reset or statistics reset. */ + STAMCOUNTER cA20Changes; + /** @} */ + +#ifdef VBOX_WITH_STATISTICS + /** These are statistics that used to be on the hyper heap. */ + PGMCPUSTATS Stats; +#endif +} PGMCPU; +/** Pointer to the per-cpu PGM data. */ +typedef PGMCPU *PPGMCPU; + + +/** @name PGM::fSyncFlags Flags + * @note Was part of saved state a long time ago. + * @{ + */ +/* 0 used to be PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL */ +/** Always sync CR3. */ +#define PGM_SYNC_ALWAYS RT_BIT(1) +/** Check guest mapping in SyncCR3. */ +#define PGM_SYNC_MAP_CR3 RT_BIT(3) +/** Clear the page pool (a light weight flush). */ +#define PGM_SYNC_CLEAR_PGM_POOL_BIT 8 +#define PGM_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT) +/** @} */ + + +#if defined(IN_RING0) || defined(DOXYGEN_RUNNING) + +/** + * PGM GVMCPU instance data. + */ +typedef struct PGMR0PERVCPU +{ +# ifdef VBOX_WITH_STATISTICS + /** R0: Which statistic this \#PF should be attributed to. */ + R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0; +# endif + uint64_t u64Dummy; +} PGMR0PERVCPU; + + +/** + * PGM GVM instance data. + */ +typedef struct PGMR0PERVM +{ + /** @name PGM Pool related stuff. + * @{ */ + /** Critical section for serializing pool growth. */ + RTCRITSECT PoolGrowCritSect; + /** The memory objects for the pool pages. */ + RTR0MEMOBJ ahPoolMemObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW]; + /** The ring-3 mapping objects for the pool pages. */ + RTR0MEMOBJ ahPoolMapObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW]; + /** @} */ + + /** Physical access handler types for ring-0. + * Initialized to callback causing return to ring-3 and invalid enmKind. */ + PGMPHYSHANDLERTYPEINTR0 aPhysHandlerTypes[PGMPHYSHANDLERTYPE_COUNT]; + /** Physical handler allocator, ring-3 edition. */ + PGMPHYSHANDLERALLOCATOR PhysHandlerAllocator; + /** The pointer to the ring-3 mapping of the physical access handler tree. */ + PPGMPHYSHANDLERTREE pPhysHandlerTree; + /** The allocation object for the physical access handler tree. */ + RTR0MEMOBJ hPhysHandlerMemObj; + /** The ring-3 mapping object for the physicall access handler tree. */ + RTR0MEMOBJ hPhysHandlerMapObj; +} PGMR0PERVM; + +#endif /* IN_RING0 || DOXYGEN_RUNNING */ + +RT_C_DECLS_BEGIN + +#if defined(VBOX_STRICT) +int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL); +# define PGM_LOCK_VOID(a_pVM) pgmLockDebug((a_pVM), true, RT_SRC_POS) +# define PGM_LOCK(a_pVM) pgmLockDebug((a_pVM), false, RT_SRC_POS) +#else +int pgmLock(PVMCC pVM, bool fVoid); +# define PGM_LOCK_VOID(a_pVM) pgmLock((a_pVM), true) +# define PGM_LOCK(a_pVM) pgmLock((a_pVM), false) +#endif +void pgmUnlock(PVMCC pVM); +# define PGM_UNLOCK(a_pVM) pgmUnlock((a_pVM)) +/** + * Asserts that the caller owns the PDM lock. + * This is the internal variant of PGMIsLockOwner. + * @param a_pVM Pointer to the VM. + */ +#define PGM_LOCK_ASSERT_OWNER(a_pVM) Assert(PDMCritSectIsOwner((a_pVM), &(a_pVM)->pgm.s.CritSectX)) +/** + * Asserts that the caller owns the PDM lock. + * This is the internal variant of PGMIsLockOwner. + * @param a_pVM Pointer to the VM. + * @param a_pVCpu The current CPU handle. + */ +#define PGM_LOCK_ASSERT_OWNER_EX(a_pVM, a_pVCpu) Assert(PDMCritSectIsOwnerEx((a_pVCpu), &(a_pVM)->pgm.s.CritSectX)) + +uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t *pcEntries, uint32_t *pcbTreeAndBitmap); +int pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, uint64_t uUser, + R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler); +int pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler); +int pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast); +int pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler); +int pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler); +void pgmR3HandlerPhysicalUpdateAll(PVM pVM); +bool pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys); +void pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, PPGMRAMRANGE pRam, + bool fDoAccounting, bool fFlushIemTlbs); +DECLHIDDEN(int) pgmHandlerPhysicalResetMmio2WithBitmap(PVMCC pVM, RTGCPHYS GCPhys, void *pvBitmap, uint32_t offBitmap); +DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs); +DECLCALLBACK(FNPGMPHYSHANDLER) pgmR3HandlerPhysicalHandlerInvalid; +#ifndef IN_RING3 +DECLCALLBACK(FNPGMPHYSHANDLER) pgmR0HandlerPhysicalHandlerToRing3; +DECLCALLBACK(FNPGMRZPHYSPFHANDLER) pgmR0HandlerPhysicalPfHandlerToRing3; +#endif + +int pgmR3InitSavedState(PVM pVM, uint64_t cbRam); + +int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys); +int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys); +#ifdef IN_RING0 +int pgmR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu, bool fRing3); +int pgmR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys); +#endif +int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage); +int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys); +int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys); +void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys); +int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys); +int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv); +int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv); +int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv); +int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv); +int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr); +int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr); +int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv); +int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock); +int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock); +void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock); +DECLCALLBACK(FNPGMPHYSHANDLER) pgmPhysRomWriteHandler; +DECLCALLBACK(FNPGMPHYSHANDLER) pgmPhysMmio2WriteHandler; +#ifndef IN_RING3 +DECLCALLBACK(FNPGMRZPHYSPFHANDLER) pgmPhysRomWritePfHandler; +DECLCALLBACK(FNPGMRZPHYSPFHANDLER) pgmPhysMmio2WritePfHandler; +#endif +int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys, + PGMPAGETYPE enmNewType); +void pgmPhysInvalidRamRangeTlbs(PVMCC pVM); +void pgmPhysInvalidatePageMapTLB(PVMCC pVM); +void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys); +PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys); +PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys); +PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys); +int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage); +int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam); +#ifdef VBOX_WITH_NATIVE_NEM +void pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State); +#endif + +#ifdef IN_RING3 +void pgmR3PhysRelinkRamRanges(PVM pVM); +int pgmR3PhysRamPreAllocate(PVM pVM); +int pgmR3PhysRamReset(PVM pVM); +int pgmR3PhysRomReset(PVM pVM); +int pgmR3PhysRamZeroAll(PVM pVM); +int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk); +int pgmR3PhysRamTerm(PVM pVM); +void pgmR3PhysRomTerm(PVM pVM); +void pgmR3PhysAssertSharedPageChecksums(PVM pVM); + +int pgmR3PoolInit(PVM pVM); +void pgmR3PoolRelocate(PVM pVM); +void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu); +void pgmR3PoolReset(PVM pVM); +void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb); +DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl); +void pgmR3PoolWriteProtectPages(PVM pVM); + +#endif /* IN_RING3 */ +#ifdef IN_RING0 +int pgmR0PoolInitVM(PGVM pGVM); +#endif +int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled, + uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage); +void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable); +void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable); +int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */); +void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys); +PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys); +PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys); +int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv); +int pgmPoolSyncCR3(PVMCPUCC pVCpu); +bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys); +void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT); +int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs); +void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte); +uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte); +void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte); +void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage); +void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage); +FNPGMPHYSHANDLER pgmPoolAccessHandler; +#ifndef IN_RING3 +FNPGMRZPHYSPFHANDLER pgmRZPoolAccessPfHandler; +#endif + +void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage); +void pgmPoolResetDirtyPages(PVMCC pVM); +void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage); + +/** Gets the ring-0 pointer for the given pool page. */ +DECLINLINE(R0PTRTYPE(PPGMPOOLPAGE)) pgmPoolConvertPageToR0(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ +#ifdef IN_RING3 + size_t offPage = (uintptr_t)pPage - (uintptr_t)pPool; +# ifdef VBOX_STRICT + size_t iPage = (offPage - RT_UOFFSETOF(PGMPOOL, aPages)) / sizeof(*pPage); + AssertReturn(iPage < pPool->cMaxPages, NIL_RTR0PTR); + AssertReturn(iPage * sizeof(*pPage) + RT_UOFFSETOF(PGMPOOL, aPages) == offPage, NIL_RTR0PTR); +# endif + return pPool->pPoolR0 + offPage; +#else + RT_NOREF(pPool); + return pPage; +#endif +} + +/** Gets the ring-3 pointer for the given pool page. */ +DECLINLINE(R3PTRTYPE(PPGMPOOLPAGE)) pgmPoolConvertPageToR3(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ +#ifdef IN_RING3 + RT_NOREF(pPool); + return pPage; +#else + size_t offPage = (uintptr_t)pPage - (uintptr_t)pPool; +# ifdef VBOX_STRICT + size_t iPage = (offPage - RT_UOFFSETOF(PGMPOOL, aPages)) / sizeof(*pPage); + AssertReturn(iPage < pPool->cMaxPages, NIL_RTR3PTR); + AssertReturn(iPage * sizeof(*pPage) + RT_UOFFSETOF(PGMPOOL, aPages) == offPage, NIL_RTR3PTR); +# endif + return pPool->pPoolR3 + offPage; +#endif +} + +int pgmR3ExitShadowModeBeforePoolFlush(PVMCPU pVCpu); +int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu); +void pgmR3RefreshShadowModeAfterA20Change(PVMCPU pVCpu); + +int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags); +int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD); +int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode); + +int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd); +int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt); +int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd); +int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppPml4); +#endif +int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk); +int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk); + +# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64 && defined(IN_RING3) +FNDBGCCMD pgmR3CmdCheckDuplicatePages; +FNDBGCCMD pgmR3CmdShowSharedModules; +# endif + +void pgmLogState(PVM pVM); + +RT_C_DECLS_END + +/** @} */ + +#endif /* !VMM_INCLUDED_SRC_include_PGMInternal_h */ + |