Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1369 insertions, 0 deletions

diff --git a/src/VBox/VMM/VMMR0/PGMR0.cpp b/src/VBox/VMM/VMMR0/PGMR0.cpp
new file mode 100644
index 00000000..b3650243
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PGMR0.cpp
@@ -0,0 +1,1369 @@
+/* $Id: PGMR0.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Ring-0.
+ */
+
+/*
+ * Copyright (C) 2007-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
+#include <VBox/rawpci.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/gmm.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#include "PGMInline.h"
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/process.h>
+#include <iprt/rand.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+
+
+/*
+ * Instantiate the ring-0 header/code templates.
+ */
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_AMD64_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
+#include "PGMR0Bth.h"
+#undef PGM_BTH_NAME
+
+
+/**
+ * Initializes the per-VM data for the PGM.
+ *
+ * This is called from under the GVMM lock, so it should only initialize the
+ * data so PGMR0CleanupVM and others will work smoothly.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ * @param   hMemObj Handle to the memory object backing pGVM.
+ */
+VMMR0_INT_DECL(int) PGMR0InitPerVMData(PGVM pGVM, RTR0MEMOBJ hMemObj)
+{
+    AssertCompile(sizeof(pGVM->pgm.s) <= sizeof(pGVM->pgm.padding));
+    AssertCompile(sizeof(pGVM->pgmr0.s) <= sizeof(pGVM->pgmr0.padding));
+
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs) == RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMapObjs));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+    }
+    pGVM->pgmr0.s.hPhysHandlerMemObj = NIL_RTR0MEMOBJ;
+    pGVM->pgmr0.s.hPhysHandlerMapObj = NIL_RTR0MEMOBJ;
+
+    /*
+     * Initialize the handler type table with return to ring-3 callbacks so we
+     * don't have to do anything special for ring-3 only registrations.
+     *
+     * Note! The random bits of the hType value is mainly for prevent trouble
+     *       with zero initialized handles w/o needing to sacrifice handle zero.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes); i++)
+    {
+        pGVM->pgmr0.s.aPhysHandlerTypes[i].hType        = i | (RTRandU64() & ~(uint64_t)PGMPHYSHANDLERTYPE_IDX_MASK);
+        pGVM->pgmr0.s.aPhysHandlerTypes[i].enmKind      = PGMPHYSHANDLERKIND_INVALID;
+        pGVM->pgmr0.s.aPhysHandlerTypes[i].pfnHandler   = pgmR0HandlerPhysicalHandlerToRing3;
+        pGVM->pgmr0.s.aPhysHandlerTypes[i].pfnPfHandler = pgmR0HandlerPhysicalPfHandlerToRing3;
+
+        pGVM->pgm.s.aPhysHandlerTypes[i].hType          = pGVM->pgmr0.s.aPhysHandlerTypes[i].hType;
+        pGVM->pgm.s.aPhysHandlerTypes[i].enmKind        = PGMPHYSHANDLERKIND_INVALID;
+    }
+
+    /*
+     * Get the physical address of the ZERO and MMIO-dummy pages.
+     */
+    AssertReturn(((uintptr_t)&pGVM->pgm.s.abZeroPg[0] & HOST_PAGE_OFFSET_MASK) == 0, VERR_INTERNAL_ERROR_2);
+    pGVM->pgm.s.HCPhysZeroPg    = RTR0MemObjGetPagePhysAddr(hMemObj, RT_UOFFSETOF_DYN(GVM, pgm.s.abZeroPg) >> HOST_PAGE_SHIFT);
+    AssertReturn(pGVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS, VERR_INTERNAL_ERROR_3);
+
+    AssertReturn(((uintptr_t)&pGVM->pgm.s.abMmioPg[0] & HOST_PAGE_OFFSET_MASK) == 0, VERR_INTERNAL_ERROR_2);
+    pGVM->pgm.s.HCPhysMmioPg    = RTR0MemObjGetPagePhysAddr(hMemObj, RT_UOFFSETOF_DYN(GVM, pgm.s.abMmioPg) >> HOST_PAGE_SHIFT);
+    AssertReturn(pGVM->pgm.s.HCPhysMmioPg != NIL_RTHCPHYS, VERR_INTERNAL_ERROR_3);
+
+    pGVM->pgm.s.HCPhysInvMmioPg = pGVM->pgm.s.HCPhysMmioPg;
+
+    return RTCritSectInit(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
+
+
+/**
+ * Initalize the per-VM PGM for ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0InitVM(PGVM pGVM)
+{
+    /*
+     * Set up the ring-0 context for our access handlers.
+     */
+    int rc = PGMR0HandlerPhysicalTypeSetUpContext(pGVM, PGMPHYSHANDLERKIND_WRITE, 0 /*fFlags*/,
+                                                  pgmPhysRomWriteHandler, pgmPhysRomWritePfHandler,
+                                                  "ROM write protection", pGVM->pgm.s.hRomPhysHandlerType);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Register the physical access handler doing dirty MMIO2 tracing.
+     */
+    rc = PGMR0HandlerPhysicalTypeSetUpContext(pGVM, PGMPHYSHANDLERKIND_WRITE, PGMPHYSHANDLER_F_KEEP_PGM_LOCK,
+                                              pgmPhysMmio2WriteHandler, pgmPhysMmio2WritePfHandler,
+                                              "MMIO2 dirty page tracing", pGVM->pgm.s.hMmio2DirtyPhysHandlerType);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * The page pool.
+     */
+    return pgmR0PoolInitVM(pGVM);
+}
+
+
+/**
+ * Called at the end of the ring-0 initialization to seal access handler types.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(void) PGMR0DoneInitVM(PGVM pGVM)
+{
+    /*
+     * Seal all the access handler types. Does both ring-3 and ring-0.
+     *
+     * Note! Since this is a void function and we don't have any ring-0 state
+     *       machinery for marking the VM as bogus, this code will just
+     *       override corrupted values as best as it can.
+     */
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes));
+    for (size_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes); i++)
+    {
+        PPGMPHYSHANDLERTYPEINTR0 const pTypeR0   = &pGVM->pgmr0.s.aPhysHandlerTypes[i];
+        PPGMPHYSHANDLERTYPEINTR3 const pTypeR3   = &pGVM->pgm.s.aPhysHandlerTypes[i];
+        PGMPHYSHANDLERKIND       const enmKindR3 = pTypeR3->enmKind;
+        PGMPHYSHANDLERKIND       const enmKindR0 = pTypeR0->enmKind;
+        AssertLogRelMsgStmt(pTypeR0->hType == pTypeR3->hType,
+                            ("i=%u %#RX64 vs %#RX64 %s\n", i, pTypeR0->hType, pTypeR3->hType, pTypeR0->pszDesc),
+                            pTypeR3->hType = pTypeR0->hType);
+        switch (enmKindR3)
+        {
+            case PGMPHYSHANDLERKIND_ALL:
+            case PGMPHYSHANDLERKIND_MMIO:
+                if (   enmKindR0 == enmKindR3
+                    || enmKindR0 == PGMPHYSHANDLERKIND_INVALID)
+                {
+                    pTypeR3->fRing0Enabled = enmKindR0 == enmKindR3;
+                    pTypeR0->uState = PGM_PAGE_HNDL_PHYS_STATE_ALL;
+                    pTypeR3->uState = PGM_PAGE_HNDL_PHYS_STATE_ALL;
+                    continue;
+                }
+                break;
+
+            case PGMPHYSHANDLERKIND_WRITE:
+                if (   enmKindR0 == enmKindR3
+                    || enmKindR0 == PGMPHYSHANDLERKIND_INVALID)
+                {
+                    pTypeR3->fRing0Enabled = enmKindR0 == enmKindR3;
+                    pTypeR0->uState = PGM_PAGE_HNDL_PHYS_STATE_WRITE;
+                    pTypeR3->uState = PGM_PAGE_HNDL_PHYS_STATE_WRITE;
+                    continue;
+                }
+                break;
+
+            default:
+                AssertLogRelMsgFailed(("i=%u enmKindR3=%d\n", i, enmKindR3));
+                RT_FALL_THROUGH();
+            case PGMPHYSHANDLERKIND_INVALID:
+                AssertLogRelMsg(enmKindR0 == PGMPHYSHANDLERKIND_INVALID,
+                                ("i=%u enmKind=%d %s\n", i, enmKindR0, pTypeR0->pszDesc));
+                AssertLogRelMsg(pTypeR0->pfnHandler == pgmR0HandlerPhysicalHandlerToRing3,
+                                ("i=%u pfnHandler=%p %s\n", i, pTypeR0->pfnHandler, pTypeR0->pszDesc));
+                AssertLogRelMsg(pTypeR0->pfnPfHandler == pgmR0HandlerPhysicalPfHandlerToRing3,
+                                ("i=%u pfnPfHandler=%p %s\n", i, pTypeR0->pfnPfHandler, pTypeR0->pszDesc));
+
+                /* Unused of bad ring-3 entry, make it and the ring-0 one harmless. */
+                pTypeR3->enmKind         = PGMPHYSHANDLERKIND_END;
+                pTypeR3->fRing0DevInsIdx = false;
+                pTypeR3->fKeepPgmLock    = false;
+                pTypeR3->uState          = 0;
+                break;
+        }
+        pTypeR3->fRing0Enabled   = false;
+
+        /* Make sure the entry is harmless and goes to ring-3. */
+        pTypeR0->enmKind         = PGMPHYSHANDLERKIND_END;
+        pTypeR0->pfnHandler      = pgmR0HandlerPhysicalHandlerToRing3;
+        pTypeR0->pfnPfHandler    = pgmR0HandlerPhysicalPfHandlerToRing3;
+        pTypeR0->fRing0DevInsIdx = false;
+        pTypeR0->fKeepPgmLock    = false;
+        pTypeR0->uState          = 0;
+        pTypeR0->pszDesc         = "invalid";
+    }
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) PGMR0CleanupVM(PGVM pGVM)
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        if (pGVM->pgmr0.s.ahPoolMapObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMapObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+        }
+
+        if (pGVM->pgmr0.s.ahPoolMemObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMemObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        }
+    }
+
+    if (pGVM->pgmr0.s.hPhysHandlerMapObj != NIL_RTR0MEMOBJ)
+    {
+        int rc = RTR0MemObjFree(pGVM->pgmr0.s.hPhysHandlerMapObj, true /*fFreeMappings*/);
+        AssertRC(rc);
+        pGVM->pgmr0.s.hPhysHandlerMapObj = NIL_RTR0MEMOBJ;
+    }
+
+    if (pGVM->pgmr0.s.hPhysHandlerMemObj != NIL_RTR0MEMOBJ)
+    {
+        int rc = RTR0MemObjFree(pGVM->pgmr0.s.hPhysHandlerMemObj, true /*fFreeMappings*/);
+        AssertRC(rc);
+        pGVM->pgmr0.s.hPhysHandlerMemObj = NIL_RTR0MEMOBJ;
+    }
+
+    if (RTCritSectIsInitialized(&pGVM->pgmr0.s.PoolGrowCritSect))
+        RTCritSectDelete(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
+
+
+/**
+ * Worker function for PGMR3PhysAllocateHandyPages and pgmPhysEnsureHandyPage.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FF cleared.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is set in this case.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ * @param   fRing3      Set if the caller is ring-3.  Determins whether to
+ *                      return VINF_EM_NO_MEMORY or not.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+int pgmR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu, bool fRing3)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    Assert(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf());
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+
+    /*
+     * Check for error injection.
+     */
+    if (RT_LIKELY(!pGVM->pgm.s.fErrInjHandyPages))
+    { /* likely */ }
+    else
+        return VERR_NO_MEMORY;
+
+    /*
+     * Try allocate a full set of handy pages.
+     */
+    uint32_t const iFirst = pGVM->pgm.s.cHandyPages;
+    AssertMsgReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), ("%#x\n", iFirst), VERR_PGM_HANDY_PAGE_IPE);
+
+    uint32_t const cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
+    if (!cPages)
+        return VINF_SUCCESS;
+
+    int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, cPages, &pGVM->pgm.s.aHandyPages[iFirst]);
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t const cHandyPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages); /** @todo allow allocating less... */
+        pGVM->pgm.s.cHandyPages = cHandyPages;
+        VM_FF_CLEAR(pGVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_CLEAR(pGVM, VM_FF_PGM_NO_MEMORY);
+
+#ifdef VBOX_STRICT
+        for (uint32_t i = 0; i < cHandyPages; i++)
+        {
+            Assert(pGVM->pgm.s.aHandyPages[i].idPage != NIL_GMM_PAGEID);
+            Assert(pGVM->pgm.s.aHandyPages[i].idPage <= GMM_PAGEID_LAST);
+            Assert(pGVM->pgm.s.aHandyPages[i].idSharedPage == NIL_GMM_PAGEID);
+            Assert(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
+            Assert(!(pGVM->pgm.s.aHandyPages[i].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
+        }
+#endif
+
+        /*
+         * Clear the pages.
+         */
+        for (uint32_t iPage = iFirst; iPage < cHandyPages; iPage++)
+        {
+            PGMMPAGEDESC pPage = &pGVM->pgm.s.aHandyPages[iPage];
+            if (!pPage->fZeroed)
+            {
+                void *pv = NULL;
+#ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+                rc = SUPR0HCPhysToVirt(pPage->HCPhysGCPhys, &pv);
+#else
+                rc = GMMR0PageIdToVirt(pGVM, pPage->idPage, &pv);
+#endif
+                AssertMsgRCReturn(rc, ("idPage=%#x HCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc), rc);
+
+                RT_BZERO(pv, GUEST_PAGE_SIZE);
+                pPage->fZeroed = true;
+            }
+#ifdef VBOX_STRICT
+            else
+            {
+                void *pv = NULL;
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+                rc = SUPR0HCPhysToVirt(pPage->HCPhysGCPhys, &pv);
+# else
+                rc = GMMR0PageIdToVirt(pGVM, pPage->idPage, &pv);
+# endif
+                AssertMsgRCReturn(rc, ("idPage=%#x HCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc), rc);
+                AssertReturn(ASMMemIsZero(pv, GUEST_PAGE_SIZE), VERR_PGM_HANDY_PAGE_IPE);
+            }
+#endif
+            Log3(("PGMR0PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
+        }
+    }
+    else
+    {
+        /*
+         * We should never get here unless there is a genuine shortage of
+         * memory (or some internal error). Flag the error so the VM can be
+         * suspended ASAP and the user informed. If we're totally out of
+         * handy pages we will return failure.
+         */
+        /* Report the failure. */
+        LogRel(("PGM: Failed to procure handy pages; rc=%Rrc cHandyPages=%#x\n"
+                "     cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
+                rc, pGVM->pgm.s.cHandyPages,
+                pGVM->pgm.s.cAllPages, pGVM->pgm.s.cPrivatePages, pGVM->pgm.s.cSharedPages, pGVM->pgm.s.cZeroPages));
+
+        GMMMEMSTATSREQ Stats = { { SUPVMMR0REQHDR_MAGIC, sizeof(Stats) }, 0, 0, 0, 0, 0 };
+        if (RT_SUCCESS(GMMR0QueryMemoryStatsReq(pGVM, idCpu, &Stats)))
+            LogRel(("GMM: Statistics:\n"
+                    "     Allocated pages: %RX64\n"
+                    "     Free      pages: %RX64\n"
+                    "     Shared    pages: %RX64\n"
+                    "     Maximum   pages: %RX64\n"
+                    "     Ballooned pages: %RX64\n",
+                    Stats.cAllocPages, Stats.cFreePages, Stats.cSharedPages, Stats.cMaxPages, Stats.cBalloonedPages));
+
+        if (   rc != VERR_NO_MEMORY
+            && rc != VERR_NO_PHYS_MEMORY
+            && rc != VERR_LOCK_FAILED)
+            for (uint32_t iPage = 0; iPage < RT_ELEMENTS(pGVM->pgm.s.aHandyPages); iPage++)
+                LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
+                        iPage, pGVM->pgm.s.aHandyPages[iPage].HCPhysGCPhys, pGVM->pgm.s.aHandyPages[iPage].idPage,
+                        pGVM->pgm.s.aHandyPages[iPage].idSharedPage));
+
+        /* Set the FFs and adjust rc. */
+        VM_FF_SET(pGVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_SET(pGVM, VM_FF_PGM_NO_MEMORY);
+        if (!fRing3)
+            if (   rc == VERR_NO_MEMORY
+                || rc == VERR_NO_PHYS_MEMORY
+                || rc == VERR_LOCK_FAILED
+                || rc == VERR_MAP_FAILED)
+                rc = VINF_EM_NO_MEMORY;
+    }
+
+    LogFlow(("PGMR0PhysAllocateHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
+    return rc;
+}
+
+
+/**
+ * Worker function for PGMR3PhysAllocateHandyPages / VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FF cleared.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is set in this case.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+
+    /*
+     * Enter the PGM lock and call the worker.
+     */
+    int rc = PGM_LOCK(pGVM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmR0PhysAllocateHandyPages(pGVM, idCpu, true /*fRing3*/);
+        PGM_UNLOCK(pGVM);
+    }
+    return rc;
+}
+
+
+/**
+ * Flushes any changes pending in the handy page array.
+ *
+ * It is very important that this gets done when page sharing is enabled.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FF cleared.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysFlushHandyPages(PGVM pGVM, VMCPUID idCpu)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID); /* caller already checked this, but just to be sure. */
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+
+    /*
+     * Try allocate a full set of handy pages.
+     */
+    uint32_t iFirst = pGVM->pgm.s.cHandyPages;
+    AssertReturn(iFirst <= RT_ELEMENTS(pGVM->pgm.s.aHandyPages), VERR_PGM_HANDY_PAGE_IPE);
+    uint32_t cPages = RT_ELEMENTS(pGVM->pgm.s.aHandyPages) - iFirst;
+    if (!cPages)
+        return VINF_SUCCESS;
+    int rc = GMMR0AllocateHandyPages(pGVM, idCpu, cPages, 0, &pGVM->pgm.s.aHandyPages[iFirst]);
+
+    LogFlow(("PGMR0PhysFlushHandyPages: cPages=%d rc=%Rrc\n", cPages, rc));
+    return rc;
+}
+
+
+/**
+ * Allocate a large page at @a GCPhys.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ * @param   GCPhys      The guest physical address of the page.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+int pgmR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys)
+{
+    STAM_PROFILE_START(&pGVM->pgm.s.Stats.StatLargePageAlloc2, a);
+    PGM_LOCK_ASSERT_OWNER_EX(pGVM, &pGVM->aCpus[idCpu]);
+
+    /*
+     * Allocate a large page.
+     */
+    RTHCPHYS HCPhys = NIL_GMMPAGEDESC_PHYS;
+    uint32_t idPage = NIL_GMM_PAGEID;
+
+    if (true) /** @todo pre-allocate 2-3 pages on the allocation thread. */
+    {
+        uint64_t const nsAllocStart = RTTimeNanoTS();
+        if (nsAllocStart < pGVM->pgm.s.nsLargePageRetry)
+        {
+            LogFlowFunc(("returns VERR_TRY_AGAIN - %RU64 ns left of hold off period\n", pGVM->pgm.s.nsLargePageRetry - nsAllocStart));
+            return VERR_TRY_AGAIN;
+        }
+
+        int const rc = GMMR0AllocateLargePage(pGVM, idCpu, _2M, &idPage, &HCPhys);
+
+        uint64_t const nsAllocEnd = RTTimeNanoTS();
+        uint64_t const cNsElapsed = nsAllocEnd - nsAllocStart;
+        STAM_REL_PROFILE_ADD_PERIOD(&pGVM->pgm.s.StatLargePageAlloc, cNsElapsed);
+        if (cNsElapsed < RT_NS_100MS)
+            pGVM->pgm.s.cLargePageLongAllocRepeats = 0;
+        else
+        {
+            /* If a large page allocation takes more than 100ms back off for a
+               while so the host OS can reshuffle memory and make some more large
+               pages available.  However if it took over a second, just disable it. */
+            STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageOverflow);
+            pGVM->pgm.s.cLargePageLongAllocRepeats++;
+            if (cNsElapsed > RT_NS_1SEC)
+            {
+                LogRel(("PGMR0PhysAllocateLargePage: Disabling large pages after %'RU64 ns allocation time.\n", cNsElapsed));
+                PGMSetLargePageUsage(pGVM, false);
+            }
+            else
+            {
+                Log(("PGMR0PhysAllocateLargePage: Suspending large page allocations for %u sec after %'RU64 ns allocation time.\n",
+                     30 * pGVM->pgm.s.cLargePageLongAllocRepeats, cNsElapsed));
+                pGVM->pgm.s.nsLargePageRetry = nsAllocEnd + RT_NS_30SEC * pGVM->pgm.s.cLargePageLongAllocRepeats;
+            }
+        }
+
+        if (RT_FAILURE(rc))
+        {
+            Log(("PGMR0PhysAllocateLargePage: Failed: %Rrc\n", rc));
+            STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageAllocFailed);
+            if (rc == VERR_NOT_SUPPORTED)
+            {
+                LogRel(("PGM: Disabling large pages because of VERR_NOT_SUPPORTED status.\n"));
+                PGMSetLargePageUsage(pGVM, false);
+            }
+            return rc;
+        }
+    }
+
+    STAM_PROFILE_STOP_START(&pGVM->pgm.s.Stats.StatLargePageAlloc2, &pGVM->pgm.s.Stats.StatLargePageSetup, a);
+
+    /*
+     * Enter the pages into PGM.
+     */
+    bool         fFlushTLBs = false;
+    VBOXSTRICTRC rc         = VINF_SUCCESS;
+    unsigned     cLeft      = _2M / GUEST_PAGE_SIZE;
+    while (cLeft-- > 0)
+    {
+        PPGMPAGE const pPage = pgmPhysGetPage(pGVM, GCPhys);
+        AssertReturn(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM && PGM_PAGE_IS_ZERO(pPage), VERR_PGM_UNEXPECTED_PAGE_STATE);
+
+        /* Make sure there are no zero mappings. */
+        uint16_t const u16Tracking = PGM_PAGE_GET_TRACKING(pPage);
+        if (u16Tracking == 0)
+            Assert(PGM_PAGE_GET_PTE_INDEX(pPage) == 0);
+        else
+        {
+            STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageZeroEvict);
+            VBOXSTRICTRC rc3 = pgmPoolTrackUpdateGCPhys(pGVM, GCPhys, pPage, true /*fFlushPTEs*/, &fFlushTLBs);
+            Log(("PGMR0PhysAllocateLargePage: GCPhys=%RGp: tracking=%#x rc3=%Rrc\n", GCPhys, u16Tracking, VBOXSTRICTRC_VAL(rc3)));
+            if (rc3 != VINF_SUCCESS && rc == VINF_SUCCESS)
+                rc = rc3; /** @todo not perfect... */
+            PGM_PAGE_SET_PTE_INDEX(pGVM, pPage, 0);
+            PGM_PAGE_SET_TRACKING(pGVM, pPage, 0);
+        }
+
+        /* Setup the new page. */
+        PGM_PAGE_SET_HCPHYS(pGVM, pPage, HCPhys);
+        PGM_PAGE_SET_STATE(pGVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+        PGM_PAGE_SET_PDE_TYPE(pGVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
+        PGM_PAGE_SET_PAGEID(pGVM, pPage, idPage);
+        Log3(("PGMR0PhysAllocateLargePage: GCPhys=%RGp: idPage=%#x HCPhys=%RGp (old tracking=%#x)\n",
+              GCPhys, idPage, HCPhys, u16Tracking));
+
+        /* advance */
+        idPage++;
+        HCPhys += GUEST_PAGE_SIZE;
+        GCPhys += GUEST_PAGE_SIZE;
+    }
+
+    STAM_COUNTER_ADD(&pGVM->pgm.s.Stats.StatRZPageReplaceZero, _2M / GUEST_PAGE_SIZE);
+    pGVM->pgm.s.cZeroPages    -= _2M / GUEST_PAGE_SIZE;
+    pGVM->pgm.s.cPrivatePages += _2M / GUEST_PAGE_SIZE;
+
+    /*
+     * Flush all TLBs.
+     */
+    if (!fFlushTLBs)
+    { /* likely as we shouldn't normally map zero pages */ }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pGVM->pgm.s.StatLargePageTlbFlush);
+        PGM_INVL_ALL_VCPU_TLBS(pGVM);
+    }
+    /** @todo this is a little expensive (~3000 ticks) since we'll have to
+     * invalidate everything.  Add a version to the TLB? */
+    pgmPhysInvalidatePageMapTLB(pGVM);
+    IEMTlbInvalidateAllPhysicalAllCpus(pGVM, idCpu);
+
+    STAM_PROFILE_STOP(&pGVM->pgm.s.Stats.StatLargePageSetup, a);
+#if 0 /** @todo returning info statuses here might not be a great idea... */
+    LogFlow(("PGMR0PhysAllocateLargePage: returns %Rrc\n", VBOXSTRICTRC_VAL(rc) ));
+    return VBOXSTRICTRC_TODO(rc);
+#else
+    LogFlow(("PGMR0PhysAllocateLargePage: returns VINF_SUCCESS (rc=%Rrc)\n", VBOXSTRICTRC_VAL(rc) ));
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Allocate a large page at @a GCPhys.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   idCpu       The ID of the calling EMT.
+ * @param   GCPhys      The guest physical address of the page.
+ *
+ * @thread  EMT(idCpu)
+ *
+ * @remarks Must be called from within the PGM critical section. The caller
+ *          must clear the new pages.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate inputs.
+     */
+    AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_NOT_OWNER);
+
+    int rc = PGM_LOCK(pGVM);
+    AssertRCReturn(rc, rc);
+
+    /* The caller might have done this already, but since we're ring-3 callable we
+       need to make sure everything is fine before starting the allocation here. */
+    for (unsigned i = 0; i < _2M / GUEST_PAGE_SIZE; i++)
+    {
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pGVM, GCPhys + i * GUEST_PAGE_SIZE, &pPage);
+        AssertRCReturnStmt(rc, PGM_UNLOCK(pGVM), rc);
+        AssertReturnStmt(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM, PGM_UNLOCK(pGVM), VERR_PGM_PHYS_NOT_RAM);
+        AssertReturnStmt(PGM_PAGE_IS_ZERO(pPage), PGM_UNLOCK(pGVM), VERR_PGM_UNEXPECTED_PAGE_STATE);
+    }
+
+    /*
+     * Call common code.
+     */
+    rc = pgmR0PhysAllocateLargePage(pGVM, idCpu, GCPhys);
+
+    PGM_UNLOCK(pGVM);
+    return rc;
+}
+
+
+/**
+ * Locate a MMIO2 range.
+ *
+ * @returns Pointer to the MMIO2 range.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance owning the region.
+ * @param   hMmio2      Handle to look up.
+ */
+DECLINLINE(PPGMREGMMIO2RANGE) pgmR0PhysMmio2Find(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
+{
+    /*
+     * We use the lookup table here as list walking is tedious in ring-0 when using
+     * ring-3 pointers and this probably will require some kind of refactoring anyway.
+     */
+    if (hMmio2 <= RT_ELEMENTS(pGVM->pgm.s.apMmio2RangesR0) && hMmio2 != 0)
+    {
+        PPGMREGMMIO2RANGE pCur = pGVM->pgm.s.apMmio2RangesR0[hMmio2 - 1];
+        if (pCur && pCur->pDevInsR3 == pDevIns->pDevInsForR3)
+        {
+            Assert(pCur->idMmio2 == hMmio2);
+            return pCur;
+        }
+        Assert(!pCur);
+    }
+    return NULL;
+}
+
+
+/**
+ * Worker for PDMDEVHLPR0::pfnMmio2SetUpContext.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance.
+ * @param   hMmio2      The MMIO2 region to map into ring-0 address space.
+ * @param   offSub      The offset into the region.
+ * @param   cbSub       The size of the mapping, zero meaning all the rest.
+ * @param   ppvMapping  Where to return the ring-0 mapping address.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysMMIO2MapKernel(PGVM pGVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
+                                            size_t offSub, size_t cbSub, void **ppvMapping)
+{
+    AssertReturn(!(offSub & HOST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
+    AssertReturn(!(cbSub  & HOST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
+
+    /*
+     * Translate hRegion into a range pointer.
+     */
+    PPGMREGMMIO2RANGE pFirstRegMmio = pgmR0PhysMmio2Find(pGVM, pDevIns, hMmio2);
+    AssertReturn(pFirstRegMmio, VERR_NOT_FOUND);
+#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    uint8_t * const pvR0  = (uint8_t *)pFirstRegMmio->pvR0;
+#else
+    RTR3PTR const  pvR3   = pFirstRegMmio->pvR3;
+#endif
+    RTGCPHYS const cbReal = pFirstRegMmio->cbReal;
+    pFirstRegMmio = NULL;
+    ASMCompilerBarrier();
+
+    AssertReturn(offSub < cbReal, VERR_OUT_OF_RANGE);
+    if (cbSub == 0)
+        cbSub = cbReal - offSub;
+    else
+        AssertReturn(cbSub < cbReal && cbSub + offSub <= cbReal, VERR_OUT_OF_RANGE);
+
+    /*
+     * Do the mapping.
+     */
+#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    AssertPtr(pvR0);
+    *ppvMapping = pvR0 + offSub;
+    return VINF_SUCCESS;
+#else
+    return SUPR0PageMapKernel(pGVM->pSession, pvR3, (uint32_t)offSub, (uint32_t)cbSub, 0 /*fFlags*/, ppvMapping);
+#endif
+}
+
+
+/**
+ * This is called during PGMR3Init to init the physical access handler allocator
+ * and tree.
+ *
+ * @returns VBox status code.
+ * @param   pGVM        Pointer to the global VM structure.
+ * @param   cEntries    Desired number of physical access handlers to reserve
+ *                      space for (will be adjusted).
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PGMR0PhysHandlerInitReqHandler(PGVM pGVM, uint32_t cEntries)
+{
+    /*
+     * Validate the input and state.
+     */
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE); /** @todo ring-0 safe state check. */
+
+    AssertReturn(pGVM->pgmr0.s.PhysHandlerAllocator.m_paNodes == NULL, VERR_WRONG_ORDER);
+    AssertReturn(pGVM->pgm.s.PhysHandlerAllocator.m_paNodes == NULL, VERR_WRONG_ORDER);
+
+    AssertLogRelMsgReturn(cEntries <= _64K, ("%#x\n", cEntries), VERR_OUT_OF_RANGE);
+
+    /*
+     * Calculate the table size and allocate it.
+     */
+    uint32_t       cbTreeAndBitmap = 0;
+    uint32_t const cbTotalAligned  = pgmHandlerPhysicalCalcTableSizes(&cEntries, &cbTreeAndBitmap);
+    RTR0MEMOBJ     hMemObj         = NIL_RTR0MEMOBJ;
+    rc = RTR0MemObjAllocPage(&hMemObj, cbTotalAligned, false);
+    if (RT_SUCCESS(rc))
+    {
+        RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
+        rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, RTR0ProcHandleSelf());
+        if (RT_SUCCESS(rc))
+        {
+            uint8_t *pb = (uint8_t *)RTR0MemObjAddress(hMemObj);
+            if (!RTR0MemObjWasZeroInitialized(hMemObj))
+                RT_BZERO(pb, cbTotalAligned);
+
+            pGVM->pgmr0.s.PhysHandlerAllocator.initSlabAllocator(cEntries, (PPGMPHYSHANDLER)&pb[cbTreeAndBitmap],
+                                                                 (uint64_t *)&pb[sizeof(PGMPHYSHANDLERTREE)]);
+            pGVM->pgmr0.s.pPhysHandlerTree = (PPGMPHYSHANDLERTREE)pb;
+            pGVM->pgmr0.s.pPhysHandlerTree->initWithAllocator(&pGVM->pgmr0.s.PhysHandlerAllocator);
+            pGVM->pgmr0.s.hPhysHandlerMemObj = hMemObj;
+            pGVM->pgmr0.s.hPhysHandlerMapObj = hMapObj;
+
+            AssertCompile(sizeof(pGVM->pgm.s.PhysHandlerAllocator) == sizeof(pGVM->pgmr0.s.PhysHandlerAllocator));
+            RTR3PTR R3Ptr = RTR0MemObjAddressR3(hMapObj);
+            pGVM->pgm.s.pPhysHandlerTree                    = R3Ptr;
+            pGVM->pgm.s.PhysHandlerAllocator.m_paNodes      = R3Ptr + cbTreeAndBitmap;
+            pGVM->pgm.s.PhysHandlerAllocator.m_pbmAlloc     = R3Ptr + sizeof(PGMPHYSHANDLERTREE);
+            pGVM->pgm.s.PhysHandlerAllocator.m_cNodes       = cEntries;
+            pGVM->pgm.s.PhysHandlerAllocator.m_cErrors      = 0;
+            pGVM->pgm.s.PhysHandlerAllocator.m_idxAllocHint = 0;
+            pGVM->pgm.s.PhysHandlerAllocator.m_uPadding     = 0;
+            return VINF_SUCCESS;
+        }
+
+        RTR0MemObjFree(hMemObj, true /*fFreeMappings*/);
+    }
+    return rc;
+}
+
+
+/**
+ * Updates a physical access handler type with ring-0 callback functions.
+ *
+ * The handler type must first have been registered in ring-3.
+ *
+ * @returns VBox status code.
+ * @param   pGVM            The global (ring-0) VM structure.
+ * @param   enmKind         The kind of access handler.
+ * @param   fFlags          PGMPHYSHANDLER_F_XXX
+ * @param   pfnHandler      Pointer to the ring-0 handler callback.
+ * @param   pfnPfHandler    Pointer to the ring-0 \#PF handler callback.
+ *                          callback.  Can be NULL (not recommended though).
+ * @param   pszDesc         The type description.
+ * @param   hType           The handle to do ring-0 callback registrations for.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PGMR0HandlerPhysicalTypeSetUpContext(PGVM pGVM, PGMPHYSHANDLERKIND enmKind, uint32_t fFlags,
+                                                         PFNPGMPHYSHANDLER pfnHandler, PFNPGMRZPHYSPFHANDLER pfnPfHandler,
+                                                         const char *pszDesc, PGMPHYSHANDLERTYPE hType)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnHandler, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnPfHandler, VERR_INVALID_POINTER);
+
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(   enmKind == PGMPHYSHANDLERKIND_WRITE
+                 || enmKind == PGMPHYSHANDLERKIND_ALL
+                 || enmKind == PGMPHYSHANDLERKIND_MMIO,
+                 VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!(fFlags & ~PGMPHYSHANDLER_F_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_FLAGS);
+
+    PPGMPHYSHANDLERTYPEINTR0 const pTypeR0 = &pGVM->pgmr0.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK];
+    AssertMsgReturn(hType == pTypeR0->hType, ("%#RX64, expected=%#RX64\n", hType, pTypeR0->hType), VERR_INVALID_HANDLE);
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == RT_ELEMENTS(pGVM->pgm.s.aPhysHandlerTypes));
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.aPhysHandlerTypes) == PGMPHYSHANDLERTYPE_IDX_MASK + 1);
+    AssertReturn(pTypeR0->enmKind == PGMPHYSHANDLERKIND_INVALID, VERR_ALREADY_INITIALIZED);
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+    VM_ASSERT_STATE_RETURN(pGVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE); /** @todo ring-0 safe state check. */
+
+    PPGMPHYSHANDLERTYPEINTR3 const pTypeR3 = &pGVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK];
+    AssertMsgReturn(pTypeR3->enmKind == enmKind,
+                    ("%#x: %d, expected %d\n", hType, pTypeR3->enmKind, enmKind),
+                    VERR_INVALID_HANDLE);
+    AssertMsgReturn(pTypeR3->fKeepPgmLock == RT_BOOL(fFlags & PGMPHYSHANDLER_F_KEEP_PGM_LOCK),
+                    ("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fKeepPgmLock, fFlags),
+                    VERR_INVALID_HANDLE);
+    AssertMsgReturn(pTypeR3->fRing0DevInsIdx == RT_BOOL(fFlags & PGMPHYSHANDLER_F_R0_DEVINS_IDX),
+                    ("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fRing0DevInsIdx, fFlags),
+                    VERR_INVALID_HANDLE);
+    AssertMsgReturn(pTypeR3->fNotInHm == RT_BOOL(fFlags & PGMPHYSHANDLER_F_NOT_IN_HM),
+                    ("%#x: %d, fFlags=%#x\n", hType, pTypeR3->fNotInHm, fFlags),
+                    VERR_INVALID_HANDLE);
+
+    /*
+     * Update the entry.
+     */
+    pTypeR0->enmKind          = enmKind;
+    pTypeR0->uState           = enmKind == PGMPHYSHANDLERKIND_WRITE
+                              ? PGM_PAGE_HNDL_PHYS_STATE_WRITE : PGM_PAGE_HNDL_PHYS_STATE_ALL;
+    pTypeR0->fKeepPgmLock     = RT_BOOL(fFlags & PGMPHYSHANDLER_F_KEEP_PGM_LOCK);
+    pTypeR0->fRing0DevInsIdx  = RT_BOOL(fFlags & PGMPHYSHANDLER_F_R0_DEVINS_IDX);
+    pTypeR0->fNotInHm         = RT_BOOL(fFlags & PGMPHYSHANDLER_F_NOT_IN_HM);
+    pTypeR0->pfnHandler       = pfnHandler;
+    pTypeR0->pfnPfHandler     = pfnPfHandler;
+    pTypeR0->pszDesc          = pszDesc;
+
+    pTypeR3->fRing0Enabled    = true;
+
+    LogFlow(("PGMR0HandlerPhysicalTypeRegister: hType=%#x: enmKind=%d fFlags=%#x pfnHandler=%p pfnPfHandler=%p pszDesc=%s\n",
+             hType, enmKind, fFlags, pfnHandler, pfnPfHandler, pszDesc));
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+/* Interface sketch.  The interface belongs to a global PCI pass-through
+   manager.  It shall use the global VM handle, not the user VM handle to
+   store the per-VM info (domain) since that is all ring-0 stuff, thus
+   passing pGVM here.  I've tentitively prefixed the functions 'GPciRawR0',
+   we can discuss the PciRaw code re-organtization when I'm back from
+   vacation.
+
+   I've implemented the initial IOMMU set up below.  For things to work
+   reliably, we will probably need add a whole bunch of checks and
+   GPciRawR0GuestPageUpdate call to the PGM code.  For the present,
+   assuming nested paging (enforced) and prealloc (enforced), no
+   ballooning (check missing), page sharing (check missing) or live
+   migration (check missing), it might work fine.  At least if some
+   VM power-off hook is present and can tear down the IOMMU page tables. */
+
+/**
+ * Tells the global PCI pass-through manager that we are about to set up the
+ * guest page to host page mappings for the specfied VM.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageBeginAssignments(PGVM pGVM)
+{
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns a host page mapping for a guest page.
+ *
+ * This is only used when setting up the mappings, i.e. between
+ * GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The address of the host page (page aligned).
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageAssign(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
+{
+    AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+    AssertReturn(!(HCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+
+    if (pGVM->rawpci.s.pfnContigMemInfo)
+        /** @todo what do we do on failure? */
+        pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, HCPhys, GCPhys, HOST_PAGE_SIZE, PCIRAW_MEMINFO_MAP);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Indicates that the specified guest page doesn't exists but doesn't have host
+ * page mapping we trust PCI pass-through with.
+ *
+ * This is only used when setting up the mappings, i.e. between
+ * GPciRawR0GuestPageBeginAssignments and GPciRawR0GuestPageEndAssignments.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The address of the host page (page aligned).
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageUnassign(PGVM pGVM, RTGCPHYS GCPhys)
+{
+    AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
+
+    if (pGVM->rawpci.s.pfnContigMemInfo)
+        /** @todo what do we do on failure? */
+        pGVM->rawpci.s.pfnContigMemInfo(&pGVM->rawpci.s, 0, GCPhys, HOST_PAGE_SIZE, PCIRAW_MEMINFO_UNMAP);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tells the global PCI pass-through manager that we have completed setting up
+ * the guest page to host page mappings for the specfied VM.
+ *
+ * This complements GPciRawR0GuestPageBeginAssignments and will be called even
+ * if some page assignment failed.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The ring-0 VM structure.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageEndAssignments(PGVM pGVM)
+{
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tells the global PCI pass-through manager that a guest page mapping has
+ * changed after the initial setup.
+ *
+ * @returns VBox status code.
+ * @param   pGVM                The ring-0 VM structure.
+ * @param   GCPhys              The address of the guest page (page aligned).
+ * @param   HCPhys              The new host page address or NIL_RTHCPHYS if
+ *                              now unassigned.
+ */
+VMMR0_INT_DECL(int) GPciRawR0GuestPageUpdate(PGVM pGVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys)
+{
+    AssertReturn(!(GCPhys & HOST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_4);
+    AssertReturn(!(HCPhys & HOST_PAGE_OFFSET_MASK) || HCPhys == NIL_RTHCPHYS, VERR_INTERNAL_ERROR_4);
+    NOREF(pGVM);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_PCI_PASSTHROUGH */
+
+
+/**
+ * Sets up the IOMMU when raw PCI device is enabled.
+ *
+ * @note    This is a hack that will probably be remodelled and refined later!
+ *
+ * @returns VBox status code.
+ *
+ * @param   pGVM                The global (ring-0) VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0PhysSetupIoMmu(PGVM pGVM)
+{
+    int rc = GVMMR0ValidateGVM(pGVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+    if (pGVM->pgm.s.fPciPassthrough)
+    {
+        /*
+         * The Simplistic Approach - Enumerate all the pages and call tell the
+         * IOMMU about each of them.
+         */
+        PGM_LOCK_VOID(pGVM);
+        rc = GPciRawR0GuestPageBeginAssignments(pGVM);
+        if (RT_SUCCESS(rc))
+        {
+            for (PPGMRAMRANGE pRam = pGVM->pgm.s.pRamRangesXR0; RT_SUCCESS(rc) && pRam; pRam = pRam->pNextR0)
+            {
+                PPGMPAGE    pPage  = &pRam->aPages[0];
+                RTGCPHYS    GCPhys = pRam->GCPhys;
+                uint32_t    cLeft  = pRam->cb >> GUEST_PAGE_SHIFT;
+                while (cLeft-- > 0)
+                {
+                    /* Only expose pages that are 100% safe for now. */
+                    if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+                        && PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED
+                        && !PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+                        rc = GPciRawR0GuestPageAssign(pGVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage));
+                    else
+                        rc = GPciRawR0GuestPageUnassign(pGVM, GCPhys);
+
+                    /* next */
+                    pPage++;
+                    GCPhys += HOST_PAGE_SIZE;
+                }
+            }
+
+            int rc2 = GPciRawR0GuestPageEndAssignments(pGVM);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        PGM_UNLOCK(pGVM);
+    }
+    else
+#endif
+        rc = VERR_NOT_SUPPORTED;
+    return rc;
+}
+
+
+/**
+ * \#PF Handler for nested paging.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   enmShwPagingMode    Paging mode for the nested page tables.
+ * @param   uErr                The trap error code.
+ * @param   pCtx                Pointer to the register context for the CPU.
+ * @param   GCPhysFault         The fault address.
+ */
+VMMR0DECL(int) PGMR0Trap0eHandlerNestedPaging(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode, RTGCUINT uErr,
+                                              PCPUMCTX pCtx, RTGCPHYS GCPhysFault)
+{
+    int rc;
+
+    LogFlow(("PGMTrap0eHandler: uErr=%RGx GCPhysFault=%RGp eip=%RGv\n", uErr, GCPhysFault, (RTGCPTR)pCtx->rip));
+    STAM_PROFILE_START(&pGVCpu->pgm.s.StatRZTrap0e, a);
+    STAM_STATS({ pGVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
+
+    /* AMD uses the host's paging mode; Intel has a single mode (EPT). */
+    AssertMsg(   enmShwPagingMode == PGMMODE_32_BIT || enmShwPagingMode == PGMMODE_PAE      || enmShwPagingMode == PGMMODE_PAE_NX
+              || enmShwPagingMode == PGMMODE_AMD64  || enmShwPagingMode == PGMMODE_AMD64_NX || enmShwPagingMode == PGMMODE_EPT,
+              ("enmShwPagingMode=%d\n", enmShwPagingMode));
+
+    /* Reserved shouldn't end up here. */
+    Assert(!(uErr & X86_TRAP_PF_RSVD));
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Error code stats.
+     */
+    if (uErr & X86_TRAP_PF_US)
+    {
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
+        else
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
+    }
+    else
+    {   /* Supervisor */
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
+    }
+#endif
+
+    /*
+     * Call the worker.
+     *
+     * Note! We pretend the guest is in protected mode without paging, so we
+     *       can use existing code to build the nested page tables.
+     */
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+    bool fLockTaken = false;
+    switch (enmShwPagingMode)
+    {
+        case PGMMODE_32_BIT:
+            rc = PGM_BTH_NAME_32BIT_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            rc = PGM_BTH_NAME_PAE_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            rc = PGM_BTH_NAME_AMD64_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
+            break;
+        case PGMMODE_EPT:
+            rc = PGM_BTH_NAME_EPT_PROT(Trap0eHandler)(pGVCpu, uErr, pCtx, GCPhysFault, &fLockTaken);
+            break;
+        default:
+            AssertFailed();
+            rc = VERR_INVALID_PARAMETER;
+            break;
+    }
+    if (fLockTaken)
+    {
+        PGM_LOCK_ASSERT_OWNER(pGVM);
+        PGM_UNLOCK(pGVM);
+    }
+
+    if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+        rc = VINF_SUCCESS;
+    /*
+     * Handle the case where we cannot interpret the instruction because we cannot get the guest physical address
+     * via its page tables, see @bugref{6043}.
+     */
+    else if (   rc == VERR_PAGE_NOT_PRESENT                 /* SMP only ; disassembly might fail. */
+             || rc == VERR_PAGE_TABLE_NOT_PRESENT           /* seen with UNI & SMP */
+             || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT   /* seen with SMP */
+             || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)     /* precaution */
+    {
+        Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGp error code %x (rip=%RGv)\n", rc, GCPhysFault, uErr, pCtx->rip));
+        /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about
+           single VCPU VMs though. */
+        rc = VINF_SUCCESS;
+    }
+
+    STAM_STATS({ if (!pGVCpu->pgmr0.s.pStatTrap0eAttributionR0)
+                    pGVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pGVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
+    STAM_PROFILE_STOP_EX(&pGVCpu->pgm.s.Stats.StatRZTrap0e, pGVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Nested \#PF Handler for nested-guest execution using nested paging.
+ *
+ * @returns Strict VBox status code (appropriate for trap handling and GC return).
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   uErr                The trap error code.
+ * @param   pCtx                Pointer to the register context for the CPU.
+ * @param   GCPhysNestedFault   The nested-guest physical address causing the fault.
+ * @param   fIsLinearAddrValid  Whether translation of a nested-guest linear address
+ *                              caused this fault. If @c false, GCPtrNestedFault
+ *                              must be 0.
+ * @param   GCPtrNestedFault    The nested-guest linear address that caused this
+ *                              fault.
+ * @param   pWalk               Where to store the SLAT walk result.
+ */
+VMMR0DECL(VBOXSTRICTRC) PGMR0NestedTrap0eHandlerNestedPaging(PGVMCPU pGVCpu, PGMMODE enmShwPagingMode, RTGCUINT uErr,
+                                                             PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
+                                                             bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk)
+{
+    Assert(enmShwPagingMode == PGMMODE_EPT);
+    NOREF(enmShwPagingMode);
+
+    bool fLockTaken;
+    VBOXSTRICTRC rcStrict = PGM_BTH_NAME_EPT_PROT(NestedTrap0eHandler)(pGVCpu, uErr, pCtx, GCPhysNestedFault,
+                                                                       fIsLinearAddrValid, GCPtrNestedFault, pWalk, &fLockTaken);
+    if (fLockTaken)
+    {
+        PGM_LOCK_ASSERT_OWNER(pGVCpu->CTX_SUFF(pVM));
+        PGM_UNLOCK(pGVCpu->CTX_SUFF(pVM));
+    }
+    Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE); /* This rc isn't used with Nested Paging and nested-EPT. */
+    return rcStrict;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * \#PF Handler for deliberate nested paging misconfiguration (/reserved bit)
+ * employed for MMIO pages.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pGVM                The global (ring-0) VM structure.
+ * @param   pGVCpu              The global (ring-0) CPU structure of the calling
+ *                              EMT.
+ * @param   enmShwPagingMode    Paging mode for the nested page tables.
+ * @param   pCtx                Pointer to the register context for the CPU.
+ * @param   GCPhysFault         The fault address.
+ * @param   uErr                The error code, UINT32_MAX if not available
+ *                              (VT-x).
+ */
+VMMR0DECL(VBOXSTRICTRC) PGMR0Trap0eHandlerNPMisconfig(PGVM pGVM, PGVMCPU pGVCpu, PGMMODE enmShwPagingMode,
+                                                      PCPUMCTX pCtx, RTGCPHYS GCPhysFault, uint32_t uErr)
+{
+#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+    STAM_PROFILE_START(&pGVCpu->CTX_SUFF(pStats)->StatR0NpMiscfg, a);
+    VBOXSTRICTRC rc;
+
+    /*
+     * Try lookup the all access physical handler for the address.
+     */
+    PGM_LOCK_VOID(pGVM);
+    PPGMPHYSHANDLER pHandler;
+    rc = pgmHandlerPhysicalLookup(pGVM, GCPhysFault, &pHandler);
+    if (RT_SUCCESS(rc))
+    {
+        PCPGMPHYSHANDLERTYPEINT pHandlerType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pGVM, pHandler);
+        if (RT_LIKELY(   pHandlerType->enmKind != PGMPHYSHANDLERKIND_WRITE
+                      && !pHandlerType->fNotInHm /*paranoia*/ ))
+        {
+            /*
+             * If the handle has aliases page or pages that have been temporarily
+             * disabled, we'll have to take a detour to make sure we resync them
+             * to avoid lots of unnecessary exits.
+             */
+            PPGMPAGE pPage;
+            if (   (   pHandler->cAliasedPages
+                    || pHandler->cTmpOffPages)
+                && (   (pPage = pgmPhysGetPage(pGVM, GCPhysFault)) == NULL
+                    || PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+               )
+            {
+                Log(("PGMR0Trap0eHandlerNPMisconfig: Resyncing aliases / tmp-off page at %RGp (uErr=%#x) %R[pgmpage]\n", GCPhysFault, uErr, pPage));
+                STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatR0NpMiscfgSyncPage);
+                rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /*cPages*/, enmShwPagingMode);
+                PGM_UNLOCK(pGVM);
+            }
+            else
+            {
+                if (pHandlerType->pfnPfHandler)
+                {
+                    uint64_t const uUser = !pHandlerType->fRing0DevInsIdx ? pHandler->uUser
+                                         : (uintptr_t)PDMDeviceRing0IdxToInstance(pGVM, pHandler->uUser);
+                    STAM_PROFILE_START(&pHandler->Stat, h);
+                    PGM_UNLOCK(pGVM);
+
+                    Log6(("PGMR0Trap0eHandlerNPMisconfig: calling %p(,%#x,,%RGp,%p)\n", pHandlerType->pfnPfHandler, uErr, GCPhysFault, uUser));
+                    rc = pHandlerType->pfnPfHandler(pGVM, pGVCpu, uErr == UINT32_MAX ? RTGCPTR_MAX : uErr, pCtx,
+                                                    GCPhysFault, GCPhysFault, uUser);
+
+                    STAM_PROFILE_STOP(&pHandler->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+                }
+                else
+                {
+                    PGM_UNLOCK(pGVM);
+                    Log(("PGMR0Trap0eHandlerNPMisconfig: %RGp (uErr=%#x) -> R3\n", GCPhysFault, uErr));
+                    rc = VINF_EM_RAW_EMULATE_INSTR;
+                }
+            }
+            STAM_PROFILE_STOP(&pGVCpu->pgm.s.Stats.StatR0NpMiscfg, a);
+            return rc;
+        }
+    }
+    else
+        AssertMsgReturn(rc == VERR_NOT_FOUND, ("%Rrc GCPhysFault=%RGp\n", VBOXSTRICTRC_VAL(rc), GCPhysFault), rc);
+
+    /*
+     * Must be out of sync, so do a SyncPage and restart the instruction.
+     *
+     * ASSUMES that ALL handlers are page aligned and covers whole pages
+     * (assumption asserted in PGMHandlerPhysicalRegisterEx).
+     */
+    Log(("PGMR0Trap0eHandlerNPMisconfig: Out of sync page at %RGp (uErr=%#x)\n", GCPhysFault, uErr));
+    STAM_COUNTER_INC(&pGVCpu->pgm.s.Stats.StatR0NpMiscfgSyncPage);
+    rc = pgmShwSyncNestedPageLocked(pGVCpu, GCPhysFault, 1 /*cPages*/, enmShwPagingMode);
+    PGM_UNLOCK(pGVM);
+
+    STAM_PROFILE_STOP(&pGVCpu->pgm.s.Stats.StatR0NpMiscfg, a);
+    return rc;
+
+#else
+    AssertLogRelFailed();
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif
+}
+